explorbot 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +93 -36
- package/dist/bin/explorbot-cli.js +71 -16
- package/dist/rules/rerunner/healing-approach.md +19 -0
- package/dist/src/action.js +8 -10
- package/dist/src/ai/historian.js +34 -3
- package/dist/src/ai/navigator.js +35 -28
- package/dist/src/ai/pilot.js +33 -9
- package/dist/src/ai/planner/session-dedup.js +3 -0
- package/dist/src/ai/planner/styles.js +3 -0
- package/dist/src/ai/planner.js +29 -10
- package/dist/src/ai/rerunner.js +472 -0
- package/dist/src/ai/researcher/cache.js +4 -3
- package/dist/src/ai/researcher/fingerprint-worker.js +7 -6
- package/dist/src/ai/researcher.js +3 -4
- package/dist/src/ai/rules.js +2 -2
- package/dist/src/ai/tools.js +2 -2
- package/dist/src/commands/add-rule-command.js +1 -2
- package/dist/src/commands/base-command.js +12 -0
- package/dist/src/commands/context-command.js +12 -5
- package/dist/src/commands/drill-command.js +0 -1
- package/dist/src/commands/explore-command.js +20 -5
- package/dist/src/commands/freesail-command.js +8 -22
- package/dist/src/commands/index.js +4 -0
- package/dist/src/commands/init-command.js +3 -3
- package/dist/src/commands/path-command.js +2 -1
- package/dist/src/commands/plan-command.js +37 -15
- package/dist/src/commands/rerun-command.js +42 -0
- package/dist/src/commands/research-command.js +10 -4
- package/dist/src/commands/runs-command.js +22 -0
- package/dist/src/commands/start-command.js +0 -1
- package/dist/src/commands/test-command.js +3 -3
- package/dist/src/components/App.js +8 -0
- package/dist/src/config.js +3 -0
- package/dist/src/explorbot.js +19 -0
- package/dist/src/explorer.js +2 -1
- package/dist/src/suite.js +115 -0
- package/dist/src/utils/html.js +2 -5
- package/dist/src/utils/rules-loader.js +33 -17
- package/dist/src/utils/test-files.js +103 -0
- package/package.json +3 -1
- package/rules/rerunner/healing-approach.md +19 -0
- package/src/action.ts +7 -9
- package/src/ai/historian.ts +37 -3
- package/src/ai/navigator.ts +35 -28
- package/src/ai/pilot.ts +33 -9
- package/src/ai/planner/session-dedup.ts +4 -0
- package/src/ai/planner/styles.ts +4 -0
- package/src/ai/planner.ts +28 -9
- package/src/ai/rerunner.ts +532 -0
- package/src/ai/researcher/cache.ts +4 -3
- package/src/ai/researcher/fingerprint-worker.ts +7 -13
- package/src/ai/researcher.ts +3 -4
- package/src/ai/rules.ts +2 -2
- package/src/ai/tools.ts +2 -2
- package/src/commands/add-rule-command.ts +1 -2
- package/src/commands/base-command.ts +13 -0
- package/src/commands/context-command.ts +12 -5
- package/src/commands/drill-command.ts +0 -1
- package/src/commands/explore-command.ts +21 -5
- package/src/commands/freesail-command.ts +6 -23
- package/src/commands/index.ts +4 -0
- package/src/commands/init-command.ts +3 -3
- package/src/commands/path-command.ts +2 -1
- package/src/commands/plan-command.ts +45 -16
- package/src/commands/rerun-command.ts +46 -0
- package/src/commands/research-command.ts +10 -4
- package/src/commands/runs-command.ts +27 -0
- package/src/commands/start-command.ts +0 -1
- package/src/commands/test-command.ts +3 -3
- package/src/components/App.tsx +8 -0
- package/src/config.ts +23 -0
- package/src/explorbot.ts +21 -0
- package/src/explorer.ts +3 -2
- package/src/suite.ts +135 -0
- package/src/utils/html.ts +1 -5
- package/src/utils/rules-loader.ts +35 -17
- package/src/utils/test-files.ts +122 -0
package/src/ai/planner.ts
CHANGED
|
@@ -23,6 +23,7 @@ import { findSimilarStateHash } from './researcher/cache.ts';
|
|
|
23
23
|
import type { Provider } from './provider.js';
|
|
24
24
|
import { hasFocusedSection } from './researcher/focus.ts';
|
|
25
25
|
import { POSSIBLE_SECTIONS, Researcher } from './researcher.ts';
|
|
26
|
+
import { Suite } from '../suite.ts';
|
|
26
27
|
import { fileUploadRule, protectionRule } from './rules.ts';
|
|
27
28
|
|
|
28
29
|
const debugLog = createDebug('explorbot:planner');
|
|
@@ -58,6 +59,7 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
58
59
|
currentPlan: Plan | null = null;
|
|
59
60
|
freshStart = false;
|
|
60
61
|
private lastStyleName = '';
|
|
62
|
+
private lastSuite: Suite | null = null;
|
|
61
63
|
researcher: Researcher;
|
|
62
64
|
private fisherman: Fisherman | null = null;
|
|
63
65
|
|
|
@@ -201,14 +203,14 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
201
203
|
this.currentPlan.url = state.url;
|
|
202
204
|
if (parentPlan) this.currentPlan.parentPlan = parentPlan;
|
|
203
205
|
const allPreviousScenarios = this.getPreviousSessionScenarios();
|
|
206
|
+
const existingTestScenarios = this.getExistingTestFileScenarios(state.url);
|
|
207
|
+
for (const s of existingTestScenarios) allPreviousScenarios.add(s);
|
|
204
208
|
for (const t of tests) {
|
|
205
209
|
if (allPreviousScenarios.has(t.scenario.toLowerCase())) continue;
|
|
206
210
|
t.style = this.lastStyleName;
|
|
207
211
|
t.startUrl = state.url;
|
|
208
212
|
this.currentPlan.addTest(t);
|
|
209
213
|
}
|
|
210
|
-
const summary = `Scenarios:\n${this.currentPlan.tests.map((t) => `- [${t.priority}] ${t.scenario}`).join('\n')}`;
|
|
211
|
-
tag('multiline').log(summary);
|
|
212
214
|
} else {
|
|
213
215
|
tag('step').log(`Expanding plan: "${this.currentPlan.title}"`);
|
|
214
216
|
this.currentPlan.nextIteration();
|
|
@@ -219,7 +221,6 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
219
221
|
}
|
|
220
222
|
}
|
|
221
223
|
|
|
222
|
-
this.moveExecutedTestsToEnd();
|
|
223
224
|
const availableStyles = Object.keys(getStyles()).join(', ');
|
|
224
225
|
tag('success').log(`Planning complete! ${this.currentPlan.tests.length} tests in plan: ${this.currentPlan.title}`);
|
|
225
226
|
tag('info').log(`Planning style: ${this.lastStyleName} (available: ${availableStyles})`);
|
|
@@ -231,12 +232,8 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
231
232
|
return this.currentPlan;
|
|
232
233
|
}
|
|
233
234
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
const pending = this.currentPlan.tests.filter((t) => t.result === null);
|
|
237
|
-
const executed = this.currentPlan.tests.filter((t) => t.result !== null);
|
|
238
|
-
this.currentPlan.tests = [...pending, ...executed];
|
|
239
|
-
this.currentPlan.notifyChange();
|
|
235
|
+
getSuite(): Suite | null {
|
|
236
|
+
return this.lastSuite;
|
|
240
237
|
}
|
|
241
238
|
|
|
242
239
|
private addNewTests(tests: Test[], defaultStartUrl: string): Test[] {
|
|
@@ -262,6 +259,17 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
262
259
|
return added;
|
|
263
260
|
}
|
|
264
261
|
|
|
262
|
+
private getExistingTestFileScenarios(currentUrl?: string): Set<string> {
|
|
263
|
+
if (!currentUrl) return new Set<string>();
|
|
264
|
+
try {
|
|
265
|
+
this.lastSuite = new Suite(currentUrl);
|
|
266
|
+
return this.lastSuite.getActiveScenarioTitles();
|
|
267
|
+
} catch (err: any) {
|
|
268
|
+
debugLog('Failed to load existing test files: %s', err.message);
|
|
269
|
+
return new Set<string>();
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
265
273
|
private cleanExperienceFlows(text: string): string | null {
|
|
266
274
|
const seenTitles = new Set<string>();
|
|
267
275
|
let result = text;
|
|
@@ -421,6 +429,17 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
421
429
|
}
|
|
422
430
|
}
|
|
423
431
|
|
|
432
|
+
if (this.lastSuite && this.lastSuite.automatedTestCount > 0) {
|
|
433
|
+
const automatedNames = this.lastSuite.getAutomatedTestNames();
|
|
434
|
+
conversation.addUserText(dedent`
|
|
435
|
+
<existing_automated_tests>
|
|
436
|
+
The following ${automatedNames.length} tests are already implemented and automated for this URL.
|
|
437
|
+
Do not propose tests that duplicate these:
|
|
438
|
+
${automatedNames.map((n) => `- ${n}`).join('\n')}
|
|
439
|
+
</existing_automated_tests>
|
|
440
|
+
`);
|
|
441
|
+
}
|
|
442
|
+
|
|
424
443
|
if (this.currentPlan) {
|
|
425
444
|
tag('step').log('Analyzing current plan to expand testing');
|
|
426
445
|
|
|
@@ -0,0 +1,532 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { relative, resolve } from 'node:path';
|
|
3
|
+
import { tool } from 'ai';
|
|
4
|
+
import { createBashTool } from 'bash-tool';
|
|
5
|
+
import chalk from 'chalk';
|
|
6
|
+
import { highlight } from 'cli-highlight';
|
|
7
|
+
import * as codeceptjs from 'codeceptjs';
|
|
8
|
+
import heal from 'codeceptjs/lib/heal';
|
|
9
|
+
import aiTracePlugin from 'codeceptjs/lib/plugin/aiTrace';
|
|
10
|
+
import figureSet from 'figures';
|
|
11
|
+
import dedent from 'dedent';
|
|
12
|
+
import { z } from 'zod';
|
|
13
|
+
import { ActionResult } from '../action-result.ts';
|
|
14
|
+
import { setActivity } from '../activity.ts';
|
|
15
|
+
import type { ExperienceTracker } from '../experience-tracker.ts';
|
|
16
|
+
import type Explorer from '../explorer.ts';
|
|
17
|
+
import type { KnowledgeTracker } from '../knowledge-tracker.ts';
|
|
18
|
+
import { Stats } from '../stats.ts';
|
|
19
|
+
import { Task, Test, TestResult } from '../test-plan.ts';
|
|
20
|
+
import { createDebug, tag } from '../utils/logger.ts';
|
|
21
|
+
import { loop } from '../utils/loop.ts';
|
|
22
|
+
import { loadTestSuites, printTestList } from '../utils/test-files.ts';
|
|
23
|
+
import type { Agent } from './agent.ts';
|
|
24
|
+
import { toolExecutionLabel } from './conversation.ts';
|
|
25
|
+
import type { Navigator } from './navigator.ts';
|
|
26
|
+
import { Provider } from './provider.ts';
|
|
27
|
+
import { locatorRule, actionRule, sectionContextRule } from './rules.ts';
|
|
28
|
+
import { TaskAgent } from './task-agent.ts';
|
|
29
|
+
import { RulesLoader } from '../utils/rules-loader.ts';
|
|
30
|
+
import { createCodeceptJSTools } from './tools.ts';
|
|
31
|
+
|
|
32
|
+
const debugLog = createDebug('explorbot:rerunner');
|
|
33
|
+
|
|
34
|
+
export class Rerunner extends TaskAgent implements Agent {
|
|
35
|
+
protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form'];
|
|
36
|
+
emoji = '🔄';
|
|
37
|
+
|
|
38
|
+
private explorer: Explorer;
|
|
39
|
+
private provider: Provider;
|
|
40
|
+
private agentTools: any;
|
|
41
|
+
private healedSteps: Array<{ test: string; original: string; healed: string }> = [];
|
|
42
|
+
private traceDir = '';
|
|
43
|
+
|
|
44
|
+
constructor(explorer: Explorer, provider: Provider, agentTools?: any) {
|
|
45
|
+
super();
|
|
46
|
+
this.explorer = explorer;
|
|
47
|
+
this.provider = provider;
|
|
48
|
+
this.agentTools = agentTools;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
protected getNavigator(): Navigator {
|
|
52
|
+
throw new Error('Rerunner does not use Navigator');
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
protected getExperienceTracker(): ExperienceTracker {
|
|
56
|
+
return this.explorer.getStateManager().getExperienceTracker();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
protected getKnowledgeTracker(): KnowledgeTracker {
|
|
60
|
+
return this.explorer.getKnowledgeTracker();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
protected getProvider(): Provider {
|
|
64
|
+
return this.provider;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private get rerunnerConfig(): Record<string, any> {
|
|
68
|
+
return (this.explorer.getConfig().ai?.agents?.rerunner as any) || {};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
private get healLimit(): number {
|
|
72
|
+
return this.rerunnerConfig.healLimit ?? 3;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private get healMaxIterations(): number {
|
|
76
|
+
return this.rerunnerConfig.healMaxIterations ?? 3;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
listTests(testsDir: string): void {
|
|
80
|
+
printTestList(loadTestSuites(testsDir));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async rerun(filePath: string, options?: { testIndices?: number[] }): Promise<RerunResult> {
|
|
84
|
+
const absPath = resolve(filePath);
|
|
85
|
+
if (!existsSync(absPath)) {
|
|
86
|
+
tag('error').log(`Test file not found: ${absPath}`);
|
|
87
|
+
return { total: 0, passed: 0, failed: 0, healed: 0 };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
tag('info').log(`Re-running tests from: ${relative(process.cwd(), absPath)}`);
|
|
91
|
+
setActivity('🔄 Re-running tests...', 'action');
|
|
92
|
+
|
|
93
|
+
this.healedSteps = [];
|
|
94
|
+
this.setupPlugins();
|
|
95
|
+
|
|
96
|
+
const testMap = new Map<string, Test>();
|
|
97
|
+
const results: { test: Test; mochaState: string }[] = [];
|
|
98
|
+
|
|
99
|
+
const onTestBefore = (mochaTest: any) => {
|
|
100
|
+
if (!mochaTest.file) mochaTest.file = absPath;
|
|
101
|
+
const task = new Test(mochaTest.title, 'normal', [], '');
|
|
102
|
+
task.start();
|
|
103
|
+
testMap.set(mochaTest.id || mochaTest.title, task);
|
|
104
|
+
Stats.tests++;
|
|
105
|
+
console.log(`\n ${chalk.green(figureSet.pointer)} ${chalk.bold(mochaTest.title)}`);
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const onStepStarted = (step: any) => {
|
|
109
|
+
if (!step.toCode) return;
|
|
110
|
+
const code = highlight(step.toCode(), { language: 'javascript' });
|
|
111
|
+
console.log(chalk.dim(` ${code}`));
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
const onStepPassed = (step: any) => {
|
|
115
|
+
const task = this.getCurrentTask(testMap);
|
|
116
|
+
if (!task || !step.toCode) return;
|
|
117
|
+
task.addStep(step.toCode(), step.duration, 'passed');
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const onStepFailed = (step: any, error: any) => {
|
|
121
|
+
const task = this.getCurrentTask(testMap);
|
|
122
|
+
if (!task || !step.toCode) return;
|
|
123
|
+
task.addStep(step.toCode(), step.duration, 'failed', error?.message);
|
|
124
|
+
console.log(chalk.red(` ${figureSet.cross} ${step.toCode()} — ${error?.message || 'failed'}`));
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
const onTestPassed = (mochaTest: any) => {
|
|
128
|
+
const task = testMap.get(mochaTest.id || mochaTest.title);
|
|
129
|
+
if (!task) return;
|
|
130
|
+
task.finish(TestResult.PASSED);
|
|
131
|
+
results.push({ test: task, mochaState: 'passed' });
|
|
132
|
+
console.log(chalk.green(` ${figureSet.tick} passed`));
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
const onTestFailed = (mochaTest: any, error: any) => {
|
|
136
|
+
const task = testMap.get(mochaTest.id || mochaTest.title);
|
|
137
|
+
if (!task) return;
|
|
138
|
+
task.addNote(`Failed: ${error?.message || 'unknown error'}`, TestResult.FAILED);
|
|
139
|
+
task.finish(TestResult.FAILED);
|
|
140
|
+
results.push({ test: task, mochaState: 'failed' });
|
|
141
|
+
console.log(chalk.red(` ${figureSet.cross} failed: ${error?.message || 'unknown'}`));
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
const { dispatcher } = codeceptjs.event;
|
|
145
|
+
dispatcher.on('test.before', onTestBefore);
|
|
146
|
+
dispatcher.on('step.start', onStepStarted);
|
|
147
|
+
dispatcher.on('step.passed', onStepPassed);
|
|
148
|
+
dispatcher.on('step.failed', onStepFailed);
|
|
149
|
+
dispatcher.on('test.passed', onTestPassed);
|
|
150
|
+
dispatcher.on('test.failed', onTestFailed);
|
|
151
|
+
|
|
152
|
+
try {
|
|
153
|
+
codeceptjs.container.createMocha();
|
|
154
|
+
const mocha = codeceptjs.container.mocha();
|
|
155
|
+
mocha.reporter(class {});
|
|
156
|
+
mocha.files = [absPath];
|
|
157
|
+
mocha.loadFiles();
|
|
158
|
+
|
|
159
|
+
let testIndex = 0;
|
|
160
|
+
for (const suite of mocha.suite.suites || []) {
|
|
161
|
+
for (const test of suite.tests || []) {
|
|
162
|
+
if (test.pending) {
|
|
163
|
+
testIndex++;
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
if (options?.testIndices?.length && !options.testIndices.includes(testIndex)) {
|
|
167
|
+
test.pending = true;
|
|
168
|
+
testIndex++;
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (!hasAssertions(test.body)) {
|
|
172
|
+
test.pending = true;
|
|
173
|
+
tag('substep').log(`Skipping: ${test.title} (no assertions)`);
|
|
174
|
+
}
|
|
175
|
+
testIndex++;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
await new Promise<void>((resolveRun) => {
|
|
180
|
+
mocha.run((failures: number) => {
|
|
181
|
+
debugLog('Mocha run finished with %d failures', failures);
|
|
182
|
+
resolveRun();
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
} catch (error) {
|
|
186
|
+
tag('error').log(`Rerun error: ${error instanceof Error ? error.message : error}`);
|
|
187
|
+
} finally {
|
|
188
|
+
dispatcher.off('test.before', onTestBefore);
|
|
189
|
+
dispatcher.off('step.start', onStepStarted);
|
|
190
|
+
dispatcher.off('step.passed', onStepPassed);
|
|
191
|
+
dispatcher.off('step.failed', onStepFailed);
|
|
192
|
+
dispatcher.off('test.passed', onTestPassed);
|
|
193
|
+
dispatcher.off('test.failed', onTestFailed);
|
|
194
|
+
this.teardownHealing();
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
if (this.healedSteps.length > 0) {
|
|
198
|
+
this.getHistorian().rewriteScenarioInFile(absPath, this.healedSteps);
|
|
199
|
+
tag('info').log(`Healed ${this.healedSteps.length} step(s), original file updated`);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const passed = results.filter((r) => r.mochaState === 'passed').length;
|
|
203
|
+
const failed = results.filter((r) => r.mochaState === 'failed').length;
|
|
204
|
+
const result: RerunResult = {
|
|
205
|
+
total: results.length,
|
|
206
|
+
passed,
|
|
207
|
+
failed,
|
|
208
|
+
healed: this.healedSteps.length,
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
this.printResults(result);
|
|
212
|
+
return result;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
private getCurrentTask(testMap: Map<string, Test>): Test | undefined {
|
|
216
|
+
const entries = [...testMap.values()];
|
|
217
|
+
return entries[entries.length - 1];
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
private setupPlugins(): void {
|
|
221
|
+
const healMod = heal.default || heal;
|
|
222
|
+
healMod.connectToEvents();
|
|
223
|
+
|
|
224
|
+
healMod.addRecipe('explorbot-ai-healer', {
|
|
225
|
+
priority: 10,
|
|
226
|
+
fn: async (context: any) => {
|
|
227
|
+
return this.healStep(context.step, context.error);
|
|
228
|
+
},
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
const userRecipes = (this.rerunnerConfig.recipes || {}) as Record<string, any>;
|
|
232
|
+
for (const [name, recipe] of Object.entries(userRecipes)) {
|
|
233
|
+
healMod.addRecipe(name, recipe);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
let currentTest: any = null;
|
|
237
|
+
let healTries = 0;
|
|
238
|
+
let isHealing = false;
|
|
239
|
+
let caughtError: any = null;
|
|
240
|
+
const healLimit = this.healLimit;
|
|
241
|
+
|
|
242
|
+
codeceptjs.event.dispatcher.on('test.before', (test: any) => {
|
|
243
|
+
currentTest = test;
|
|
244
|
+
healTries = 0;
|
|
245
|
+
caughtError = null;
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
codeceptjs.event.dispatcher.on('step.after', (step: any) => {
|
|
249
|
+
if (isHealing) return;
|
|
250
|
+
if (healTries >= healLimit) return;
|
|
251
|
+
if (!healMod.hasCorrespondingRecipes(step)) return;
|
|
252
|
+
|
|
253
|
+
codeceptjs.recorder.catchWithoutStop(async (err: any) => {
|
|
254
|
+
isHealing = true;
|
|
255
|
+
if (caughtError === err) throw err;
|
|
256
|
+
caughtError = err;
|
|
257
|
+
|
|
258
|
+
codeceptjs.recorder.session.start('heal');
|
|
259
|
+
debugLog('Healing started for: %s', step.toCode());
|
|
260
|
+
|
|
261
|
+
await healMod.healStep(step, err, { test: currentTest });
|
|
262
|
+
|
|
263
|
+
healTries++;
|
|
264
|
+
|
|
265
|
+
codeceptjs.recorder.add('close healing session', () => {
|
|
266
|
+
codeceptjs.recorder.reset();
|
|
267
|
+
codeceptjs.recorder.session.restore('heal');
|
|
268
|
+
codeceptjs.recorder.ignoreErr(err);
|
|
269
|
+
});
|
|
270
|
+
await codeceptjs.recorder.promise();
|
|
271
|
+
|
|
272
|
+
isHealing = false;
|
|
273
|
+
});
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
(global as any).container = codeceptjs.container;
|
|
277
|
+
|
|
278
|
+
codeceptjs.recorder.retry({
|
|
279
|
+
retries: 3,
|
|
280
|
+
when: (err: any) => {
|
|
281
|
+
if (!err?.message) return false;
|
|
282
|
+
return err.message.includes('was not found') || err.message.includes('Timeout') || err.message.includes('exceeded');
|
|
283
|
+
},
|
|
284
|
+
minTimeout: 2000,
|
|
285
|
+
maxTimeout: 5000,
|
|
286
|
+
factor: 1.5,
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
290
|
+
const outputDir = (global as any).output_dir || 'output';
|
|
291
|
+
this.traceDir = `${outputDir}/rerun_${timestamp}`;
|
|
292
|
+
|
|
293
|
+
const aiTrace = aiTracePlugin.default || aiTracePlugin;
|
|
294
|
+
aiTrace(this.rerunnerConfig.aiTrace || { output: this.traceDir });
|
|
295
|
+
|
|
296
|
+
import('@testomatio/reporter/codecept')
|
|
297
|
+
.then((mod) => {
|
|
298
|
+
const plugin = mod.default || mod;
|
|
299
|
+
plugin({ enabled: true });
|
|
300
|
+
})
|
|
301
|
+
.catch(() => debugLog('Testomatio reporter plugin not available'));
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
private teardownHealing(): void {
|
|
305
|
+
const healMod = heal.default || heal;
|
|
306
|
+
healMod.recipes['explorbot-ai-healer'] = undefined;
|
|
307
|
+
for (const name of Object.keys(this.rerunnerConfig.recipes || {})) {
|
|
308
|
+
healMod.recipes[name] = undefined;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
private async healStep(step: any, error: Error): Promise<((deps: { I: any }) => Promise<void>) | null> {
|
|
313
|
+
const failedCode = step.toCode?.() || '';
|
|
314
|
+
console.log(chalk.yellow(` ${figureSet.arrowRight} Healing: ${failedCode}`));
|
|
315
|
+
|
|
316
|
+
return async ({ I }: { I: any }) => {
|
|
317
|
+
const bashTool = await createBashTool({
|
|
318
|
+
destination: this.traceDir,
|
|
319
|
+
onBeforeBashCall: ({ command }) => {
|
|
320
|
+
if (/>[^>]|>>|\btee\b|\brm\b/.test(command)) {
|
|
321
|
+
return { command: 'echo "Read-only" >&2 && exit 1' };
|
|
322
|
+
}
|
|
323
|
+
return { command };
|
|
324
|
+
},
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
const healTask = new Task(`Heal: ${failedCode}`);
|
|
328
|
+
const codeceptTools = createCodeceptJSTools(this.explorer, healTask);
|
|
329
|
+
|
|
330
|
+
let healed = false;
|
|
331
|
+
let healedCommand = '';
|
|
332
|
+
|
|
333
|
+
const tools = {
|
|
334
|
+
bash: bashTool.bash,
|
|
335
|
+
...codeceptTools,
|
|
336
|
+
...this.agentTools,
|
|
337
|
+
wait: tool({
|
|
338
|
+
description: 'Wait N seconds for page to load. Use when loading indicators are detected.',
|
|
339
|
+
inputSchema: z.object({
|
|
340
|
+
seconds: z.number().describe('Seconds to wait'),
|
|
341
|
+
note: z.string().optional().describe('What are you waiting for'),
|
|
342
|
+
}),
|
|
343
|
+
execute: async ({ seconds, note }) => {
|
|
344
|
+
if (note) {
|
|
345
|
+
healTask.addNote(note);
|
|
346
|
+
tag('substep').log(note);
|
|
347
|
+
}
|
|
348
|
+
const action = this.explorer.createAction();
|
|
349
|
+
await action.execute(`I.wait(${seconds})`);
|
|
350
|
+
const state = this.explorer.getStateManager().getCurrentState();
|
|
351
|
+
const ar = state ? ActionResult.fromState(state) : null;
|
|
352
|
+
return {
|
|
353
|
+
success: true,
|
|
354
|
+
message: `Waited ${seconds}s`,
|
|
355
|
+
url: state?.url,
|
|
356
|
+
title: state?.title,
|
|
357
|
+
aria: ar?.getInteractiveARIA(),
|
|
358
|
+
};
|
|
359
|
+
},
|
|
360
|
+
}),
|
|
361
|
+
done: tool({
|
|
362
|
+
description: 'Healing succeeded. Report the command that fixed the step.',
|
|
363
|
+
inputSchema: z.object({
|
|
364
|
+
healedCommand: z.string().describe('The CodeceptJS command that fixed the step'),
|
|
365
|
+
}),
|
|
366
|
+
execute: async ({ healedCommand: cmd }) => {
|
|
367
|
+
healed = true;
|
|
368
|
+
healedCommand = cmd;
|
|
369
|
+
return { success: true, healedCommand: cmd };
|
|
370
|
+
},
|
|
371
|
+
}),
|
|
372
|
+
giveUp: tool({
|
|
373
|
+
description: 'Cannot heal. The issue is not fixable (missing data, page fundamentally different).',
|
|
374
|
+
inputSchema: z.object({
|
|
375
|
+
reason: z.string().describe('Why healing is not possible'),
|
|
376
|
+
}),
|
|
377
|
+
execute: async ({ reason }) => {
|
|
378
|
+
console.log(chalk.gray(` ${figureSet.line} Cannot heal: ${reason}`));
|
|
379
|
+
return { success: false, reason };
|
|
380
|
+
},
|
|
381
|
+
}),
|
|
382
|
+
};
|
|
383
|
+
|
|
384
|
+
const conversation = this.provider.startConversation(this.getHealSystemPrompt(), 'rerunner');
|
|
385
|
+
conversation.addUserText(this.getHealUserPrompt(failedCode, error));
|
|
386
|
+
|
|
387
|
+
await loop(
|
|
388
|
+
async ({ stop }) => {
|
|
389
|
+
if (healed) {
|
|
390
|
+
stop();
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
const result = await this.provider.invokeConversation(conversation, tools, {
|
|
395
|
+
maxToolRoundtrips: 5,
|
|
396
|
+
toolChoice: 'auto',
|
|
397
|
+
});
|
|
398
|
+
|
|
399
|
+
if (!result?.toolExecutions?.length) {
|
|
400
|
+
stop();
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
for (const exec of result.toolExecutions) {
|
|
405
|
+
const icon = exec.wasSuccessful ? chalk.green(figureSet.tick) : chalk.red(figureSet.cross);
|
|
406
|
+
let label = toolExecutionLabel(exec.input) || exec.toolName;
|
|
407
|
+
if (exec.toolName === 'bash') label = `bash [${this.traceDir}]: ${(exec.input?.command || '').substring(0, 100)}`;
|
|
408
|
+
tag('substep').log(`${icon} ${label}`);
|
|
409
|
+
|
|
410
|
+
if (exec.toolName === 'done') {
|
|
411
|
+
healed = true;
|
|
412
|
+
stop();
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
if (exec.toolName === 'giveUp') {
|
|
416
|
+
stop();
|
|
417
|
+
throw new Error(exec.input?.reason || 'Healing aborted');
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
},
|
|
421
|
+
{
|
|
422
|
+
maxAttempts: this.healMaxIterations,
|
|
423
|
+
catch: async ({ error: err, stop }) => {
|
|
424
|
+
if (err.message?.includes('Healing aborted')) throw err;
|
|
425
|
+
tag('warning').log(`Healing error: ${err.message}`);
|
|
426
|
+
stop();
|
|
427
|
+
},
|
|
428
|
+
}
|
|
429
|
+
);
|
|
430
|
+
|
|
431
|
+
if (!healed) {
|
|
432
|
+
throw new Error(`Could not heal: ${failedCode}`);
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
this.healedSteps.push({ test: '', original: failedCode, healed: healedCommand });
|
|
436
|
+
console.log(chalk.green(` ${figureSet.tick} Healed: ${healedCommand}`));
|
|
437
|
+
};
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
private getHealSystemPrompt(): string {
|
|
441
|
+
const customRules = this.provider.getSystemPromptForAgent('rerunner', this.explorer.getStateManager().getCurrentState()?.url) || '';
|
|
442
|
+
const currentUrl = this.explorer.getStateManager().getCurrentState()?.url || '';
|
|
443
|
+
const approach = RulesLoader.loadRules('rerunner', ['healing-approach'], currentUrl);
|
|
444
|
+
|
|
445
|
+
return dedent`
|
|
446
|
+
<role>
|
|
447
|
+
You are a senior test automation engineer healing a failed CodeceptJS test step.
|
|
448
|
+
The failed step did NOT execute. You MUST perform the action it was supposed to do.
|
|
449
|
+
</role>
|
|
450
|
+
|
|
451
|
+
${approach}
|
|
452
|
+
|
|
453
|
+
<tools>
|
|
454
|
+
- You MUST execute the replacement action — not just diagnose
|
|
455
|
+
- Use click() for buttons, links — commands array is FALLBACK LOCATORS for the SAME element
|
|
456
|
+
- Use form() for text input, dropdown selection, file uploads
|
|
457
|
+
- Use pressKey() for special keys or key combinations
|
|
458
|
+
- Use wait() when page is loading — returns fresh ARIA automatically
|
|
459
|
+
- Use research() to understand page structure, sections, and available UI elements
|
|
460
|
+
- Use xpathCheck() to search large HTML when element can't be found in ARIA
|
|
461
|
+
- Use see() for visual verification when unsure
|
|
462
|
+
- Use context() to refresh ARIA/HTML after actions
|
|
463
|
+
- Use bash to read trace files (cat */trace.md, grep *_console.json, cat *_aria.txt)
|
|
464
|
+
</tools>
|
|
465
|
+
|
|
466
|
+
${locatorRule}
|
|
467
|
+
|
|
468
|
+
${actionRule}
|
|
469
|
+
|
|
470
|
+
${sectionContextRule}
|
|
471
|
+
|
|
472
|
+
${customRules}
|
|
473
|
+
`;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
private getHealUserPrompt(failedCode: string, error: Error): string {
|
|
477
|
+
const state = this.explorer.getStateManager().getCurrentState();
|
|
478
|
+
const actionResult = state ? ActionResult.fromState(state) : null;
|
|
479
|
+
|
|
480
|
+
const headings: string[] = [];
|
|
481
|
+
if (state?.h1) headings.push(`H1: ${state.h1}`);
|
|
482
|
+
if (state?.h2) headings.push(`H2: ${state.h2}`);
|
|
483
|
+
if (state?.h3) headings.push(`H3: ${state.h3}`);
|
|
484
|
+
if (state?.h4) headings.push(`H4: ${state.h4}`);
|
|
485
|
+
|
|
486
|
+
return dedent`
|
|
487
|
+
A test step failed and needs healing.
|
|
488
|
+
|
|
489
|
+
<failed_step>
|
|
490
|
+
Command: ${failedCode}
|
|
491
|
+
Error: ${error.message}
|
|
492
|
+
</failed_step>
|
|
493
|
+
|
|
494
|
+
<page>
|
|
495
|
+
URL: ${state?.url || 'unknown'}
|
|
496
|
+
Title: ${state?.title || 'unknown'}
|
|
497
|
+
${headings.join('\n')}
|
|
498
|
+
</page>
|
|
499
|
+
|
|
500
|
+
<page_aria>
|
|
501
|
+
${actionResult?.getInteractiveARIA() || 'No ARIA available'}
|
|
502
|
+
</page_aria>
|
|
503
|
+
|
|
504
|
+
Trace directory: ${this.traceDir}
|
|
505
|
+
|
|
506
|
+
Diagnose and fix the failed step. You MUST execute the replacement action.
|
|
507
|
+
`;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
private printResults(result: RerunResult): void {
|
|
511
|
+
const parts = [];
|
|
512
|
+
if (result.passed > 0) parts.push(`${result.passed} passed`);
|
|
513
|
+
if (result.failed > 0) parts.push(`${result.failed} failed`);
|
|
514
|
+
if (result.healed > 0) parts.push(`${result.healed} healed`);
|
|
515
|
+
console.log(`\n${chalk.bold(`${result.total}`)} tests — ${parts.join(', ')}`);
|
|
516
|
+
if (this.traceDir) {
|
|
517
|
+
console.log(chalk.gray(`Traces: ${this.traceDir}`));
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function hasAssertions(body: string | undefined): boolean {
|
|
523
|
+
if (!body) return false;
|
|
524
|
+
return /I\.(see|dontSee|seeElement|dontSeeElement|seeInField|seeInSource|dontSeeInSource)\b/.test(body);
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
interface RerunResult {
|
|
528
|
+
total: number;
|
|
529
|
+
passed: number;
|
|
530
|
+
failed: number;
|
|
531
|
+
healed: number;
|
|
532
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import { Worker } from 'node:worker_threads';
|
|
3
4
|
import { outputPath } from '../../config.ts';
|
|
4
5
|
import { computeHtmlFingerprint } from '../../utils/html-diff.ts';
|
|
5
6
|
import { debugLog } from './mixin.ts';
|
|
@@ -76,9 +77,9 @@ function findSimilarMatch(combinedHtml: string): Promise<{ hash: string; similar
|
|
|
76
77
|
resolve(null);
|
|
77
78
|
}, FINGERPRINT_WORKER_TIMEOUT_MS);
|
|
78
79
|
|
|
79
|
-
worker.
|
|
80
|
+
worker.on('message', (data: { matchHash: string | null; similarity: number }) => {
|
|
80
81
|
clearTimeout(timeout);
|
|
81
|
-
const { matchHash, similarity } =
|
|
82
|
+
const { matchHash, similarity } = data;
|
|
82
83
|
if (!matchHash) {
|
|
83
84
|
resolve(null);
|
|
84
85
|
return;
|
|
@@ -86,7 +87,7 @@ function findSimilarMatch(combinedHtml: string): Promise<{ hash: string; similar
|
|
|
86
87
|
|
|
87
88
|
debugLog(`Similar fingerprint found: ${matchHash} (${similarity}% similar)`);
|
|
88
89
|
resolve({ hash: matchHash, similarity });
|
|
89
|
-
};
|
|
90
|
+
});
|
|
90
91
|
|
|
91
92
|
worker.postMessage({
|
|
92
93
|
html: combinedHtml,
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import { parentPort } from 'node:worker_threads';
|
|
3
4
|
import { computeHtmlFingerprint } from '../../utils/html-diff.ts';
|
|
4
5
|
|
|
5
|
-
declare const self: Worker;
|
|
6
|
-
|
|
7
6
|
function diceSimilarity(a: Set<string>, b: Set<string>): number {
|
|
8
7
|
let intersection = 0;
|
|
9
8
|
for (const item of a) {
|
|
@@ -14,22 +13,17 @@ function diceSimilarity(a: Set<string>, b: Set<string>): number {
|
|
|
14
13
|
return Math.round(((2 * intersection) / total) * 100);
|
|
15
14
|
}
|
|
16
15
|
|
|
17
|
-
|
|
18
|
-
const { html, statesDir, maxAgeMs, threshold } =
|
|
19
|
-
html: string;
|
|
20
|
-
statesDir: string;
|
|
21
|
-
maxAgeMs: number;
|
|
22
|
-
threshold: number;
|
|
23
|
-
};
|
|
16
|
+
parentPort!.on('message', (data: { html: string; statesDir: string; maxAgeMs: number; threshold: number }) => {
|
|
17
|
+
const { html, statesDir, maxAgeMs, threshold } = data;
|
|
24
18
|
|
|
25
19
|
if (!existsSync(statesDir)) {
|
|
26
|
-
|
|
20
|
+
parentPort!.postMessage({ matchHash: null, similarity: 0 });
|
|
27
21
|
return;
|
|
28
22
|
}
|
|
29
23
|
|
|
30
24
|
const currentFingerprint = new Set(computeHtmlFingerprint(html));
|
|
31
25
|
if (currentFingerprint.size === 0) {
|
|
32
|
-
|
|
26
|
+
parentPort!.postMessage({ matchHash: null, similarity: 0 });
|
|
33
27
|
return;
|
|
34
28
|
}
|
|
35
29
|
|
|
@@ -55,5 +49,5 @@ self.onmessage = (event: MessageEvent) => {
|
|
|
55
49
|
}
|
|
56
50
|
|
|
57
51
|
const matched = bestSimilarity >= threshold;
|
|
58
|
-
|
|
59
|
-
};
|
|
52
|
+
parentPort!.postMessage({ matchHash: matched ? bestHash : null, similarity: bestSimilarity });
|
|
53
|
+
});
|