explorbot 0.1.24 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +1 -1
- package/dist/src/ai/historian/codeceptjs.js +1 -1
- package/dist/src/ai/historian/experience.js +1 -1
- package/dist/src/ai/historian/playwright.js +1 -1
- package/dist/src/ai/historian/screencast.js +3 -3
- package/dist/src/ai/historian.js +1 -1
- package/dist/src/ai/navigator.js +4 -4
- package/dist/src/ai/pilot.js +8 -5
- package/dist/src/ai/planner.js +2 -4
- package/dist/src/ai/provider.js +5 -3
- package/dist/src/ai/researcher/locators.js +1 -1
- package/dist/src/ai/researcher.js +4 -4
- package/dist/src/ai/rules.js +20 -0
- package/dist/src/ai/session-analyst.js +4 -1
- package/dist/src/ai/task-agent.js +2 -2
- package/dist/src/ai/tester.js +3 -1
- package/dist/src/ai/tools.js +2 -0
- package/dist/src/commands/context-aria-command.js +1 -1
- package/dist/src/commands/explore-command.js +67 -25
- package/dist/src/commands/test-command.js +2 -1
- package/dist/src/components/App.js +1 -33
- package/dist/src/components/LogPane.js +9 -3
- package/dist/src/experience-tracker.js +2 -2
- package/dist/src/explorbot.js +1 -1
- package/dist/src/reporter.js +24 -5
- package/dist/src/utils/log-filters.js +27 -0
- package/dist/src/utils/logger.js +28 -1
- package/dist/src/utils/next-steps.js +1 -7
- package/package.json +1 -1
- package/src/ai/historian/codeceptjs.ts +1 -1
- package/src/ai/historian/experience.ts +1 -1
- package/src/ai/historian/playwright.ts +1 -1
- package/src/ai/historian/screencast.ts +3 -3
- package/src/ai/historian.ts +1 -1
- package/src/ai/navigator.ts +4 -4
- package/src/ai/pilot.ts +8 -5
- package/src/ai/planner.ts +1 -3
- package/src/ai/provider.ts +5 -3
- package/src/ai/researcher/locators.ts +1 -1
- package/src/ai/researcher.ts +4 -4
- package/src/ai/rules.ts +21 -0
- package/src/ai/session-analyst.ts +4 -1
- package/src/ai/task-agent.ts +2 -2
- package/src/ai/tester.ts +3 -1
- package/src/ai/tools.ts +2 -0
- package/src/commands/context-aria-command.ts +1 -1
- package/src/commands/explore-command.ts +68 -23
- package/src/commands/test-command.ts +2 -1
- package/src/components/App.tsx +0 -33
- package/src/components/LogPane.tsx +18 -3
- package/src/experience-tracker.ts +2 -2
- package/src/explorbot.ts +1 -1
- package/src/reporter.ts +24 -6
- package/src/utils/log-filters.ts +26 -0
- package/src/utils/logger.ts +24 -2
- package/src/utils/next-steps.ts +1 -6
|
@@ -10,7 +10,6 @@ const LogPane = React.memo(({ verboseMode }) => {
|
|
|
10
10
|
const [logs, setLogs] = useState([]);
|
|
11
11
|
const pendingLogsRef = React.useRef([]);
|
|
12
12
|
const flushTimeoutRef = React.useRef(null);
|
|
13
|
-
const MAX_MULTILINE_LINES = 16;
|
|
14
13
|
const MAX_STEP_LINES = 8;
|
|
15
14
|
const MAX_SUBSTEP_LINES = 6;
|
|
16
15
|
const formatCollapsedContent = useCallback((lines, collapsedCount, label) => {
|
|
@@ -88,6 +87,8 @@ const LogPane = React.memo(({ verboseMode }) => {
|
|
|
88
87
|
return { color: 'yellow' };
|
|
89
88
|
case 'debug':
|
|
90
89
|
return { color: 'gray', dimColor: true };
|
|
90
|
+
case 'operation':
|
|
91
|
+
return { color: 'gray', dimColor: true };
|
|
91
92
|
case 'substep':
|
|
92
93
|
return { color: 'gray', dimColor: true };
|
|
93
94
|
case 'step':
|
|
@@ -116,7 +117,8 @@ const LogPane = React.memo(({ verboseMode }) => {
|
|
|
116
117
|
const cleaned = stripAnsi(dedent(log.content));
|
|
117
118
|
const parsed = parseMarkdownToTerminal(cleaned);
|
|
118
119
|
const lines = parsed.split('\n');
|
|
119
|
-
const
|
|
120
|
+
const maxLines = log.maxLines || 16;
|
|
121
|
+
const truncated = lines.length > maxLines ? `${lines.slice(0, maxLines).join('\n')}\n... (${lines.length - maxLines} more lines)` : parsed;
|
|
120
122
|
return (React.createElement(Box, { key: index, borderStyle: "classic", borderLeft: false, borderRight: false, marginY: 1, padding: 1, borderColor: "dim", overflow: "hidden" },
|
|
121
123
|
React.createElement(Text, { color: "gray", dimColor: true }, truncated)));
|
|
122
124
|
}
|
|
@@ -127,6 +129,7 @@ const LogPane = React.memo(({ verboseMode }) => {
|
|
|
127
129
|
type: 'multiline',
|
|
128
130
|
content: `HTML Content:\n\n${markdown}`,
|
|
129
131
|
timestamp: log.timestamp,
|
|
132
|
+
maxLines: 10,
|
|
130
133
|
};
|
|
131
134
|
return renderLogEntry(multilineLog, index);
|
|
132
135
|
}
|
|
@@ -134,6 +137,9 @@ const LogPane = React.memo(({ verboseMode }) => {
|
|
|
134
137
|
if (log.type === 'substep') {
|
|
135
138
|
return (React.createElement(Box, { key: index, marginLeft: 2, flexDirection: "column" }, lines.map((line, lineIndex) => (React.createElement(Text, { key: `${index}-${lineIndex}`, ...styles }, lineIndex === 0 ? `> ${line}` : ` ${line}`)))));
|
|
136
139
|
}
|
|
140
|
+
if (log.type === 'operation') {
|
|
141
|
+
return (React.createElement(Box, { key: index, marginLeft: 2, flexDirection: "column" }, lines.map((line, lineIndex) => (React.createElement(Text, { key: `${index}-${lineIndex}`, ...styles }, lineIndex === 0 ? `· ${line}` : ` ${line}`)))));
|
|
142
|
+
}
|
|
137
143
|
if (log.type === 'step') {
|
|
138
144
|
return (React.createElement(Box, { key: index, flexDirection: "column", paddingLeft: 2 }, lines.map((line, lineIndex) => (React.createElement(Text, { key: `${index}-${lineIndex}`, ...styles }, line)))));
|
|
139
145
|
}
|
|
@@ -145,7 +151,7 @@ const LogPane = React.memo(({ verboseMode }) => {
|
|
|
145
151
|
icon && React.createElement(Text, { ...styles }, icon),
|
|
146
152
|
React.createElement(Box, { flexDirection: "column" }, lines.map((line, lineIndex) => (React.createElement(Text, { key: `${index}-${lineIndex}`, ...styles }, line))))));
|
|
147
153
|
};
|
|
148
|
-
const maxLogs =
|
|
154
|
+
const maxLogs = 80;
|
|
149
155
|
const visibleLogs = logs.length > maxLogs ? logs.slice(-maxLogs) : logs;
|
|
150
156
|
return React.createElement(Box, { flexDirection: "column" }, visibleLogs.map((log, index) => renderLogEntry(log, index)).filter(Boolean));
|
|
151
157
|
});
|
|
@@ -164,7 +164,7 @@ export class ExperienceTracker {
|
|
|
164
164
|
const newEntry = generateActionContent(title, filteredCode, action.explanation);
|
|
165
165
|
const updatedContent = `${newEntry}\n\n${content}`;
|
|
166
166
|
this.writeExperienceFile(stateHash, updatedContent, data);
|
|
167
|
-
tag('
|
|
167
|
+
tag('operation').log(`Added ACTION to: ${stateHash}.md`);
|
|
168
168
|
}
|
|
169
169
|
writeFlow(state, body, relatedUrls) {
|
|
170
170
|
if (this.disabled || this.isWritingDisabled(state))
|
|
@@ -190,7 +190,7 @@ export class ExperienceTracker {
|
|
|
190
190
|
}
|
|
191
191
|
const updatedContent = `${body}\n${content}`;
|
|
192
192
|
this.writeExperienceFile(stateHash, updatedContent, data);
|
|
193
|
-
tag('
|
|
193
|
+
tag('operation').log(`Added FLOW to: ${stateHash}.md`);
|
|
194
194
|
}
|
|
195
195
|
getAllExperience() {
|
|
196
196
|
const allFiles = [];
|
package/dist/src/explorbot.js
CHANGED
|
@@ -422,7 +422,7 @@ export class ExplorBot {
|
|
|
422
422
|
this.lastReportedTestCount = tests.length;
|
|
423
423
|
return;
|
|
424
424
|
}
|
|
425
|
-
tag('multiline').log(markdown);
|
|
425
|
+
tag('multiline').log(markdown, { maxLines: 22 });
|
|
426
426
|
const filePath = this.agentSessionAnalyst().writeReport(markdown);
|
|
427
427
|
tag('info').log(`Session report saved: ${relativeToCwd(filePath)}`);
|
|
428
428
|
const reporter = this.explorer?.getReporter();
|
package/dist/src/reporter.js
CHANGED
|
@@ -82,10 +82,12 @@ export class Reporter {
|
|
|
82
82
|
return;
|
|
83
83
|
}
|
|
84
84
|
try {
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
85
|
+
const result = await withQuietReporterLogs(async () => {
|
|
86
|
+
this.client = new Client({ apiKey: process.env.TESTOMATIO || '', title: this.buildTitle() });
|
|
87
|
+
const timeoutMs = Number(process.env.TESTOMATIO_TIMEOUT_MS || '15000');
|
|
88
|
+
const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve('timeout'), timeoutMs));
|
|
89
|
+
return await Promise.race([this.client.createRun({ configuration: { exploratory: true } }).then(() => 'success'), timeoutPromise]);
|
|
90
|
+
});
|
|
89
91
|
if (result === 'timeout') {
|
|
90
92
|
debugLog('Reporter run creation timed out');
|
|
91
93
|
return;
|
|
@@ -252,7 +254,9 @@ export class Reporter {
|
|
|
252
254
|
return;
|
|
253
255
|
}
|
|
254
256
|
try {
|
|
255
|
-
await
|
|
257
|
+
await withQuietReporterLogs(async () => {
|
|
258
|
+
await this.client.updateRunStatus('finished');
|
|
259
|
+
});
|
|
256
260
|
this.isRunStarted = false;
|
|
257
261
|
debugLog('Testomat.io run finished');
|
|
258
262
|
}
|
|
@@ -299,3 +303,18 @@ export class Reporter {
|
|
|
299
303
|
return;
|
|
300
304
|
}
|
|
301
305
|
}
|
|
306
|
+
async function withQuietReporterLogs(fn) {
|
|
307
|
+
const previousLevel = process.env.TESTOMATIO_LOG_LEVEL;
|
|
308
|
+
process.env.TESTOMATIO_LOG_LEVEL = 'ERROR';
|
|
309
|
+
try {
|
|
310
|
+
return await fn();
|
|
311
|
+
}
|
|
312
|
+
finally {
|
|
313
|
+
if (previousLevel === undefined) {
|
|
314
|
+
process.env.TESTOMATIO_LOG_LEVEL = undefined;
|
|
315
|
+
}
|
|
316
|
+
else {
|
|
317
|
+
process.env.TESTOMATIO_LOG_LEVEL = previousLevel;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export class RecentStepFilter {
|
|
2
|
+
ttlMs;
|
|
3
|
+
recentStepKeys = new Map();
|
|
4
|
+
constructor(ttlMs = 15000) {
|
|
5
|
+
this.ttlMs = ttlMs;
|
|
6
|
+
}
|
|
7
|
+
shouldSuppress(content, now = Date.now()) {
|
|
8
|
+
const key = normalizeStepCommand(content);
|
|
9
|
+
if (!key)
|
|
10
|
+
return false;
|
|
11
|
+
for (const [existingKey, timestamp] of this.recentStepKeys) {
|
|
12
|
+
if (now - timestamp > this.ttlMs) {
|
|
13
|
+
this.recentStepKeys.delete(existingKey);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
if (this.recentStepKeys.has(key))
|
|
17
|
+
return true;
|
|
18
|
+
this.recentStepKeys.set(key, now);
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function normalizeStepCommand(content) {
|
|
23
|
+
const normalized = content.replace(/\s+/g, ' ').trim();
|
|
24
|
+
if (!normalized.startsWith('I.'))
|
|
25
|
+
return null;
|
|
26
|
+
return normalized.toLowerCase();
|
|
27
|
+
}
|
package/dist/src/utils/logger.js
CHANGED
|
@@ -7,6 +7,7 @@ import dedent from 'dedent';
|
|
|
7
7
|
import stripAnsi from 'strip-ansi';
|
|
8
8
|
import { ConfigParser } from '../config.js';
|
|
9
9
|
import { Observability } from "../observability.js";
|
|
10
|
+
import { RecentStepFilter } from "./log-filters.js";
|
|
10
11
|
import { parseMarkdownToTerminal } from "./markdown-terminal.js";
|
|
11
12
|
class DebugFilter {
|
|
12
13
|
patterns = [];
|
|
@@ -51,6 +52,7 @@ const debugFilter = new DebugFilter();
|
|
|
51
52
|
class ConsoleDestination {
|
|
52
53
|
verboseMode = false;
|
|
53
54
|
forceEnabled = false;
|
|
55
|
+
recentSteps = new RecentStepFilter();
|
|
54
56
|
isEnabled() {
|
|
55
57
|
return this.forceEnabled || !process.env.INK_RUNNING;
|
|
56
58
|
}
|
|
@@ -65,6 +67,10 @@ class ConsoleDestination {
|
|
|
65
67
|
return;
|
|
66
68
|
if (entry.type === 'html')
|
|
67
69
|
return;
|
|
70
|
+
if (entry.type === 'operation' && !this.verboseMode)
|
|
71
|
+
return;
|
|
72
|
+
if (entry.type === 'step' && !this.verboseMode && this.recentSteps.shouldSuppress(entry.content))
|
|
73
|
+
return;
|
|
68
74
|
let content = entry.content;
|
|
69
75
|
if (entry.type === 'multiline') {
|
|
70
76
|
const cleaned = stripAnsi(dedent(entry.content));
|
|
@@ -84,6 +90,9 @@ class ConsoleDestination {
|
|
|
84
90
|
else if (entry.type === 'step') {
|
|
85
91
|
content = chalk.gray(` ${content}`);
|
|
86
92
|
}
|
|
93
|
+
else if (entry.type === 'operation') {
|
|
94
|
+
content = chalk.gray(` · ${content}`);
|
|
95
|
+
}
|
|
87
96
|
else if (entry.type === 'substep') {
|
|
88
97
|
content = chalk.gray(` > ${content}`);
|
|
89
98
|
}
|
|
@@ -213,6 +222,8 @@ class ReactDestination {
|
|
|
213
222
|
return this.debugMode;
|
|
214
223
|
}
|
|
215
224
|
shouldWrite(entry) {
|
|
225
|
+
if (entry.type === 'operation' && !this.debugMode)
|
|
226
|
+
return false;
|
|
216
227
|
if (entry.type !== 'debug')
|
|
217
228
|
return true;
|
|
218
229
|
if (this.debugMode)
|
|
@@ -269,7 +280,7 @@ class CaptainDestination {
|
|
|
269
280
|
}
|
|
270
281
|
stopCapture() {
|
|
271
282
|
this.capturing = false;
|
|
272
|
-
const logs = this.entries.filter((e) => e.type !== 'debug' && e.type !== 'html' && e.type !== 'multiline').map((e) => `[${e.type}] ${e.content}`);
|
|
283
|
+
const logs = this.entries.filter((e) => e.type !== 'debug' && e.type !== 'html' && e.type !== 'multiline' && e.type !== 'operation').map((e) => `[${e.type}] ${e.content}`);
|
|
273
284
|
this.entries = [];
|
|
274
285
|
return logs;
|
|
275
286
|
}
|
|
@@ -377,6 +388,7 @@ class Logger {
|
|
|
377
388
|
}
|
|
378
389
|
return;
|
|
379
390
|
}
|
|
391
|
+
const options = this.extractLogOptions(type, args);
|
|
380
392
|
let content = this.processArgs(args);
|
|
381
393
|
if (type === 'step' && args[0]?.toCode) {
|
|
382
394
|
content = args[0].toCode();
|
|
@@ -386,6 +398,7 @@ class Logger {
|
|
|
386
398
|
content,
|
|
387
399
|
timestamp: new Date(),
|
|
388
400
|
originalArgs: args,
|
|
401
|
+
maxLines: options?.maxLines,
|
|
389
402
|
};
|
|
390
403
|
if (this.file.isEnabled())
|
|
391
404
|
this.file.write(entry);
|
|
@@ -421,6 +434,20 @@ class Logger {
|
|
|
421
434
|
multiline(...args) {
|
|
422
435
|
this.log('multiline', ...args);
|
|
423
436
|
}
|
|
437
|
+
extractLogOptions(type, args) {
|
|
438
|
+
if (type !== 'multiline')
|
|
439
|
+
return null;
|
|
440
|
+
const last = args[args.length - 1];
|
|
441
|
+
if (!last || typeof last !== 'object' || Array.isArray(last))
|
|
442
|
+
return null;
|
|
443
|
+
if (!('maxLines' in last))
|
|
444
|
+
return null;
|
|
445
|
+
args.pop();
|
|
446
|
+
const maxLines = Number(last.maxLines);
|
|
447
|
+
if (!Number.isFinite(maxLines) || maxLines <= 0)
|
|
448
|
+
return null;
|
|
449
|
+
return { maxLines };
|
|
450
|
+
}
|
|
424
451
|
}
|
|
425
452
|
const logger = Logger.getInstance();
|
|
426
453
|
let stepSpanParent = null;
|
|
@@ -27,11 +27,5 @@ export function printNextSteps(sections) {
|
|
|
27
27
|
}
|
|
28
28
|
blocks.push(lines.join('\n'));
|
|
29
29
|
}
|
|
30
|
-
|
|
31
|
-
if (i > 0)
|
|
32
|
-
tag('info').log('');
|
|
33
|
-
for (const line of blocks[i].split('\n')) {
|
|
34
|
-
tag('info').log(line);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
30
|
+
tag('multiline').log(blocks.join('\n\n'), { maxLines: 18 });
|
|
37
31
|
}
|
package/package.json
CHANGED
|
@@ -102,7 +102,7 @@ export function WithCodeceptJS<T extends Constructor>(Base: T) {
|
|
|
102
102
|
writeFileSync(filePath, lines.join('\n'));
|
|
103
103
|
this.savedFiles.add(filePath);
|
|
104
104
|
|
|
105
|
-
tag('
|
|
105
|
+
tag('operation').log(`Saved plan tests to: ${relativeToCwd(filePath)}`);
|
|
106
106
|
return filePath;
|
|
107
107
|
}
|
|
108
108
|
|
|
@@ -56,7 +56,7 @@ export function WithExperience<T extends Constructor>(Base: T) {
|
|
|
56
56
|
|
|
57
57
|
await this.stopScreencast();
|
|
58
58
|
|
|
59
|
-
tag('
|
|
59
|
+
tag('operation').log(`Historian saved session for: ${task.description}`);
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
private async reportSession(test: Test, steps: SessionStep[]): Promise<void> {
|
|
@@ -140,7 +140,7 @@ export function WithPlaywright<T extends Constructor>(Base: T) {
|
|
|
140
140
|
writeFileSync(filePath, lines.join('\n'));
|
|
141
141
|
this.savedFiles.add(filePath);
|
|
142
142
|
|
|
143
|
-
tag('
|
|
143
|
+
tag('operation').log(`Saved plan tests to: ${relativeToCwd(filePath)}`);
|
|
144
144
|
return filePath;
|
|
145
145
|
}
|
|
146
146
|
|
|
@@ -92,7 +92,7 @@ export function WithScreencast<T extends Constructor>(Base: T) {
|
|
|
92
92
|
this.screencastTask = test?._explorbotTest || null;
|
|
93
93
|
this.screencastLastChapter = null;
|
|
94
94
|
} catch (err) {
|
|
95
|
-
tag('
|
|
95
|
+
tag('operation').log(`Screencast start failed: ${(err as Error).message}`);
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
|
|
@@ -116,7 +116,7 @@ export function WithScreencast<T extends Constructor>(Base: T) {
|
|
|
116
116
|
try {
|
|
117
117
|
await this.screencastPage.screencast.stop();
|
|
118
118
|
} catch (err) {
|
|
119
|
-
tag('
|
|
119
|
+
tag('operation').log(`Screencast stop failed: ${(err as Error).message}`);
|
|
120
120
|
}
|
|
121
121
|
this.screencastActive = false;
|
|
122
122
|
this.screencastPage = null;
|
|
@@ -126,7 +126,7 @@ export function WithScreencast<T extends Constructor>(Base: T) {
|
|
|
126
126
|
if (path) {
|
|
127
127
|
this.savedFiles.add(path);
|
|
128
128
|
task?.addArtifact?.(path);
|
|
129
|
-
tag('
|
|
129
|
+
tag('operation').log(`Saved screencast: ${relativeToCwd(path)}`);
|
|
130
130
|
}
|
|
131
131
|
}
|
|
132
132
|
};
|
package/src/ai/historian.ts
CHANGED
|
@@ -62,6 +62,6 @@ export class Historian extends HistorianBase {
|
|
|
62
62
|
|
|
63
63
|
writeFileSync(filePath, content);
|
|
64
64
|
this.savedFiles.add(filePath);
|
|
65
|
-
tag('
|
|
65
|
+
tag('operation').log(`Updated test file with healed steps: ${relativeToCwd(filePath)}`);
|
|
66
66
|
}
|
|
67
67
|
}
|
package/src/ai/navigator.ts
CHANGED
|
@@ -206,7 +206,7 @@ class Navigator implements Agent {
|
|
|
206
206
|
if (!actionResult.isInsideIframe) {
|
|
207
207
|
const successful = this.experienceTracker.getSuccessfulExperience(actionResult);
|
|
208
208
|
if (successful.length > 0) {
|
|
209
|
-
tag('
|
|
209
|
+
tag('operation').log(`Found ${successful.length} experience ${pluralize(successful.length, 'file')} for: ${actionResult.url}`);
|
|
210
210
|
experience = `<experience>\nPast successful recipes recorded from prior runs for this page. Prefer these solutions first if they match the goal.\n\n${successful.join('\n\n')}\n</experience>`;
|
|
211
211
|
}
|
|
212
212
|
}
|
|
@@ -307,7 +307,7 @@ class Navigator implements Agent {
|
|
|
307
307
|
stop();
|
|
308
308
|
return;
|
|
309
309
|
}
|
|
310
|
-
tag('
|
|
310
|
+
tag('operation').log('Feeding failures back to AI for a new batch...');
|
|
311
311
|
let contextMsg = 'Previous solutions did not work. Analyze the failures and try DIFFERENT strategies (not syntactic variants of the same locator).\n\n';
|
|
312
312
|
if (batchFailures.length > 0) {
|
|
313
313
|
const lines = batchFailures
|
|
@@ -633,7 +633,7 @@ class Navigator implements Agent {
|
|
|
633
633
|
|
|
634
634
|
const cachedVerification = actionResult.getVerification(message);
|
|
635
635
|
if (cachedVerification !== null) {
|
|
636
|
-
tag('
|
|
636
|
+
tag('operation').log(`Reusing cached verification: ${cachedVerification ? 'PASS' : 'FAIL'}`);
|
|
637
637
|
return { verified: cachedVerification, successfulCodes: [], assertionSteps: [], totalAttempted: 0 };
|
|
638
638
|
}
|
|
639
639
|
|
|
@@ -654,7 +654,7 @@ class Navigator implements Agent {
|
|
|
654
654
|
const toc = this.experienceTracker.getExperienceTableOfContents(actionResult);
|
|
655
655
|
if (toc.length > 0) {
|
|
656
656
|
const totalSections = toc.reduce((sum, entry) => sum + entry.sections.length, 0);
|
|
657
|
-
tag('
|
|
657
|
+
tag('operation').log(`Found ${toc.length} experience ${pluralize(toc.length, 'file')} (${totalSections} sections) for: ${actionResult.url}`);
|
|
658
658
|
experience = renderExperienceToc(toc);
|
|
659
659
|
}
|
|
660
660
|
}
|
package/src/ai/pilot.ts
CHANGED
|
@@ -104,7 +104,7 @@ export class Pilot implements Agent {
|
|
|
104
104
|
|
|
105
105
|
const schema = z.object({
|
|
106
106
|
decision: z.enum(['pass', 'fail', 'continue', 'skipped']).describe('pass = test succeeded, fail = test failed, continue = tester should keep going, skipped = scenario is irrelevant OR systematic execution failures prevented testing'),
|
|
107
|
-
reason: z.string().describe('
|
|
107
|
+
reason: z.string().describe('Concise user-facing reason, maximum 1 short sentence and 120 characters. Do NOT repeat the decision status; explain only the evidence. For continue: explain why rejected and suggest alternatives.'),
|
|
108
108
|
guidance: z.string().nullable().describe('Required for "continue": specific actionable instruction for the tester — what exactly to verify, retry differently, or complete next. Be concrete.'),
|
|
109
109
|
requestVerification: z
|
|
110
110
|
.string()
|
|
@@ -177,7 +177,7 @@ export class Pilot implements Agent {
|
|
|
177
177
|
}
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
-
tag('info').log(`Pilot: ${result.decision}
|
|
180
|
+
tag('info').log(`Pilot: ${result.decision} - ${result.reason}`);
|
|
181
181
|
task.summary = result.reason;
|
|
182
182
|
|
|
183
183
|
const verdictState = screenshotState || currentState;
|
|
@@ -221,7 +221,7 @@ export class Pilot implements Agent {
|
|
|
221
221
|
|
|
222
222
|
const schema = z.object({
|
|
223
223
|
decision: z.enum(['allow', 'fail', 'continue', 'skipped']).describe('allow = reset proceeds, fail = test failed (stop looping), continue = veto reset, tester should act on current page instead, skipped = scenario is irrelevant or cannot be executed'),
|
|
224
|
-
reason: z.string().describe('
|
|
224
|
+
reason: z.string().describe('Concise evidence-only reason, maximum 1 short sentence and 120 characters. Do not restate the decision.'),
|
|
225
225
|
guidance: z.string().nullable().describe('Required for "continue": concrete instruction for what the tester should do instead of resetting (e.g. which tool to call, what to verify).'),
|
|
226
226
|
});
|
|
227
227
|
|
|
@@ -388,8 +388,9 @@ export class Pilot implements Agent {
|
|
|
388
388
|
- "continue": tester hasn't completed the goal; provide concrete guidance (which tool, what to check).
|
|
389
389
|
If a verify() asserted a state that was ALREADY TRUE before the test, it proves nothing — reject.
|
|
390
390
|
|
|
391
|
-
reason field:
|
|
392
|
-
what was verified, what failed,
|
|
391
|
+
reason field: one short sentence, maximum 120 characters. Do NOT restate the decision
|
|
392
|
+
("scenario goal achieved/not achieved"). State what happened: what was verified, what failed,
|
|
393
|
+
or what evidence was found.
|
|
393
394
|
`;
|
|
394
395
|
}
|
|
395
396
|
|
|
@@ -1017,6 +1018,8 @@ export class Pilot implements Agent {
|
|
|
1017
1018
|
Response format:
|
|
1018
1019
|
PROGRESS: <1 sentence assessment>
|
|
1019
1020
|
NEXT: <specific actionable instruction for Tester>
|
|
1021
|
+
|
|
1022
|
+
Keep user-facing reasons concise: one short sentence, maximum 120 characters, evidence only, no repeated verdict wording.
|
|
1020
1023
|
`;
|
|
1021
1024
|
}
|
|
1022
1025
|
}
|
package/src/ai/planner.ts
CHANGED
|
@@ -36,7 +36,7 @@ const TasksSchema = z.object({
|
|
|
36
36
|
scenario: z.string().describe('A single sentence describing what to test'),
|
|
37
37
|
priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
|
|
38
38
|
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL (only for tests on visited subpages)'),
|
|
39
|
-
steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "
|
|
39
|
+
steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),
|
|
40
40
|
expectedOutcomes: z
|
|
41
41
|
.array(z.string())
|
|
42
42
|
.describe('List of expected outcomes that can be verified. Each outcome should be simple, specific, and easy to check (e.g., "Success message appears", "URL changes to /dashboard", "Form field shows error"). Keep outcomes atomic - do not combine multiple checks into one.'),
|
|
@@ -226,9 +226,7 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
226
226
|
}
|
|
227
227
|
}
|
|
228
228
|
|
|
229
|
-
const availableStyles = Object.keys(getStyles()).join(', ');
|
|
230
229
|
tag('success').log(`Planning complete! ${this.currentPlan.tests.length} tests in plan: ${this.currentPlan.title}`);
|
|
231
|
-
tag('info').log(`Planning style: ${this.lastStyleName} (available: ${availableStyles})`);
|
|
232
230
|
|
|
233
231
|
if (state.url) registerPlan(state.url, this.currentPlan, feature, state.hash);
|
|
234
232
|
|
package/src/ai/provider.ts
CHANGED
|
@@ -294,8 +294,11 @@ export class Provider {
|
|
|
294
294
|
}
|
|
295
295
|
throw new ContextLengthError(error.message || error.toString());
|
|
296
296
|
}
|
|
297
|
-
|
|
298
|
-
|
|
297
|
+
const message = error.message || error.toString();
|
|
298
|
+
if (message !== 'No response text from AI') {
|
|
299
|
+
tag('error').log(message);
|
|
300
|
+
}
|
|
301
|
+
throw new AiError(message);
|
|
299
302
|
}
|
|
300
303
|
}
|
|
301
304
|
|
|
@@ -376,7 +379,6 @@ export class Provider {
|
|
|
376
379
|
} catch (error: any) {
|
|
377
380
|
clearActivity();
|
|
378
381
|
if (error?.message?.includes('Tool choice is required')) {
|
|
379
|
-
tag('warning').log('Model completed without calling a tool, returning empty result');
|
|
380
382
|
return { text: '', toolCalls: [], toolResults: [], response: { messages: [] }, usage: null };
|
|
381
383
|
}
|
|
382
384
|
if (error?.name === 'AbortError') throw error;
|
|
@@ -80,7 +80,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
|
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
tag('
|
|
83
|
+
tag('operation').log(`Validated ${locators.length} locators: ${locators.length - broken} valid, ${broken} broken`);
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
async fixBrokenSections(result: ResearchResult, conversation: Conversation): Promise<void> {
|
package/src/ai/researcher.ts
CHANGED
|
@@ -151,7 +151,7 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
151
151
|
if (!deep && !force) {
|
|
152
152
|
const similar = await findSimilarResearch(combinedHtml);
|
|
153
153
|
if (similar) {
|
|
154
|
-
tag('
|
|
154
|
+
tag('operation').log('Similar research found, reusing cached result');
|
|
155
155
|
if (stateHash) saveResearch(stateHash, similar, combinedHtml);
|
|
156
156
|
tag('multiline').log(formatResearchSummary(similar));
|
|
157
157
|
tag('success').log('Research complete (reused)');
|
|
@@ -316,10 +316,10 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
316
316
|
|
|
317
317
|
tag('multiline').log(formatResearchSummary(result.text, { visionUsed: this.hasScreenshotToAnalyze }));
|
|
318
318
|
tag('success').log('Research complete');
|
|
319
|
-
if (researchFile) tag('
|
|
319
|
+
if (researchFile) tag('operation').log(`Research file saved to: ${researchFile}`);
|
|
320
320
|
if (this.actionResult?.screenshotFile) {
|
|
321
321
|
const screenshotPath = outputPath('states', this.actionResult.screenshotFile);
|
|
322
|
-
tag('
|
|
322
|
+
tag('operation').log(`UI screenshot: file://${screenshotPath}`);
|
|
323
323
|
}
|
|
324
324
|
|
|
325
325
|
await this.hooksRunner.runAfterHook('researcher', state.url);
|
|
@@ -467,7 +467,7 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
467
467
|
.filter((k) => !!k)
|
|
468
468
|
.join('\n\n');
|
|
469
469
|
|
|
470
|
-
tag('
|
|
470
|
+
tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')} for: ${this.actionResult.url}`);
|
|
471
471
|
knowledge = `
|
|
472
472
|
<hint>
|
|
473
473
|
Here is relevant knowledge for this page:
|
package/src/ai/rules.ts
CHANGED
|
@@ -131,6 +131,12 @@ export const fileUploadRule = dedent`
|
|
|
131
131
|
</file_upload>
|
|
132
132
|
`;
|
|
133
133
|
|
|
134
|
+
export const formRequirementsRule = dedent`
|
|
135
|
+
<form_requirements>
|
|
136
|
+
Before filling a form that persists data (create/update), read each control's requirements (required, type/format, length, placeholder/aria-describedby hints) from <page_aria> and page HTML — call context() if not visible — and enter values that satisfy them. Search/filter/sort forms that only change the view do not need this.
|
|
137
|
+
</form_requirements>
|
|
138
|
+
`;
|
|
139
|
+
|
|
134
140
|
// in rage mode we do not protect from irreversible actions
|
|
135
141
|
export const protectionRule = dedent`
|
|
136
142
|
<important>
|
|
@@ -270,6 +276,8 @@ export const actionRule = dedent`
|
|
|
270
276
|
If locator doesn't work, try CSS or XPath locators.
|
|
271
277
|
If nothing works, use I.clickXY(x, y) as last resort.
|
|
272
278
|
|
|
279
|
+
For checkboxes, prefer I.checkOption/I.uncheckOption over I.click.
|
|
280
|
+
|
|
273
281
|
|
|
274
282
|
### I.fillField
|
|
275
283
|
|
|
@@ -356,6 +364,19 @@ export const actionRule = dedent`
|
|
|
356
364
|
I.selectOption('form select[name=account]', 'Premium');
|
|
357
365
|
</example>
|
|
358
366
|
|
|
367
|
+
### I.checkOption / I.uncheckOption
|
|
368
|
+
|
|
369
|
+
Set a checkbox/radio to a definite state — idempotent, never toggles. Use for checkboxes instead of I.click. Run via form(), not click().
|
|
370
|
+
|
|
371
|
+
I.checkOption(<locator>, <context>)
|
|
372
|
+
I.uncheckOption(<locator>, <context>)
|
|
373
|
+
|
|
374
|
+
<example>
|
|
375
|
+
I.checkOption('Subscribe');
|
|
376
|
+
I.checkOption({ role: 'checkbox', text: 'Agree' });
|
|
377
|
+
I.uncheckOption('Subscribe', '.preferences');
|
|
378
|
+
</example>
|
|
379
|
+
|
|
359
380
|
### I.attachFile
|
|
360
381
|
|
|
361
382
|
Attaches a file to a file input element.
|
|
@@ -41,6 +41,9 @@ export class SessionAnalyst implements Agent {
|
|
|
41
41
|
|
|
42
42
|
Crucial distinction: "the app misbehaved" vs "the automation could not interact with the app". ONLY the first is a Defect. If the automation gives up before the app responds — timeout, retries exhausted, dead loop / loop detected, could not click or find an element — that is an Execution issue regardless of what the log calls it. Failure inside the automation ≠ failure inside the product.
|
|
43
43
|
|
|
44
|
+
The action log is more authoritative than the scenario title. If the actual submitted data, page state, or action sequence does not match the scenario title, classify it as Execution issue and do not list that scenario under What works. Do NOT infer a product Defect or UX issue from behavior caused by incorrect test data or an automation mismatch.
|
|
45
|
+
Negative test data is valid when it matches a negative scenario. Do not call intentionally invalid input wrong data when the scenario expects rejection or validation feedback.
|
|
46
|
+
|
|
44
47
|
A solitary failure where adjacent tests on the same feature passed → Execution, not Defect.
|
|
45
48
|
|
|
46
49
|
## Severity (defects only)
|
|
@@ -76,7 +79,7 @@ export class SessionAnalyst implements Agent {
|
|
|
76
79
|
|
|
77
80
|
## Brevity rules
|
|
78
81
|
|
|
79
|
-
- Headline: 2 sentences MAX. About the FEATURE, not the run. No counts, no "N tests", no "this session".
|
|
82
|
+
- Headline: 2 sentences MAX. About the FEATURE, not the run. No counts, no "N tests", no "this session". Never use these words: "exercised", "comprehensive", "notably", "this session", "module", "targeted", "covered creation".
|
|
80
83
|
- What works: feature name + test refs. NO parentheticals, NO caveats. If there's a caveat, the entry doesn't belong here.
|
|
81
84
|
- Defect title is the BUG ("Search returns non-matching results"), never the scenario name.
|
|
82
85
|
- Reproduce steps are imperative one-liners drawn from the log.
|
package/src/ai/task-agent.ts
CHANGED
|
@@ -44,7 +44,7 @@ export abstract class TaskAgent {
|
|
|
44
44
|
.filter((k) => !!k)
|
|
45
45
|
.join('\n\n');
|
|
46
46
|
|
|
47
|
-
tag('
|
|
47
|
+
tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')}`);
|
|
48
48
|
return dedent`
|
|
49
49
|
<knowledge>
|
|
50
50
|
Here is relevant knowledge for this page:
|
|
@@ -61,7 +61,7 @@ export abstract class TaskAgent {
|
|
|
61
61
|
|
|
62
62
|
const totalSections = toc.reduce((sum, entry) => sum + entry.sections.length, 0);
|
|
63
63
|
debugLog(`injecting experience TOC (${toc.length} files, ${totalSections} sections)`);
|
|
64
|
-
tag('
|
|
64
|
+
tag('operation').log(`Found ${toc.length} experience ${pluralize(toc.length, 'file')} (${totalSections} sections)`);
|
|
65
65
|
return renderExperienceToc(toc);
|
|
66
66
|
}
|
|
67
67
|
|
package/src/ai/tester.ts
CHANGED
|
@@ -25,7 +25,7 @@ import { Navigator } from './navigator.ts';
|
|
|
25
25
|
import type { Pilot } from './pilot.ts';
|
|
26
26
|
import { Provider } from './provider.ts';
|
|
27
27
|
import { Researcher } from './researcher.ts';
|
|
28
|
-
import { actionRule, focusedElementRule, locatorRule, multipleTabsRule, protectionRule, sectionContextRule } from './rules.ts';
|
|
28
|
+
import { actionRule, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, protectionRule, sectionContextRule } from './rules.ts';
|
|
29
29
|
import { TaskAgent } from './task-agent.ts';
|
|
30
30
|
import { createCodeceptJSTools, createSpecialContextTools } from './tools.ts';
|
|
31
31
|
|
|
@@ -773,6 +773,8 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
773
773
|
|
|
774
774
|
${sectionContextRule}
|
|
775
775
|
|
|
776
|
+
${formRequirementsRule}
|
|
777
|
+
|
|
776
778
|
${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
|
|
777
779
|
`;
|
|
778
780
|
}
|
package/src/ai/tools.ts
CHANGED
|
@@ -307,6 +307,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
307
307
|
|
|
308
308
|
Use cases:
|
|
309
309
|
- Typing into input fields (I.fillField, I.type)
|
|
310
|
+
- Setting checkboxes/radios to a definite state (I.checkOption, I.uncheckOption)
|
|
310
311
|
- Working with iframes (switch context with I.switchTo)
|
|
311
312
|
- Performing multiple form actions in a single batch
|
|
312
313
|
- Complex interactions requiring sequential commands
|
|
@@ -314,6 +315,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
|
|
|
314
315
|
Example - filling a form with context (PREFERRED):
|
|
315
316
|
I.fillField('Username', 'John', '.login-form')
|
|
316
317
|
I.selectOption('Country', 'USA', '.address-section')
|
|
318
|
+
I.checkOption('Agree', '.terms-section')
|
|
317
319
|
I.attachFile('input[type="file"]', 'path/to/file', '.upload-section')
|
|
318
320
|
|
|
319
321
|
Example - filling a form with ARIA locators:
|
|
@@ -17,6 +17,6 @@ export class ContextAriaCommand extends BaseCommand {
|
|
|
17
17
|
throw new Error('No ARIA snapshot available for current page');
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
tag('multiline').log(`ARIA Snapshot:\n\n${ariaSnapshot}
|
|
20
|
+
tag('multiline').log(`ARIA Snapshot:\n\n${ariaSnapshot}`, { maxLines: 10 });
|
|
21
21
|
}
|
|
22
22
|
}
|