explorbot 0.1.25 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +1 -1
- package/dist/src/ai/historian/codeceptjs.js +1 -1
- package/dist/src/ai/historian/experience.js +1 -1
- package/dist/src/ai/historian/playwright.js +1 -1
- package/dist/src/ai/historian/screencast.js +3 -3
- package/dist/src/ai/historian.js +1 -1
- package/dist/src/ai/navigator.js +4 -4
- package/dist/src/ai/pilot.js +8 -5
- package/dist/src/ai/planner.js +2 -4
- package/dist/src/ai/provider.js +5 -3
- package/dist/src/ai/researcher/locators.js +1 -1
- package/dist/src/ai/researcher.js +4 -4
- package/dist/src/ai/session-analyst.js +4 -1
- package/dist/src/ai/task-agent.js +2 -2
- package/dist/src/commands/context-aria-command.js +1 -1
- package/dist/src/commands/explore-command.js +67 -25
- package/dist/src/commands/test-command.js +2 -1
- package/dist/src/components/App.js +1 -33
- package/dist/src/components/LogPane.js +9 -3
- package/dist/src/experience-tracker.js +2 -2
- package/dist/src/explorbot.js +1 -1
- package/dist/src/reporter.js +24 -5
- package/dist/src/utils/log-filters.js +27 -0
- package/dist/src/utils/logger.js +28 -1
- package/dist/src/utils/next-steps.js +1 -7
- package/package.json +1 -1
- package/src/ai/historian/codeceptjs.ts +1 -1
- package/src/ai/historian/experience.ts +1 -1
- package/src/ai/historian/playwright.ts +1 -1
- package/src/ai/historian/screencast.ts +3 -3
- package/src/ai/historian.ts +1 -1
- package/src/ai/navigator.ts +4 -4
- package/src/ai/pilot.ts +8 -5
- package/src/ai/planner.ts +1 -3
- package/src/ai/provider.ts +5 -3
- package/src/ai/researcher/locators.ts +1 -1
- package/src/ai/researcher.ts +4 -4
- package/src/ai/session-analyst.ts +4 -1
- package/src/ai/task-agent.ts +2 -2
- package/src/commands/context-aria-command.ts +1 -1
- package/src/commands/explore-command.ts +68 -23
- package/src/commands/test-command.ts +2 -1
- package/src/components/App.tsx +0 -33
- package/src/components/LogPane.tsx +18 -3
- package/src/experience-tracker.ts +2 -2
- package/src/explorbot.ts +1 -1
- package/src/reporter.ts +24 -6
- package/src/utils/log-filters.ts +26 -0
- package/src/utils/logger.ts +24 -2
- package/src/utils/next-steps.ts +1 -6
package/dist/src/reporter.js
CHANGED
|
@@ -82,10 +82,12 @@ export class Reporter {
|
|
|
82
82
|
return;
|
|
83
83
|
}
|
|
84
84
|
try {
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
85
|
+
const result = await withQuietReporterLogs(async () => {
|
|
86
|
+
this.client = new Client({ apiKey: process.env.TESTOMATIO || '', title: this.buildTitle() });
|
|
87
|
+
const timeoutMs = Number(process.env.TESTOMATIO_TIMEOUT_MS || '15000');
|
|
88
|
+
const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve('timeout'), timeoutMs));
|
|
89
|
+
return await Promise.race([this.client.createRun({ configuration: { exploratory: true } }).then(() => 'success'), timeoutPromise]);
|
|
90
|
+
});
|
|
89
91
|
if (result === 'timeout') {
|
|
90
92
|
debugLog('Reporter run creation timed out');
|
|
91
93
|
return;
|
|
@@ -252,7 +254,9 @@ export class Reporter {
|
|
|
252
254
|
return;
|
|
253
255
|
}
|
|
254
256
|
try {
|
|
255
|
-
await
|
|
257
|
+
await withQuietReporterLogs(async () => {
|
|
258
|
+
await this.client.updateRunStatus('finished');
|
|
259
|
+
});
|
|
256
260
|
this.isRunStarted = false;
|
|
257
261
|
debugLog('Testomat.io run finished');
|
|
258
262
|
}
|
|
@@ -299,3 +303,18 @@ export class Reporter {
|
|
|
299
303
|
return;
|
|
300
304
|
}
|
|
301
305
|
}
|
|
306
|
+
async function withQuietReporterLogs(fn) {
|
|
307
|
+
const previousLevel = process.env.TESTOMATIO_LOG_LEVEL;
|
|
308
|
+
process.env.TESTOMATIO_LOG_LEVEL = 'ERROR';
|
|
309
|
+
try {
|
|
310
|
+
return await fn();
|
|
311
|
+
}
|
|
312
|
+
finally {
|
|
313
|
+
if (previousLevel === undefined) {
|
|
314
|
+
process.env.TESTOMATIO_LOG_LEVEL = undefined;
|
|
315
|
+
}
|
|
316
|
+
else {
|
|
317
|
+
process.env.TESTOMATIO_LOG_LEVEL = previousLevel;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export class RecentStepFilter {
|
|
2
|
+
ttlMs;
|
|
3
|
+
recentStepKeys = new Map();
|
|
4
|
+
constructor(ttlMs = 15000) {
|
|
5
|
+
this.ttlMs = ttlMs;
|
|
6
|
+
}
|
|
7
|
+
shouldSuppress(content, now = Date.now()) {
|
|
8
|
+
const key = normalizeStepCommand(content);
|
|
9
|
+
if (!key)
|
|
10
|
+
return false;
|
|
11
|
+
for (const [existingKey, timestamp] of this.recentStepKeys) {
|
|
12
|
+
if (now - timestamp > this.ttlMs) {
|
|
13
|
+
this.recentStepKeys.delete(existingKey);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
if (this.recentStepKeys.has(key))
|
|
17
|
+
return true;
|
|
18
|
+
this.recentStepKeys.set(key, now);
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function normalizeStepCommand(content) {
|
|
23
|
+
const normalized = content.replace(/\s+/g, ' ').trim();
|
|
24
|
+
if (!normalized.startsWith('I.'))
|
|
25
|
+
return null;
|
|
26
|
+
return normalized.toLowerCase();
|
|
27
|
+
}
|
package/dist/src/utils/logger.js
CHANGED
|
@@ -7,6 +7,7 @@ import dedent from 'dedent';
|
|
|
7
7
|
import stripAnsi from 'strip-ansi';
|
|
8
8
|
import { ConfigParser } from '../config.js';
|
|
9
9
|
import { Observability } from "../observability.js";
|
|
10
|
+
import { RecentStepFilter } from "./log-filters.js";
|
|
10
11
|
import { parseMarkdownToTerminal } from "./markdown-terminal.js";
|
|
11
12
|
class DebugFilter {
|
|
12
13
|
patterns = [];
|
|
@@ -51,6 +52,7 @@ const debugFilter = new DebugFilter();
|
|
|
51
52
|
class ConsoleDestination {
|
|
52
53
|
verboseMode = false;
|
|
53
54
|
forceEnabled = false;
|
|
55
|
+
recentSteps = new RecentStepFilter();
|
|
54
56
|
isEnabled() {
|
|
55
57
|
return this.forceEnabled || !process.env.INK_RUNNING;
|
|
56
58
|
}
|
|
@@ -65,6 +67,10 @@ class ConsoleDestination {
|
|
|
65
67
|
return;
|
|
66
68
|
if (entry.type === 'html')
|
|
67
69
|
return;
|
|
70
|
+
if (entry.type === 'operation' && !this.verboseMode)
|
|
71
|
+
return;
|
|
72
|
+
if (entry.type === 'step' && !this.verboseMode && this.recentSteps.shouldSuppress(entry.content))
|
|
73
|
+
return;
|
|
68
74
|
let content = entry.content;
|
|
69
75
|
if (entry.type === 'multiline') {
|
|
70
76
|
const cleaned = stripAnsi(dedent(entry.content));
|
|
@@ -84,6 +90,9 @@ class ConsoleDestination {
|
|
|
84
90
|
else if (entry.type === 'step') {
|
|
85
91
|
content = chalk.gray(` ${content}`);
|
|
86
92
|
}
|
|
93
|
+
else if (entry.type === 'operation') {
|
|
94
|
+
content = chalk.gray(` · ${content}`);
|
|
95
|
+
}
|
|
87
96
|
else if (entry.type === 'substep') {
|
|
88
97
|
content = chalk.gray(` > ${content}`);
|
|
89
98
|
}
|
|
@@ -213,6 +222,8 @@ class ReactDestination {
|
|
|
213
222
|
return this.debugMode;
|
|
214
223
|
}
|
|
215
224
|
shouldWrite(entry) {
|
|
225
|
+
if (entry.type === 'operation' && !this.debugMode)
|
|
226
|
+
return false;
|
|
216
227
|
if (entry.type !== 'debug')
|
|
217
228
|
return true;
|
|
218
229
|
if (this.debugMode)
|
|
@@ -269,7 +280,7 @@ class CaptainDestination {
|
|
|
269
280
|
}
|
|
270
281
|
stopCapture() {
|
|
271
282
|
this.capturing = false;
|
|
272
|
-
const logs = this.entries.filter((e) => e.type !== 'debug' && e.type !== 'html' && e.type !== 'multiline').map((e) => `[${e.type}] ${e.content}`);
|
|
283
|
+
const logs = this.entries.filter((e) => e.type !== 'debug' && e.type !== 'html' && e.type !== 'multiline' && e.type !== 'operation').map((e) => `[${e.type}] ${e.content}`);
|
|
273
284
|
this.entries = [];
|
|
274
285
|
return logs;
|
|
275
286
|
}
|
|
@@ -377,6 +388,7 @@ class Logger {
|
|
|
377
388
|
}
|
|
378
389
|
return;
|
|
379
390
|
}
|
|
391
|
+
const options = this.extractLogOptions(type, args);
|
|
380
392
|
let content = this.processArgs(args);
|
|
381
393
|
if (type === 'step' && args[0]?.toCode) {
|
|
382
394
|
content = args[0].toCode();
|
|
@@ -386,6 +398,7 @@ class Logger {
|
|
|
386
398
|
content,
|
|
387
399
|
timestamp: new Date(),
|
|
388
400
|
originalArgs: args,
|
|
401
|
+
maxLines: options?.maxLines,
|
|
389
402
|
};
|
|
390
403
|
if (this.file.isEnabled())
|
|
391
404
|
this.file.write(entry);
|
|
@@ -421,6 +434,20 @@ class Logger {
|
|
|
421
434
|
multiline(...args) {
|
|
422
435
|
this.log('multiline', ...args);
|
|
423
436
|
}
|
|
437
|
+
extractLogOptions(type, args) {
|
|
438
|
+
if (type !== 'multiline')
|
|
439
|
+
return null;
|
|
440
|
+
const last = args[args.length - 1];
|
|
441
|
+
if (!last || typeof last !== 'object' || Array.isArray(last))
|
|
442
|
+
return null;
|
|
443
|
+
if (!('maxLines' in last))
|
|
444
|
+
return null;
|
|
445
|
+
args.pop();
|
|
446
|
+
const maxLines = Number(last.maxLines);
|
|
447
|
+
if (!Number.isFinite(maxLines) || maxLines <= 0)
|
|
448
|
+
return null;
|
|
449
|
+
return { maxLines };
|
|
450
|
+
}
|
|
424
451
|
}
|
|
425
452
|
const logger = Logger.getInstance();
|
|
426
453
|
let stepSpanParent = null;
|
|
@@ -27,11 +27,5 @@ export function printNextSteps(sections) {
|
|
|
27
27
|
}
|
|
28
28
|
blocks.push(lines.join('\n'));
|
|
29
29
|
}
|
|
30
|
-
|
|
31
|
-
if (i > 0)
|
|
32
|
-
tag('info').log('');
|
|
33
|
-
for (const line of blocks[i].split('\n')) {
|
|
34
|
-
tag('info').log(line);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
30
|
+
tag('multiline').log(blocks.join('\n\n'), { maxLines: 18 });
|
|
37
31
|
}
|
package/package.json
CHANGED
|
@@ -102,7 +102,7 @@ export function WithCodeceptJS<T extends Constructor>(Base: T) {
|
|
|
102
102
|
writeFileSync(filePath, lines.join('\n'));
|
|
103
103
|
this.savedFiles.add(filePath);
|
|
104
104
|
|
|
105
|
-
tag('
|
|
105
|
+
tag('operation').log(`Saved plan tests to: ${relativeToCwd(filePath)}`);
|
|
106
106
|
return filePath;
|
|
107
107
|
}
|
|
108
108
|
|
|
@@ -56,7 +56,7 @@ export function WithExperience<T extends Constructor>(Base: T) {
|
|
|
56
56
|
|
|
57
57
|
await this.stopScreencast();
|
|
58
58
|
|
|
59
|
-
tag('
|
|
59
|
+
tag('operation').log(`Historian saved session for: ${task.description}`);
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
private async reportSession(test: Test, steps: SessionStep[]): Promise<void> {
|
|
@@ -140,7 +140,7 @@ export function WithPlaywright<T extends Constructor>(Base: T) {
|
|
|
140
140
|
writeFileSync(filePath, lines.join('\n'));
|
|
141
141
|
this.savedFiles.add(filePath);
|
|
142
142
|
|
|
143
|
-
tag('
|
|
143
|
+
tag('operation').log(`Saved plan tests to: ${relativeToCwd(filePath)}`);
|
|
144
144
|
return filePath;
|
|
145
145
|
}
|
|
146
146
|
|
|
@@ -92,7 +92,7 @@ export function WithScreencast<T extends Constructor>(Base: T) {
|
|
|
92
92
|
this.screencastTask = test?._explorbotTest || null;
|
|
93
93
|
this.screencastLastChapter = null;
|
|
94
94
|
} catch (err) {
|
|
95
|
-
tag('
|
|
95
|
+
tag('operation').log(`Screencast start failed: ${(err as Error).message}`);
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
|
|
@@ -116,7 +116,7 @@ export function WithScreencast<T extends Constructor>(Base: T) {
|
|
|
116
116
|
try {
|
|
117
117
|
await this.screencastPage.screencast.stop();
|
|
118
118
|
} catch (err) {
|
|
119
|
-
tag('
|
|
119
|
+
tag('operation').log(`Screencast stop failed: ${(err as Error).message}`);
|
|
120
120
|
}
|
|
121
121
|
this.screencastActive = false;
|
|
122
122
|
this.screencastPage = null;
|
|
@@ -126,7 +126,7 @@ export function WithScreencast<T extends Constructor>(Base: T) {
|
|
|
126
126
|
if (path) {
|
|
127
127
|
this.savedFiles.add(path);
|
|
128
128
|
task?.addArtifact?.(path);
|
|
129
|
-
tag('
|
|
129
|
+
tag('operation').log(`Saved screencast: ${relativeToCwd(path)}`);
|
|
130
130
|
}
|
|
131
131
|
}
|
|
132
132
|
};
|
package/src/ai/historian.ts
CHANGED
|
@@ -62,6 +62,6 @@ export class Historian extends HistorianBase {
|
|
|
62
62
|
|
|
63
63
|
writeFileSync(filePath, content);
|
|
64
64
|
this.savedFiles.add(filePath);
|
|
65
|
-
tag('
|
|
65
|
+
tag('operation').log(`Updated test file with healed steps: ${relativeToCwd(filePath)}`);
|
|
66
66
|
}
|
|
67
67
|
}
|
package/src/ai/navigator.ts
CHANGED
|
@@ -206,7 +206,7 @@ class Navigator implements Agent {
|
|
|
206
206
|
if (!actionResult.isInsideIframe) {
|
|
207
207
|
const successful = this.experienceTracker.getSuccessfulExperience(actionResult);
|
|
208
208
|
if (successful.length > 0) {
|
|
209
|
-
tag('
|
|
209
|
+
tag('operation').log(`Found ${successful.length} experience ${pluralize(successful.length, 'file')} for: ${actionResult.url}`);
|
|
210
210
|
experience = `<experience>\nPast successful recipes recorded from prior runs for this page. Prefer these solutions first if they match the goal.\n\n${successful.join('\n\n')}\n</experience>`;
|
|
211
211
|
}
|
|
212
212
|
}
|
|
@@ -307,7 +307,7 @@ class Navigator implements Agent {
|
|
|
307
307
|
stop();
|
|
308
308
|
return;
|
|
309
309
|
}
|
|
310
|
-
tag('
|
|
310
|
+
tag('operation').log('Feeding failures back to AI for a new batch...');
|
|
311
311
|
let contextMsg = 'Previous solutions did not work. Analyze the failures and try DIFFERENT strategies (not syntactic variants of the same locator).\n\n';
|
|
312
312
|
if (batchFailures.length > 0) {
|
|
313
313
|
const lines = batchFailures
|
|
@@ -633,7 +633,7 @@ class Navigator implements Agent {
|
|
|
633
633
|
|
|
634
634
|
const cachedVerification = actionResult.getVerification(message);
|
|
635
635
|
if (cachedVerification !== null) {
|
|
636
|
-
tag('
|
|
636
|
+
tag('operation').log(`Reusing cached verification: ${cachedVerification ? 'PASS' : 'FAIL'}`);
|
|
637
637
|
return { verified: cachedVerification, successfulCodes: [], assertionSteps: [], totalAttempted: 0 };
|
|
638
638
|
}
|
|
639
639
|
|
|
@@ -654,7 +654,7 @@ class Navigator implements Agent {
|
|
|
654
654
|
const toc = this.experienceTracker.getExperienceTableOfContents(actionResult);
|
|
655
655
|
if (toc.length > 0) {
|
|
656
656
|
const totalSections = toc.reduce((sum, entry) => sum + entry.sections.length, 0);
|
|
657
|
-
tag('
|
|
657
|
+
tag('operation').log(`Found ${toc.length} experience ${pluralize(toc.length, 'file')} (${totalSections} sections) for: ${actionResult.url}`);
|
|
658
658
|
experience = renderExperienceToc(toc);
|
|
659
659
|
}
|
|
660
660
|
}
|
package/src/ai/pilot.ts
CHANGED
|
@@ -104,7 +104,7 @@ export class Pilot implements Agent {
|
|
|
104
104
|
|
|
105
105
|
const schema = z.object({
|
|
106
106
|
decision: z.enum(['pass', 'fail', 'continue', 'skipped']).describe('pass = test succeeded, fail = test failed, continue = tester should keep going, skipped = scenario is irrelevant OR systematic execution failures prevented testing'),
|
|
107
|
-
reason: z.string().describe('
|
|
107
|
+
reason: z.string().describe('Concise user-facing reason, maximum 1 short sentence and 120 characters. Do NOT repeat the decision status; explain only the evidence. For continue: explain why rejected and suggest alternatives.'),
|
|
108
108
|
guidance: z.string().nullable().describe('Required for "continue": specific actionable instruction for the tester — what exactly to verify, retry differently, or complete next. Be concrete.'),
|
|
109
109
|
requestVerification: z
|
|
110
110
|
.string()
|
|
@@ -177,7 +177,7 @@ export class Pilot implements Agent {
|
|
|
177
177
|
}
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
-
tag('info').log(`Pilot: ${result.decision}
|
|
180
|
+
tag('info').log(`Pilot: ${result.decision} - ${result.reason}`);
|
|
181
181
|
task.summary = result.reason;
|
|
182
182
|
|
|
183
183
|
const verdictState = screenshotState || currentState;
|
|
@@ -221,7 +221,7 @@ export class Pilot implements Agent {
|
|
|
221
221
|
|
|
222
222
|
const schema = z.object({
|
|
223
223
|
decision: z.enum(['allow', 'fail', 'continue', 'skipped']).describe('allow = reset proceeds, fail = test failed (stop looping), continue = veto reset, tester should act on current page instead, skipped = scenario is irrelevant or cannot be executed'),
|
|
224
|
-
reason: z.string().describe('
|
|
224
|
+
reason: z.string().describe('Concise evidence-only reason, maximum 1 short sentence and 120 characters. Do not restate the decision.'),
|
|
225
225
|
guidance: z.string().nullable().describe('Required for "continue": concrete instruction for what the tester should do instead of resetting (e.g. which tool to call, what to verify).'),
|
|
226
226
|
});
|
|
227
227
|
|
|
@@ -388,8 +388,9 @@ export class Pilot implements Agent {
|
|
|
388
388
|
- "continue": tester hasn't completed the goal; provide concrete guidance (which tool, what to check).
|
|
389
389
|
If a verify() asserted a state that was ALREADY TRUE before the test, it proves nothing — reject.
|
|
390
390
|
|
|
391
|
-
reason field:
|
|
392
|
-
what was verified, what failed,
|
|
391
|
+
reason field: one short sentence, maximum 120 characters. Do NOT restate the decision
|
|
392
|
+
("scenario goal achieved/not achieved"). State what happened: what was verified, what failed,
|
|
393
|
+
or what evidence was found.
|
|
393
394
|
`;
|
|
394
395
|
}
|
|
395
396
|
|
|
@@ -1017,6 +1018,8 @@ export class Pilot implements Agent {
|
|
|
1017
1018
|
Response format:
|
|
1018
1019
|
PROGRESS: <1 sentence assessment>
|
|
1019
1020
|
NEXT: <specific actionable instruction for Tester>
|
|
1021
|
+
|
|
1022
|
+
Keep user-facing reasons concise: one short sentence, maximum 120 characters, evidence only, no repeated verdict wording.
|
|
1020
1023
|
`;
|
|
1021
1024
|
}
|
|
1022
1025
|
}
|
package/src/ai/planner.ts
CHANGED
|
@@ -36,7 +36,7 @@ const TasksSchema = z.object({
|
|
|
36
36
|
scenario: z.string().describe('A single sentence describing what to test'),
|
|
37
37
|
priority: z.enum(['critical', 'important', 'high', 'normal', 'low']).describe('Priority of the task based on business importance'),
|
|
38
38
|
startUrl: z.string().nullable().describe('Start URL for the test if different from plan URL (only for tests on visited subpages)'),
|
|
39
|
-
steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "
|
|
39
|
+
steps: z.array(z.string()).describe('List of steps to perform for this scenario. Each step should be a specific action (e.g., "Open the form", "Enter required data", "Submit the form"). Keep steps atomic and actionable.'),
|
|
40
40
|
expectedOutcomes: z
|
|
41
41
|
.array(z.string())
|
|
42
42
|
.describe('List of expected outcomes that can be verified. Each outcome should be simple, specific, and easy to check (e.g., "Success message appears", "URL changes to /dashboard", "Form field shows error"). Keep outcomes atomic - do not combine multiple checks into one.'),
|
|
@@ -226,9 +226,7 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
226
226
|
}
|
|
227
227
|
}
|
|
228
228
|
|
|
229
|
-
const availableStyles = Object.keys(getStyles()).join(', ');
|
|
230
229
|
tag('success').log(`Planning complete! ${this.currentPlan.tests.length} tests in plan: ${this.currentPlan.title}`);
|
|
231
|
-
tag('info').log(`Planning style: ${this.lastStyleName} (available: ${availableStyles})`);
|
|
232
230
|
|
|
233
231
|
if (state.url) registerPlan(state.url, this.currentPlan, feature, state.hash);
|
|
234
232
|
|
package/src/ai/provider.ts
CHANGED
|
@@ -294,8 +294,11 @@ export class Provider {
|
|
|
294
294
|
}
|
|
295
295
|
throw new ContextLengthError(error.message || error.toString());
|
|
296
296
|
}
|
|
297
|
-
|
|
298
|
-
|
|
297
|
+
const message = error.message || error.toString();
|
|
298
|
+
if (message !== 'No response text from AI') {
|
|
299
|
+
tag('error').log(message);
|
|
300
|
+
}
|
|
301
|
+
throw new AiError(message);
|
|
299
302
|
}
|
|
300
303
|
}
|
|
301
304
|
|
|
@@ -376,7 +379,6 @@ export class Provider {
|
|
|
376
379
|
} catch (error: any) {
|
|
377
380
|
clearActivity();
|
|
378
381
|
if (error?.message?.includes('Tool choice is required')) {
|
|
379
|
-
tag('warning').log('Model completed without calling a tool, returning empty result');
|
|
380
382
|
return { text: '', toolCalls: [], toolResults: [], response: { messages: [] }, usage: null };
|
|
381
383
|
}
|
|
382
384
|
if (error?.name === 'AbortError') throw error;
|
|
@@ -80,7 +80,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
|
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
tag('
|
|
83
|
+
tag('operation').log(`Validated ${locators.length} locators: ${locators.length - broken} valid, ${broken} broken`);
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
async fixBrokenSections(result: ResearchResult, conversation: Conversation): Promise<void> {
|
package/src/ai/researcher.ts
CHANGED
|
@@ -151,7 +151,7 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
151
151
|
if (!deep && !force) {
|
|
152
152
|
const similar = await findSimilarResearch(combinedHtml);
|
|
153
153
|
if (similar) {
|
|
154
|
-
tag('
|
|
154
|
+
tag('operation').log('Similar research found, reusing cached result');
|
|
155
155
|
if (stateHash) saveResearch(stateHash, similar, combinedHtml);
|
|
156
156
|
tag('multiline').log(formatResearchSummary(similar));
|
|
157
157
|
tag('success').log('Research complete (reused)');
|
|
@@ -316,10 +316,10 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
316
316
|
|
|
317
317
|
tag('multiline').log(formatResearchSummary(result.text, { visionUsed: this.hasScreenshotToAnalyze }));
|
|
318
318
|
tag('success').log('Research complete');
|
|
319
|
-
if (researchFile) tag('
|
|
319
|
+
if (researchFile) tag('operation').log(`Research file saved to: ${researchFile}`);
|
|
320
320
|
if (this.actionResult?.screenshotFile) {
|
|
321
321
|
const screenshotPath = outputPath('states', this.actionResult.screenshotFile);
|
|
322
|
-
tag('
|
|
322
|
+
tag('operation').log(`UI screenshot: file://${screenshotPath}`);
|
|
323
323
|
}
|
|
324
324
|
|
|
325
325
|
await this.hooksRunner.runAfterHook('researcher', state.url);
|
|
@@ -467,7 +467,7 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
467
467
|
.filter((k) => !!k)
|
|
468
468
|
.join('\n\n');
|
|
469
469
|
|
|
470
|
-
tag('
|
|
470
|
+
tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')} for: ${this.actionResult.url}`);
|
|
471
471
|
knowledge = `
|
|
472
472
|
<hint>
|
|
473
473
|
Here is relevant knowledge for this page:
|
|
@@ -41,6 +41,9 @@ export class SessionAnalyst implements Agent {
|
|
|
41
41
|
|
|
42
42
|
Crucial distinction: "the app misbehaved" vs "the automation could not interact with the app". ONLY the first is a Defect. If the automation gives up before the app responds — timeout, retries exhausted, dead loop / loop detected, could not click or find an element — that is an Execution issue regardless of what the log calls it. Failure inside the automation ≠ failure inside the product.
|
|
43
43
|
|
|
44
|
+
The action log is more authoritative than the scenario title. If the actual submitted data, page state, or action sequence does not match the scenario title, classify it as Execution issue and do not list that scenario under What works. Do NOT infer a product Defect or UX issue from behavior caused by incorrect test data or an automation mismatch.
|
|
45
|
+
Negative test data is valid when it matches a negative scenario. Do not call intentionally invalid input wrong data when the scenario expects rejection or validation feedback.
|
|
46
|
+
|
|
44
47
|
A solitary failure where adjacent tests on the same feature passed → Execution, not Defect.
|
|
45
48
|
|
|
46
49
|
## Severity (defects only)
|
|
@@ -76,7 +79,7 @@ export class SessionAnalyst implements Agent {
|
|
|
76
79
|
|
|
77
80
|
## Brevity rules
|
|
78
81
|
|
|
79
|
-
- Headline: 2 sentences MAX. About the FEATURE, not the run. No counts, no "N tests", no "this session".
|
|
82
|
+
- Headline: 2 sentences MAX. About the FEATURE, not the run. No counts, no "N tests", no "this session". Never use these words: "exercised", "comprehensive", "notably", "this session", "module", "targeted", "covered creation".
|
|
80
83
|
- What works: feature name + test refs. NO parentheticals, NO caveats. If there's a caveat, the entry doesn't belong here.
|
|
81
84
|
- Defect title is the BUG ("Search returns non-matching results"), never the scenario name.
|
|
82
85
|
- Reproduce steps are imperative one-liners drawn from the log.
|
package/src/ai/task-agent.ts
CHANGED
|
@@ -44,7 +44,7 @@ export abstract class TaskAgent {
|
|
|
44
44
|
.filter((k) => !!k)
|
|
45
45
|
.join('\n\n');
|
|
46
46
|
|
|
47
|
-
tag('
|
|
47
|
+
tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')}`);
|
|
48
48
|
return dedent`
|
|
49
49
|
<knowledge>
|
|
50
50
|
Here is relevant knowledge for this page:
|
|
@@ -61,7 +61,7 @@ export abstract class TaskAgent {
|
|
|
61
61
|
|
|
62
62
|
const totalSections = toc.reduce((sum, entry) => sum + entry.sections.length, 0);
|
|
63
63
|
debugLog(`injecting experience TOC (${toc.length} files, ${totalSections} sections)`);
|
|
64
|
-
tag('
|
|
64
|
+
tag('operation').log(`Found ${toc.length} experience ${pluralize(toc.length, 'file')} (${totalSections} sections)`);
|
|
65
65
|
return renderExperienceToc(toc);
|
|
66
66
|
}
|
|
67
67
|
|
|
@@ -17,6 +17,6 @@ export class ContextAriaCommand extends BaseCommand {
|
|
|
17
17
|
throw new Error('No ARIA snapshot available for current page');
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
tag('multiline').log(`ARIA Snapshot:\n\n${ariaSnapshot}
|
|
20
|
+
tag('multiline').log(`ARIA Snapshot:\n\n${ariaSnapshot}`, { maxLines: 10 });
|
|
21
21
|
}
|
|
22
22
|
}
|
|
@@ -7,7 +7,6 @@ import { type Plan, type Test, TestResult } from '../test-plan.js';
|
|
|
7
7
|
import { getCliName } from '../utils/cli-name.ts';
|
|
8
8
|
import { ErrorPageError } from '../utils/error-page.ts';
|
|
9
9
|
import { tag } from '../utils/logger.js';
|
|
10
|
-
import { jsonToTable } from '../utils/markdown-parser.js';
|
|
11
10
|
import { type NextStepSection, printNextSteps, relativeToCwd } from '../utils/next-steps.ts';
|
|
12
11
|
import { safeFilename } from '../utils/strings.ts';
|
|
13
12
|
import { BaseCommand, type Suggestion } from './base-command.js';
|
|
@@ -88,14 +87,14 @@ export class ExploreCommand extends BaseCommand {
|
|
|
88
87
|
const t = tests[i];
|
|
89
88
|
lines.push(` ${String(i + 1).padStart(2)}. [${this.originLabel(t)}] [${t.priority.padEnd(9)}] ${t.scenario}`);
|
|
90
89
|
}
|
|
91
|
-
tag('multiline').log(lines.join('\n'));
|
|
90
|
+
tag('multiline').log(lines.join('\n'), { maxLines: 24 });
|
|
92
91
|
}
|
|
93
92
|
|
|
94
93
|
private async runFreshMode(mainUrl: string | undefined, feature: string | undefined, styles?: string[]): Promise<void> {
|
|
95
94
|
await this.runAllStyles(mainUrl, feature, undefined, undefined, styles);
|
|
95
|
+
this.rememberCurrentPlan();
|
|
96
96
|
const mainPlan = this.explorBot.getCurrentPlan();
|
|
97
97
|
if (!mainPlan) return;
|
|
98
|
-
this.completedPlans.push(mainPlan);
|
|
99
98
|
|
|
100
99
|
if (feature || this.isLimitReached()) return;
|
|
101
100
|
|
|
@@ -270,6 +269,7 @@ export class ExploreCommand extends BaseCommand {
|
|
|
270
269
|
const styleList = styles ?? Object.keys(getStyles());
|
|
271
270
|
let fresh = true;
|
|
272
271
|
for (const style of styleList) {
|
|
272
|
+
if (this.isLimitReached()) break;
|
|
273
273
|
if (!fresh && pageUrl && !this.dryRun) {
|
|
274
274
|
await this.explorBot.visit(pageUrl);
|
|
275
275
|
}
|
|
@@ -278,10 +278,19 @@ export class ExploreCommand extends BaseCommand {
|
|
|
278
278
|
if (this.dryRun) opts.noSave = true;
|
|
279
279
|
await this.planWithRetry(feature, opts, pageUrl);
|
|
280
280
|
await this.runPendingTests();
|
|
281
|
+
this.rememberCurrentPlan();
|
|
281
282
|
fresh = false;
|
|
282
283
|
}
|
|
283
284
|
}
|
|
284
285
|
|
|
286
|
+
private rememberCurrentPlan(): void {
|
|
287
|
+
const plan = this.explorBot.getCurrentPlan();
|
|
288
|
+
if (!plan) return;
|
|
289
|
+
if (this.completedPlans.includes(plan)) return;
|
|
290
|
+
if (plan.tests.every((test) => test.startTime == null)) return;
|
|
291
|
+
this.completedPlans.push(plan);
|
|
292
|
+
}
|
|
293
|
+
|
|
285
294
|
private async planWithRetry(feature: string | undefined, opts: { fresh: boolean; style: string; extend?: Plan; completedPlans?: Plan[]; noSave?: boolean }, pageUrl?: string): Promise<void> {
|
|
286
295
|
const before = new Set(this.explorBot.getCurrentPlan()?.tests ?? []);
|
|
287
296
|
|
|
@@ -401,34 +410,63 @@ export class ExploreCommand extends BaseCommand {
|
|
|
401
410
|
|
|
402
411
|
if (allTests.length === 0) return;
|
|
403
412
|
|
|
404
|
-
const hasSubPages = this.completedPlans.
|
|
413
|
+
const hasSubPages = new Set(this.completedPlans.map((plan) => plan.title)).size > 1;
|
|
405
414
|
const hasOrigin = this.oldTestRefs.size > 0;
|
|
406
|
-
const
|
|
415
|
+
const completed = allTests.map(({ test, planTitle }, index) => {
|
|
407
416
|
const durationMs = test.getDurationMs();
|
|
408
417
|
const duration = durationMs != null ? `${(durationMs / 1000).toFixed(1)}s` : '-';
|
|
409
418
|
let status = 'failed';
|
|
410
419
|
if (test.isSuccessful) status = 'passed';
|
|
411
420
|
else if (test.isSkipped) status = 'skipped';
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
421
|
+
return {
|
|
422
|
+
index: index + 1,
|
|
423
|
+
status,
|
|
424
|
+
title: test.scenario.replace(/\s+/g, ' ').trim(),
|
|
425
|
+
priority: test.priority,
|
|
426
|
+
duration,
|
|
427
|
+
durationMs: durationMs ?? 0,
|
|
428
|
+
steps: Object.keys(test.notes).length,
|
|
429
|
+
origin: hasOrigin ? this.originLabel(test) : '',
|
|
430
|
+
planTitle: hasSubPages ? planTitle : '',
|
|
419
431
|
};
|
|
420
|
-
|
|
421
|
-
|
|
432
|
+
});
|
|
433
|
+
const passed = completed.filter((t) => t.status === 'passed').length;
|
|
434
|
+
const failed = completed.filter((t) => t.status === 'failed').length;
|
|
435
|
+
const skipped = completed.filter((t) => t.status === 'skipped').length;
|
|
436
|
+
const totalSeconds = completed.reduce((sum, t) => sum + t.durationMs, 0) / 1000;
|
|
437
|
+
const lines = [`Results: ${passed} passed, ${failed} failed, ${skipped} skipped - ${formatDuration(totalSeconds)}`];
|
|
438
|
+
|
|
439
|
+
const failedTests = completed.filter((t) => t.status === 'failed');
|
|
440
|
+
if (failedTests.length > 0) {
|
|
441
|
+
lines.push('', 'Failed tests:');
|
|
442
|
+
for (const test of failedTests) {
|
|
443
|
+
lines.push(` #${test.index} [${test.priority}] ${test.title} (${test.duration}, ${test.steps} steps)`);
|
|
422
444
|
}
|
|
423
|
-
|
|
424
|
-
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
const slowTests = completed
|
|
448
|
+
.filter((t) => t.durationMs >= 1000)
|
|
449
|
+
.sort((a, b) => b.durationMs - a.durationMs)
|
|
450
|
+
.slice(0, 3);
|
|
451
|
+
if (slowTests.length > 0) {
|
|
452
|
+
lines.push('', 'Slowest tests:');
|
|
453
|
+
for (const test of slowTests) {
|
|
454
|
+
lines.push(` #${test.index} ${test.duration} - ${test.title}`);
|
|
425
455
|
}
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
const
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
const detailLines = completed
|
|
459
|
+
.map((test) => {
|
|
460
|
+
const details = [test.origin, test.planTitle].filter(Boolean).join(' - ');
|
|
461
|
+
return details ? ` #${test.index} ${details}` : '';
|
|
462
|
+
})
|
|
463
|
+
.filter(Boolean);
|
|
464
|
+
if (detailLines.length > 0) {
|
|
465
|
+
lines.push('', 'Details:');
|
|
466
|
+
lines.push(...detailLines);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
tag('multiline').log(lines.join('\n'));
|
|
432
470
|
tag('info').log(`${figureSet.tick} ${allTests.length} tests completed`);
|
|
433
471
|
}
|
|
434
472
|
|
|
@@ -463,8 +501,8 @@ export class ExploreCommand extends BaseCommand {
|
|
|
463
501
|
}
|
|
464
502
|
|
|
465
503
|
if (screencasts.length > 0) {
|
|
466
|
-
const commands = screencasts.map((f) => ({ label: '', command: relativeToCwd(f) }));
|
|
467
504
|
const screencastDir = relativeToCwd(outputPath('screencasts'));
|
|
505
|
+
const commands = [{ label: 'Folder', command: screencastDir }];
|
|
468
506
|
const planSlugs = [...new Set(this.completedPlans.map((p) => safeFilename(p.title)).filter(Boolean))];
|
|
469
507
|
for (const slug of planSlugs) {
|
|
470
508
|
commands.push({ label: 'Browse plan', command: `ls ${screencastDir}/${slug}-*` });
|
|
@@ -529,3 +567,10 @@ function parseRatio(s: string): number | null {
|
|
|
529
567
|
if (Number.isNaN(n) || n < 0 || n > 1) return null;
|
|
530
568
|
return n;
|
|
531
569
|
}
|
|
570
|
+
|
|
571
|
+
function formatDuration(seconds: number): string {
|
|
572
|
+
if (seconds < 60) return `${seconds.toFixed(1)}s`;
|
|
573
|
+
const minutes = Math.floor(seconds / 60);
|
|
574
|
+
const remainingSeconds = Math.round(seconds % 60);
|
|
575
|
+
return `${minutes}m ${remainingSeconds}s`;
|
|
576
|
+
}
|
|
@@ -69,7 +69,8 @@ export class TestCommand extends BaseCommand {
|
|
|
69
69
|
|
|
70
70
|
tag('info').log(`Launching ${toExecute.length} test scenario(s).`);
|
|
71
71
|
const tester = this.explorBot.agentTester();
|
|
72
|
-
for (const test of toExecute) {
|
|
72
|
+
for (const [index, test] of toExecute.entries()) {
|
|
73
|
+
tag('info').log(`Starting test ${index + 1}/${toExecute.length}: ${test.scenario}`);
|
|
73
74
|
await tester.test(test);
|
|
74
75
|
}
|
|
75
76
|
tag('success').log('Test execution finished');
|