explorbot 0.5.7 → 0.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +25 -3
- package/boat/prima/README.md +3 -0
- package/boat/prima/src/cli.ts +7 -1
- package/boat/prima/src/envelope.ts +18 -4
- package/boat/prima/src/help.ts +15 -4
- package/boat/prima/src/prima.ts +181 -25
- package/dist/bin/explorbot-cli.js +31 -3
- package/dist/boat/prima/src/cli.js +6 -1
- package/dist/boat/prima/src/envelope.js +16 -3
- package/dist/boat/prima/src/help.js +14 -4
- package/dist/boat/prima/src/prima.js +191 -27
- package/dist/package.json +1 -1
- package/dist/rules/navigator/verification-actions.md +5 -3
- package/dist/rules/planner/ignored-pages.md +6 -0
- package/dist/rules/quartermaster/ux/accessibility.md +8 -0
- package/dist/rules/quartermaster/ux/charts.md +6 -0
- package/dist/rules/quartermaster/ux/copy.md +9 -0
- package/dist/rules/quartermaster/ux/forms.md +7 -0
- package/dist/rules/quartermaster/ux/interaction.md +7 -0
- package/dist/rules/quartermaster/ux/motion.md +5 -0
- package/dist/rules/quartermaster/ux/navigation.md +7 -0
- package/dist/rules/quartermaster/ux/readability.md +6 -0
- package/dist/rules/quartermaster/ux/responsive.md +7 -0
- package/dist/src/ai/judge.d.ts +3 -1
- package/dist/src/ai/judge.js +13 -3
- package/dist/src/ai/navigator.js +3 -2
- package/dist/src/ai/pilot.js +10 -4
- package/dist/src/ai/planner.js +9 -1
- package/dist/src/ai/provider.d.ts +4 -0
- package/dist/src/ai/provider.js +27 -5
- package/dist/src/ai/quartermaster.d.ts +31 -3
- package/dist/src/ai/quartermaster.js +262 -17
- package/dist/src/ai/tester.js +6 -3
- package/dist/src/commands/explore-command.js +7 -0
- package/dist/src/commands/init-command.d.ts +1 -0
- package/dist/src/commands/init-command.js +87 -36
- package/dist/src/components/InitWizard.d.ts +5 -1
- package/dist/src/components/InitWizard.js +29 -11
- package/dist/src/config.d.ts +2 -0
- package/dist/src/config.js +29 -7
- package/dist/src/explorbot.d.ts +1 -1
- package/dist/src/explorbot.js +8 -13
- package/dist/src/explorer.js +1 -0
- package/dist/src/global-config.js +6 -2
- package/dist/src/utils/ux-probe.d.ts +25 -0
- package/dist/src/utils/ux-probe.js +155 -0
- package/docs/basics/getting-started.md +23 -10
- package/docs/basics/providers.md +2 -0
- package/docs/reference/commands.md +5 -2
- package/docs/reference/configuration.md +1 -0
- package/package.json +1 -1
- package/rules/navigator/verification-actions.md +5 -3
- package/rules/planner/ignored-pages.md +6 -0
- package/rules/quartermaster/ux/accessibility.md +8 -0
- package/rules/quartermaster/ux/charts.md +6 -0
- package/rules/quartermaster/ux/copy.md +9 -0
- package/rules/quartermaster/ux/forms.md +7 -0
- package/rules/quartermaster/ux/interaction.md +7 -0
- package/rules/quartermaster/ux/motion.md +5 -0
- package/rules/quartermaster/ux/navigation.md +7 -0
- package/rules/quartermaster/ux/readability.md +6 -0
- package/rules/quartermaster/ux/responsive.md +7 -0
- package/src/ai/judge.ts +11 -3
- package/src/ai/navigator.ts +2 -2
- package/src/ai/pilot.ts +10 -4
- package/src/ai/planner.ts +10 -1
- package/src/ai/provider.ts +29 -6
- package/src/ai/quartermaster.ts +300 -18
- package/src/ai/tester.ts +5 -3
- package/src/commands/explore-command.ts +8 -0
- package/src/commands/init-command.ts +105 -38
- package/src/components/InitWizard.tsx +43 -11
- package/src/config.ts +32 -7
- package/src/explorbot.ts +9 -13
- package/src/explorer.ts +1 -0
- package/src/global-config.ts +5 -2
- package/src/utils/ux-probe.ts +163 -0
package/src/ai/quartermaster.ts
CHANGED
|
@@ -1,36 +1,53 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import dedent from 'dedent';
|
|
4
|
+
import type { Locator, Page } from 'playwright';
|
|
3
5
|
import { z } from 'zod';
|
|
4
6
|
import type { ActionResult } from '../action-result.ts';
|
|
5
7
|
import { outputPath } from '../config.ts';
|
|
6
8
|
import type Explorer from '../explorer.ts';
|
|
7
9
|
import type { StateManager, StateTransition, WebPageState } from '../state-manager.ts';
|
|
8
10
|
import type { Task } from '../test-plan.ts';
|
|
11
|
+
import { ariaRefSelector, ariaRefSnapshot, parseAriaRefs } from '../utils/aria-ref.ts';
|
|
12
|
+
import { compactAriaSnapshot } from '../utils/aria.ts';
|
|
9
13
|
import { createDebug, tag } from '../utils/logger.ts';
|
|
14
|
+
import { RulesLoader } from '../utils/rules-loader.ts';
|
|
10
15
|
import { isCodeceptToolName } from '../utils/step-analyzer.ts';
|
|
16
|
+
import { UX_FOCUS_ATTR, type UxMeasurements, countRunningAnimations, measureUx, renderUxMeasurements, uxProbeArgs } from '../utils/ux-probe.ts';
|
|
11
17
|
import type { Conversation, ToolExecution } from './conversation.ts';
|
|
12
18
|
import type { Provider } from './provider.ts';
|
|
13
19
|
|
|
14
20
|
const debugLog = createDebug('explorbot:quartermaster');
|
|
15
21
|
|
|
22
|
+
const UX_RULES = ['ux/accessibility', 'ux/interaction', 'ux/responsive', 'ux/navigation', 'ux/copy', 'ux/forms', 'ux/readability', 'ux/motion', 'ux/charts'];
|
|
23
|
+
const UX_BREAKPOINTS = [768, 375];
|
|
24
|
+
const UX_FOCUS_SELECTOR = `[${UX_FOCUS_ATTR}]`;
|
|
25
|
+
const UX_SETTLE_MS = 400;
|
|
26
|
+
const UX_ARIA_CAP = 8000;
|
|
27
|
+
const UX_SCREENSHOT_TIMEOUT = 5000;
|
|
28
|
+
const UX_MAX_FINDINGS = 6;
|
|
29
|
+
const AGENT_NAME = 'quartermaster';
|
|
30
|
+
const FOCUS_CONTAINER_ROLES = ['main', 'region', 'form', 'dialog', 'alertdialog', 'navigation', 'complementary', 'article', 'table', 'grid', 'tabpanel', 'list', 'group', 'banner', 'contentinfo', 'search'];
|
|
31
|
+
|
|
16
32
|
export class Quartermaster {
|
|
17
33
|
private provider: Provider;
|
|
18
34
|
private model?: any;
|
|
19
35
|
private outputDir: string;
|
|
20
36
|
private pageAnalyses: Map<string, PageAnalysis> = new Map();
|
|
21
37
|
private unsubscribe: (() => void) | null = null;
|
|
22
|
-
private explorer: Explorer
|
|
38
|
+
private explorer: Explorer;
|
|
23
39
|
private pendingAnalyses: Promise<void>[] = [];
|
|
24
40
|
|
|
25
|
-
constructor(provider: Provider, options?: { model?: any }) {
|
|
41
|
+
constructor(provider: Provider, explorer: Explorer, options?: { model?: any }) {
|
|
26
42
|
this.provider = provider;
|
|
43
|
+
this.explorer = explorer;
|
|
27
44
|
this.model = options?.model;
|
|
28
45
|
|
|
29
46
|
this.outputDir = outputPath('a11y');
|
|
30
47
|
}
|
|
31
48
|
|
|
32
|
-
start(
|
|
33
|
-
this.
|
|
49
|
+
start(stateManager: StateManager): void {
|
|
50
|
+
if (this.unsubscribe) return;
|
|
34
51
|
mkdirSync(this.outputDir, { recursive: true });
|
|
35
52
|
|
|
36
53
|
this.unsubscribe = stateManager.onStateChange((event) => {
|
|
@@ -61,26 +78,14 @@ export class Quartermaster {
|
|
|
61
78
|
}
|
|
62
79
|
|
|
63
80
|
private async runAxeAnalysis(state: WebPageState): Promise<void> {
|
|
64
|
-
const page = this.explorer
|
|
81
|
+
const page = this.explorer.page;
|
|
65
82
|
if (!page || !state.hash) {
|
|
66
83
|
debugLog('No page available for axe analysis');
|
|
67
84
|
return;
|
|
68
85
|
}
|
|
69
86
|
|
|
70
87
|
try {
|
|
71
|
-
const
|
|
72
|
-
const results = await new AxeBuilder({ page }).analyze();
|
|
73
|
-
|
|
74
|
-
const violations: AxeViolation[] = results.violations.map((v) => ({
|
|
75
|
-
id: v.id,
|
|
76
|
-
impact: v.impact as AxeViolation['impact'],
|
|
77
|
-
description: v.description,
|
|
78
|
-
helpUrl: v.helpUrl,
|
|
79
|
-
nodes: v.nodes.map((n) => ({
|
|
80
|
-
html: n.html,
|
|
81
|
-
target: n.target as string[],
|
|
82
|
-
})),
|
|
83
|
-
}));
|
|
88
|
+
const violations = await this.axeViolations(page);
|
|
84
89
|
|
|
85
90
|
this.pageAnalyses.set(state.hash, {
|
|
86
91
|
url: state.url,
|
|
@@ -138,6 +143,24 @@ export class Quartermaster {
|
|
|
138
143
|
}
|
|
139
144
|
}
|
|
140
145
|
|
|
146
|
+
async auditUx(result: ActionResult, context: UxAuditContext): Promise<string> {
|
|
147
|
+
const focus = await this.markUxFocus(result, context);
|
|
148
|
+
tag('step').log(`Auditing UX of ${focus.area}`);
|
|
149
|
+
|
|
150
|
+
try {
|
|
151
|
+
const evidence = await this.explorer.withPage((page) => this.collectUxEvidence(page));
|
|
152
|
+
return await this.writeUxReport(result, focus, evidence, context);
|
|
153
|
+
} finally {
|
|
154
|
+
await this.explorer
|
|
155
|
+
.withPage((page) =>
|
|
156
|
+
page.evaluate((attr) => {
|
|
157
|
+
for (const el of Array.from(document.querySelectorAll(`[${attr}]`))) el.removeAttribute(attr);
|
|
158
|
+
}, UX_FOCUS_ATTR)
|
|
159
|
+
)
|
|
160
|
+
.catch(() => {});
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
141
164
|
private async generateSemanticAnalysis(axeViolations: AxeViolation[], executions: ToolExecution[], initialState: ActionResult): Promise<SemanticIssue[]> {
|
|
142
165
|
const failedExecs = executions.filter((e) => !e.wasSuccessful);
|
|
143
166
|
if (failedExecs.length === 0 && axeViolations.length === 0) return [];
|
|
@@ -247,6 +270,265 @@ Focus on what would confuse a real user or caused the agent to make mistakes.`;
|
|
|
247
270
|
|
|
248
271
|
return content;
|
|
249
272
|
}
|
|
273
|
+
|
|
274
|
+
private async axeViolations(page: Page, include?: string): Promise<AxeViolation[]> {
|
|
275
|
+
const { AxeBuilder } = await import('@axe-core/playwright');
|
|
276
|
+
let builder = new AxeBuilder({ page });
|
|
277
|
+
if (include) builder = builder.include(include);
|
|
278
|
+
const results = await builder.analyze();
|
|
279
|
+
|
|
280
|
+
return results.violations.map((v) => ({
|
|
281
|
+
id: v.id,
|
|
282
|
+
impact: v.impact as AxeViolation['impact'],
|
|
283
|
+
description: v.description,
|
|
284
|
+
helpUrl: v.helpUrl,
|
|
285
|
+
nodes: v.nodes.map((n) => ({
|
|
286
|
+
html: n.html,
|
|
287
|
+
target: n.target as string[],
|
|
288
|
+
})),
|
|
289
|
+
}));
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
private async markUxFocus(result: ActionResult, context: UxAuditContext): Promise<UxFocus> {
|
|
293
|
+
const mark = (locator: Locator) =>
|
|
294
|
+
locator
|
|
295
|
+
.first()
|
|
296
|
+
.evaluate((el, attr) => el.setAttribute(attr, ''), UX_FOCUS_ATTR)
|
|
297
|
+
.then(() => true)
|
|
298
|
+
.catch(() => false);
|
|
299
|
+
|
|
300
|
+
const region = result.overlay;
|
|
301
|
+
const locateRegion = (page: Page) => {
|
|
302
|
+
if (region.root) return page.locator(region.root);
|
|
303
|
+
if (region.xpath) return page.locator(`xpath=${region.xpath}`);
|
|
304
|
+
const name = region.name || undefined;
|
|
305
|
+
return page.getByRole('dialog', { name }).or(page.getByRole('alertdialog', { name }));
|
|
306
|
+
};
|
|
307
|
+
const regionFocus = { area: region.name || `open ${region.type}`, reason: `it is the ${region.type} open on the page` };
|
|
308
|
+
|
|
309
|
+
if (!context.focus && region.isOpen && (await this.explorer.withPage((page) => mark(locateRegion(page))))) return regionFocus;
|
|
310
|
+
|
|
311
|
+
if (context.focus || context.commands.length) {
|
|
312
|
+
const picked = await this.pickUxFocus(context, mark);
|
|
313
|
+
if (picked) return picked;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
if (region.isOpen && (await this.explorer.withPage((page) => mark(locateRegion(page))))) return regionFocus;
|
|
317
|
+
|
|
318
|
+
let reason = 'nothing narrower is open or was touched by the session on this page';
|
|
319
|
+
if (context.focus) reason = `"${context.focus}" matched no single container`;
|
|
320
|
+
return this.explorer.withPage(async (page) => {
|
|
321
|
+
if (await mark(page.locator('main, [role="main"]'))) return { area: 'main content', reason };
|
|
322
|
+
await mark(page.locator('body'));
|
|
323
|
+
return { area: 'whole page', reason: `${reason} and the page has no main landmark` };
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
private async pickUxFocus(context: UxAuditContext, mark: (locator: Locator) => Promise<boolean>): Promise<UxFocus | null> {
|
|
328
|
+
const snapshot = await this.explorer.withPage(ariaRefSnapshot).catch(() => '');
|
|
329
|
+
const candidates = parseAriaRefs(snapshot).filter((entry) => FOCUS_CONTAINER_ROLES.includes(entry.role));
|
|
330
|
+
if (!candidates.length) return null;
|
|
331
|
+
|
|
332
|
+
const hints: string[] = [];
|
|
333
|
+
if (context.focus) hints.push(`The developer asked to review: ${context.focus}`);
|
|
334
|
+
if (context.commands.length) hints.push('Commands the session ran on this page, latest last:', ...context.commands.map((command) => `- ${command}`));
|
|
335
|
+
|
|
336
|
+
const schema = z.object({
|
|
337
|
+
ref: z.enum(['none', ...candidates.map((entry) => entry.ref)]).describe('ref of the chosen container, or none'),
|
|
338
|
+
area: z.string().describe('the area named as a user would call it, a few words'),
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
const prompt = dedent`
|
|
342
|
+
Pick the page area a developer is working on, so a UX review covers it and nothing else.
|
|
343
|
+
Choose the smallest container that holds everything the hints point at.
|
|
344
|
+
Answer none when no listed container fits the hints.
|
|
345
|
+
|
|
346
|
+
<hints>
|
|
347
|
+
${hints.join('\n')}
|
|
348
|
+
</hints>
|
|
349
|
+
|
|
350
|
+
<containers>
|
|
351
|
+
${candidates.map((entry) => `- [${entry.ref}] ${entry.role} "${entry.name}"`).join('\n')}
|
|
352
|
+
</containers>
|
|
353
|
+
|
|
354
|
+
<page_aria>
|
|
355
|
+
${snapshot.slice(0, UX_ARIA_CAP)}
|
|
356
|
+
</page_aria>
|
|
357
|
+
`;
|
|
358
|
+
|
|
359
|
+
const response = await this.provider.generateObject([{ role: 'user', content: prompt }], schema, this.model, { agentName: AGENT_NAME }).catch(() => null);
|
|
360
|
+
const ref = response?.object?.ref;
|
|
361
|
+
if (!ref || ref === 'none') return null;
|
|
362
|
+
if (!(await this.explorer.withPage((page) => mark(page.locator(ariaRefSelector(ref)))))) return null;
|
|
363
|
+
|
|
364
|
+
let reason = 'the session commands on this page act inside it';
|
|
365
|
+
if (context.focus) reason = `it holds "${context.focus}"`;
|
|
366
|
+
return { area: response.object.area, reason };
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
private async collectUxEvidence(page: Page): Promise<UxEvidence> {
|
|
370
|
+
const evidence: UxEvidence = { breakpoints: [], axe: [], notes: [] };
|
|
371
|
+
evidence.axe = await this.axeViolations(page, UX_FOCUS_SELECTOR).catch((error) => {
|
|
372
|
+
evidence.notes.push(`axe could not run: ${error.message}`);
|
|
373
|
+
return [];
|
|
374
|
+
});
|
|
375
|
+
|
|
376
|
+
const ignoredMotion = await this.animationsIgnoringReducedMotion(page);
|
|
377
|
+
if (ignoredMotion) evidence.notes.push(`${ignoredMotion} animations in the area keep running when reduced motion is requested`);
|
|
378
|
+
|
|
379
|
+
const width = await page.evaluate(() => window.innerWidth);
|
|
380
|
+
evidence.breakpoints.push(await this.captureUxBreakpoint(page));
|
|
381
|
+
|
|
382
|
+
const narrower = UX_BREAKPOINTS.filter((breakpoint) => breakpoint < width);
|
|
383
|
+
if (!narrower.length) return evidence;
|
|
384
|
+
|
|
385
|
+
const viewport = page.viewportSize();
|
|
386
|
+
if (!viewport) {
|
|
387
|
+
evidence.notes.push('the browser window has no fixed viewport, so narrower widths were not measured');
|
|
388
|
+
return evidence;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
try {
|
|
392
|
+
for (const breakpoint of narrower) {
|
|
393
|
+
await page.setViewportSize({ width: breakpoint, height: viewport.height });
|
|
394
|
+
await page.waitForTimeout(UX_SETTLE_MS);
|
|
395
|
+
if ((await page.evaluate(() => window.innerWidth)) !== breakpoint) {
|
|
396
|
+
evidence.notes.push(`resizing to ${breakpoint}px had no effect, so that width was not measured`);
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
evidence.breakpoints.push(await this.captureUxBreakpoint(page));
|
|
400
|
+
}
|
|
401
|
+
} finally {
|
|
402
|
+
await page.setViewportSize(viewport);
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
return evidence;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
private async animationsIgnoringReducedMotion(page: Page): Promise<number> {
|
|
409
|
+
if (!(await page.evaluate(countRunningAnimations, UX_FOCUS_SELECTOR))) return 0;
|
|
410
|
+
|
|
411
|
+
try {
|
|
412
|
+
await page.emulateMedia({ reducedMotion: 'reduce' });
|
|
413
|
+
await page.waitForTimeout(UX_SETTLE_MS);
|
|
414
|
+
return await page.evaluate(countRunningAnimations, UX_FOCUS_SELECTOR);
|
|
415
|
+
} finally {
|
|
416
|
+
await page.emulateMedia({ reducedMotion: null });
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
private async captureUxBreakpoint(page: Page): Promise<UxBreakpoint> {
|
|
421
|
+
const focus = page.locator(UX_FOCUS_SELECTOR).first();
|
|
422
|
+
const measurements = await page.evaluate(measureUx, uxProbeArgs(UX_FOCUS_SELECTOR));
|
|
423
|
+
const aria = await focus.ariaSnapshot().catch(() => '');
|
|
424
|
+
|
|
425
|
+
const breakpoint: UxBreakpoint = { measurements, aria: compactAriaSnapshot(aria, true).slice(0, UX_ARIA_CAP) };
|
|
426
|
+
if (!this.provider.hasVision()) return breakpoint;
|
|
427
|
+
|
|
428
|
+
const screenshot = await focus.screenshot({ timeout: UX_SCREENSHOT_TIMEOUT }).catch(() => null);
|
|
429
|
+
if (screenshot) breakpoint.screenshot = screenshot.toString('base64');
|
|
430
|
+
return breakpoint;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
private async writeUxReport(result: ActionResult, focus: UxFocus, evidence: UxEvidence, context: UxAuditContext): Promise<string> {
|
|
434
|
+
const url = result.url || '';
|
|
435
|
+
const rules = [RulesLoader.loadRules(AGENT_NAME, UX_RULES, url), this.provider.getSystemPromptForAgent(AGENT_NAME, url)].filter(Boolean).join('\n\n');
|
|
436
|
+
|
|
437
|
+
const system = dedent`
|
|
438
|
+
<role>
|
|
439
|
+
You review the UX of one area of a web page a developer is building right now.
|
|
440
|
+
The report goes back to that developer, so every finding must point at something they can find in that area.
|
|
441
|
+
</role>
|
|
442
|
+
|
|
443
|
+
<focus>
|
|
444
|
+
Area under review: ${focus.area}, chosen because ${focus.reason}.
|
|
445
|
+
Report only on elements inside it. Page-wide header, navigation and footer are out of scope unless they are the area under review.
|
|
446
|
+
</focus>
|
|
447
|
+
|
|
448
|
+
<scope>
|
|
449
|
+
You see one page of one app, not the app.
|
|
450
|
+
- Report that a text fails and why. Never propose replacement wording, terminology, tone or language: the app's vocabulary is unknown to you.
|
|
451
|
+
- Never recommend colors, fonts, spacing values, design tokens or visual style: the project owns its design system.
|
|
452
|
+
- Sizes, gaps, overflow and contrast come only from <measurements> and <axe>. Never estimate them from a screenshot.
|
|
453
|
+
- Report only what the evidence shows. A check the evidence cannot settle is skipped, not guessed.
|
|
454
|
+
</scope>
|
|
455
|
+
|
|
456
|
+
<rules>
|
|
457
|
+
Each rule set carries the priority its findings are reported under.
|
|
458
|
+
|
|
459
|
+
${rules}
|
|
460
|
+
</rules>
|
|
461
|
+
|
|
462
|
+
<output>
|
|
463
|
+
Markdown, no preamble.
|
|
464
|
+
|
|
465
|
+
## Focus
|
|
466
|
+
One line: the area reviewed and the widths covered.
|
|
467
|
+
|
|
468
|
+
Then one section per priority that has findings, in this order: ## Critical, ## High, ## Medium, ## Low. Omit empty sections.
|
|
469
|
+
Each finding is one bullet of one sentence, at most 30 words:
|
|
470
|
+
- **<element as the user sees it>** (<rule set>, <width or "all widths">): <evidence>, <what it costs the user>.
|
|
471
|
+
|
|
472
|
+
One bullet per cause: elements failing the same check share a bullet naming at most three of them.
|
|
473
|
+
Write at most ${UX_MAX_FINDINGS} bullets in total across all sections, most severe first; drop the rest.
|
|
474
|
+
No fix wording, no rule ids, no list of skipped checks, no summary, no tables.
|
|
475
|
+
When nothing fails, write the Focus section and the line: No UX issues found in this area.
|
|
476
|
+
</output>
|
|
477
|
+
`;
|
|
478
|
+
|
|
479
|
+
const sections = [`<page url="${url}" title="${result.title || ''}"></page>`];
|
|
480
|
+
if (context.commands.length) sections.push(`<session_commands>\n${context.commands.join('\n')}\n</session_commands>`);
|
|
481
|
+
sections.push(`<axe>\n${evidence.axe.map((v) => `[${v.impact}] ${v.id}: ${v.description} — ${v.nodes[0]?.html.slice(0, 100) || ''}`).join('\n') || 'no violations'}\n</axe>`);
|
|
482
|
+
if (evidence.notes.length) sections.push(`<notes>\n${evidence.notes.join('\n')}\n</notes>`);
|
|
483
|
+
for (const breakpoint of evidence.breakpoints) {
|
|
484
|
+
const aria = breakpoint.aria || 'the area is not rendered at this width';
|
|
485
|
+
sections.push(`<breakpoint width="${breakpoint.measurements.width}">\n<measurements>\n${renderUxMeasurements(breakpoint.measurements)}\n</measurements>\n<aria>\n${aria}\n</aria>\n</breakpoint>`);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
const shots = evidence.breakpoints.filter((breakpoint) => breakpoint.screenshot);
|
|
489
|
+
if (!shots.length) sections.push('No screenshots are attached: judge from ARIA and measurements only, and say in Focus that visual checks were not possible.');
|
|
490
|
+
|
|
491
|
+
const content: any[] = [{ type: 'text', text: sections.join('\n\n') }];
|
|
492
|
+
for (const shot of shots) {
|
|
493
|
+
content.push({ type: 'text', text: `Screenshot of the area at ${shot.measurements.width}px:` });
|
|
494
|
+
content.push({ type: 'file', mediaType: 'image/png', data: shot.screenshot });
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
let model = this.model || this.provider.getModelForAgent(AGENT_NAME);
|
|
498
|
+
if (shots.length) model = this.provider.getVisionModel();
|
|
499
|
+
|
|
500
|
+
const response = await this.provider.chat(
|
|
501
|
+
[
|
|
502
|
+
{ role: 'system', content: system },
|
|
503
|
+
{ role: 'user', content },
|
|
504
|
+
],
|
|
505
|
+
model,
|
|
506
|
+
{ agentName: AGENT_NAME }
|
|
507
|
+
);
|
|
508
|
+
return response.text;
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
export interface UxAuditContext {
|
|
513
|
+
focus?: string;
|
|
514
|
+
commands: string[];
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
interface UxFocus {
|
|
518
|
+
area: string;
|
|
519
|
+
reason: string;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
interface UxBreakpoint {
|
|
523
|
+
measurements: UxMeasurements;
|
|
524
|
+
aria: string;
|
|
525
|
+
screenshot?: string;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
interface UxEvidence {
|
|
529
|
+
breakpoints: UxBreakpoint[];
|
|
530
|
+
axe: AxeViolation[];
|
|
531
|
+
notes: string[];
|
|
250
532
|
}
|
|
251
533
|
|
|
252
534
|
interface AxeViolation {
|
package/src/ai/tester.ts
CHANGED
|
@@ -366,8 +366,9 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
366
366
|
}
|
|
367
367
|
|
|
368
368
|
if (actionPerformed && !wasSuccessful) {
|
|
369
|
+
const noted = new Set(Object.values(task.notes).map((note) => note.message));
|
|
369
370
|
result?.toolExecutions
|
|
370
|
-
?.filter((execution: any) => !execution.wasSuccessful && execution.input?.explanation)
|
|
371
|
+
?.filter((execution: any) => !execution.wasSuccessful && execution.input?.explanation && !noted.has(execution.input.explanation))
|
|
371
372
|
.forEach((execution: any) => {
|
|
372
373
|
task.addNote(`Failed to ${execution.input.explanation} (${execution.toolName})`, TestResult.FAILED);
|
|
373
374
|
});
|
|
@@ -379,8 +380,9 @@ export class Tester extends TaskAgent implements Agent {
|
|
|
379
380
|
}
|
|
380
381
|
|
|
381
382
|
if (assertionPerformed) {
|
|
382
|
-
const
|
|
383
|
-
task.addNote(message
|
|
383
|
+
const passedVerifications = result?.toolExecutions?.filter((execution: any) => this.ASSERTION_TOOLS.includes(execution.toolName) && execution.wasSuccessful) || [];
|
|
384
|
+
for (const execution of passedVerifications) task.addNote(execution.output?.message || '', TestResult.PASSED);
|
|
385
|
+
const message = passedVerifications.at(-1)?.output?.message || '';
|
|
384
386
|
if (wasSuccessful) {
|
|
385
387
|
conversation.addUserText(dedent`
|
|
386
388
|
Assertion "${message}" successfully passed!
|
|
@@ -583,6 +583,14 @@ export class ExploreCommand extends BaseCommand {
|
|
|
583
583
|
});
|
|
584
584
|
}
|
|
585
585
|
|
|
586
|
+
sections.push({
|
|
587
|
+
label: 'Explore more',
|
|
588
|
+
commands: [
|
|
589
|
+
{ label: 'Whole site', command: `${cli} explore /` },
|
|
590
|
+
{ label: 'Docs', command: 'https://testomat.ai/docs/explorbot' },
|
|
591
|
+
],
|
|
592
|
+
});
|
|
593
|
+
|
|
586
594
|
printNextSteps(sections);
|
|
587
595
|
}
|
|
588
596
|
|
|
@@ -1,25 +1,30 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { dirname, extname, join, resolve } from 'node:path';
|
|
3
3
|
import chalk from 'chalk';
|
|
4
|
+
import figureSet from 'figures';
|
|
4
5
|
import { ConfigParser, type ModelRole, PROVIDERS, missingModelRoles } from '../config.ts';
|
|
5
6
|
import { findGlobalConfig, globalConfigPath, globalDir, globalEnvPath } from '../global-config.ts';
|
|
6
7
|
import { getCliName } from '../utils/cli-name.ts';
|
|
7
8
|
import { log, tag } from '../utils/logger.js';
|
|
8
|
-
import { relativeToCwd } from '../utils/next-steps.ts';
|
|
9
|
+
import { type NextStepCommand, type NextStepSection, printNextSteps, relativeToCwd } from '../utils/next-steps.ts';
|
|
9
10
|
|
|
10
|
-
function defaultConfigTemplate(provider: string): string {
|
|
11
|
+
function defaultConfigTemplate(provider: string, url?: string): string {
|
|
12
|
+
const appUrl = url || 'http://<your-app-host-here>';
|
|
11
13
|
return `// 'provider/model-id' uses a bundled provider.
|
|
12
14
|
// It is also possible to import provider as a module from Vercel AI SDK.
|
|
13
|
-
// https://
|
|
15
|
+
// https://testomat.ai/docs/explorbot
|
|
14
16
|
|
|
15
17
|
const config = {
|
|
16
18
|
web: {
|
|
17
19
|
// use application host without path prefix (e.g., http://localhost:3000)
|
|
18
|
-
url: '
|
|
20
|
+
url: '${appUrl}',
|
|
19
21
|
},
|
|
20
22
|
|
|
21
23
|
ai: {
|
|
22
|
-
${
|
|
24
|
+
// every role (model, visionModel, agenticModel) takes the recommended ${provider} models bundled with this version
|
|
25
|
+
// override a single role as 'provider/model-id':
|
|
26
|
+
// model: '${provider}/<model-id>',
|
|
27
|
+
recommendedModels: '${provider}',
|
|
23
28
|
},
|
|
24
29
|
|
|
25
30
|
reporter: {
|
|
@@ -37,8 +42,9 @@ export default config;
|
|
|
37
42
|
}
|
|
38
43
|
|
|
39
44
|
export function envTemplate(provider: string): string {
|
|
45
|
+
const { keyUrl } = PROVIDERS[provider];
|
|
40
46
|
const keyLines = Object.entries(PROVIDERS).map(([name, { envKey }]) => {
|
|
41
|
-
if (name === provider) return
|
|
47
|
+
if (name === provider) return `# Create a key at ${keyUrl} and paste it after '='\n${envKey}=`;
|
|
42
48
|
return `# ${envKey}=`;
|
|
43
49
|
});
|
|
44
50
|
|
|
@@ -69,10 +75,10 @@ export async function runInit(options: InitCommandOptions): Promise<void> {
|
|
|
69
75
|
const choice = await renderInitWizard('choose');
|
|
70
76
|
if (choice !== 'local') return;
|
|
71
77
|
|
|
72
|
-
const
|
|
73
|
-
if (!
|
|
78
|
+
const setup = await renderLocalSetupWizard();
|
|
79
|
+
if (!setup) return;
|
|
74
80
|
|
|
75
|
-
runInitCommand({ ...options,
|
|
81
|
+
runInitCommand({ ...options, ...setup });
|
|
76
82
|
}
|
|
77
83
|
|
|
78
84
|
export function writeGlobalConfig(provider: string, apiKey?: string): void {
|
|
@@ -82,24 +88,28 @@ export function writeGlobalConfig(provider: string, apiKey?: string): void {
|
|
|
82
88
|
|
|
83
89
|
mkdirSync(globalDir(), { recursive: true });
|
|
84
90
|
writeFileSync(globalConfigPath(), globalConfigTemplate(provider), 'utf8');
|
|
85
|
-
log(
|
|
91
|
+
tag('success').log(`${figureSet.tick} Created global config: ${globalConfigPath()}`);
|
|
86
92
|
|
|
87
93
|
const envKey = PROVIDERS[provider].envKey;
|
|
88
94
|
writeEnvKey(envKey, apiKey || '');
|
|
89
|
-
log(
|
|
95
|
+
tag('success').log(`${figureSet.tick} Stored ${envKey} in ${globalEnvPath()}`);
|
|
90
96
|
|
|
91
97
|
const missing = missingModelRoles(provider);
|
|
92
98
|
if (missing.length) {
|
|
93
99
|
tag('warning').log(`No recommended ${missing.join(' and ')} for ${provider} — set the model ids in ${globalConfigPath()}`);
|
|
94
100
|
}
|
|
95
101
|
if (!apiKey && !process.env[envKey]) {
|
|
96
|
-
|
|
102
|
+
printApiKeySteps(provider, globalEnvPath());
|
|
97
103
|
}
|
|
98
104
|
|
|
99
105
|
log('');
|
|
100
|
-
log('
|
|
101
|
-
|
|
102
|
-
|
|
106
|
+
log(`${figureSet.star} What's next:`);
|
|
107
|
+
printInitNextSteps({
|
|
108
|
+
siteCommands: [
|
|
109
|
+
{ label: 'Any site', command: `${getCliName()} explore https://your-app.example.com` },
|
|
110
|
+
{ label: 'Saved sites', command: `${getCliName()} sites` },
|
|
111
|
+
],
|
|
112
|
+
});
|
|
103
113
|
}
|
|
104
114
|
|
|
105
115
|
export function runInitCommand(options: InitCommandOptions): void {
|
|
@@ -141,38 +151,42 @@ export function runInitCommand(options: InitCommandOptions): void {
|
|
|
141
151
|
process.exit(1);
|
|
142
152
|
}
|
|
143
153
|
|
|
144
|
-
writeFileSync(outPath, defaultConfigTemplate(provider), 'utf8');
|
|
145
|
-
log(
|
|
154
|
+
writeFileSync(outPath, defaultConfigTemplate(provider, options.url), 'utf8');
|
|
155
|
+
tag('success').log(`${figureSet.tick} Created config file: ${relativeToCwd(outPath)}`);
|
|
146
156
|
|
|
147
157
|
const envPath = resolve(process.cwd(), '.env');
|
|
148
158
|
if (!existsSync(envPath)) {
|
|
149
159
|
writeFileSync(envPath, `${envTemplate(provider)}\n`, 'utf8');
|
|
150
|
-
log(
|
|
160
|
+
tag('success').log(`${figureSet.tick} Created env file: ${relativeToCwd(envPath)}`);
|
|
151
161
|
} else {
|
|
152
162
|
log(`Env file already exists: ${relativeToCwd(envPath)}`);
|
|
153
163
|
}
|
|
154
164
|
|
|
165
|
+
const envKey = PROVIDERS[provider].envKey;
|
|
166
|
+
if (options.apiKey) {
|
|
167
|
+
writeEnvKey(envKey, options.apiKey, envPath);
|
|
168
|
+
tag('success').log(`${figureSet.tick} Stored ${envKey} in ${relativeToCwd(envPath)}`);
|
|
169
|
+
}
|
|
170
|
+
if (!options.apiKey && !process.env[envKey]) {
|
|
171
|
+
printApiKeySteps(provider, relativeToCwd(envPath));
|
|
172
|
+
}
|
|
173
|
+
if (!options.url) {
|
|
174
|
+
tag('warning').log(`Set your application URL in web.url: ${relativeToCwd(outPath)}`);
|
|
175
|
+
}
|
|
176
|
+
|
|
155
177
|
const missing = missingModelRoles(provider);
|
|
156
178
|
if (missing.length) {
|
|
157
179
|
tag('warning').log(`No recommended ${missing.join(' and ')} for ${provider} — set the model ids in ${relativeToCwd(outPath)}`);
|
|
158
180
|
}
|
|
159
181
|
|
|
160
|
-
log('');
|
|
161
|
-
log('Next steps:');
|
|
162
|
-
log('1. Configure AI provider in .env');
|
|
163
|
-
log('2. Set AI models config file');
|
|
164
|
-
log('3. Set web application URL in the config file');
|
|
165
|
-
log('4. Add initial knowledge (how to authorize to the application, etc.)');
|
|
166
|
-
tag('substep').log(chalk.yellow(`${getCliName()} learn '*' 'to authorize use these credentials: admin@example.com / secret123'`));
|
|
167
|
-
tag('substep').log('You can use ${env.LOGIN} and ${env.PASSWORD} to reference environment variables.');
|
|
168
|
-
|
|
169
|
-
log('5. Launch application on a relative URL');
|
|
170
|
-
tag('substep').log(chalk.yellow(`${getCliName()} start /dashboard`));
|
|
171
|
-
|
|
172
182
|
if (!existsSync('./output')) {
|
|
173
183
|
mkdirSync('./output', { recursive: true });
|
|
174
|
-
log(
|
|
184
|
+
tag('success').log(`${figureSet.tick} Created directory: output`);
|
|
175
185
|
}
|
|
186
|
+
|
|
187
|
+
log('');
|
|
188
|
+
log(`${figureSet.star} What's next:`);
|
|
189
|
+
printInitNextSteps({ url: options.url });
|
|
176
190
|
} catch (error) {
|
|
177
191
|
log('Failed to create config file:', error);
|
|
178
192
|
process.exit(1);
|
|
@@ -227,13 +241,13 @@ async function renderInitWizard(mode: 'choose' | 'global'): Promise<'local' | 'g
|
|
|
227
241
|
});
|
|
228
242
|
}
|
|
229
243
|
|
|
230
|
-
async function
|
|
244
|
+
async function renderLocalSetupWizard(): Promise<LocalSetup | null> {
|
|
231
245
|
const [{ render }, React, InitWizard] = await Promise.all([import('ink'), import('react'), import('../components/InitWizard.js').then((m) => m.default)]);
|
|
232
246
|
|
|
233
247
|
return new Promise((resolve) => {
|
|
234
|
-
const finish = (
|
|
248
|
+
const finish = (setup: LocalSetup | null) => {
|
|
235
249
|
unmount();
|
|
236
|
-
resolve(
|
|
250
|
+
resolve(setup);
|
|
237
251
|
};
|
|
238
252
|
const { unmount } = render(
|
|
239
253
|
React.createElement(InitWizard, {
|
|
@@ -242,7 +256,7 @@ async function renderLocalProviderWizard(): Promise<string | null> {
|
|
|
242
256
|
onLocal: () => finish(null),
|
|
243
257
|
onComplete: () => finish(null),
|
|
244
258
|
onCancel: () => finish(null),
|
|
245
|
-
|
|
259
|
+
onLocalSetup: (setup) => finish(setup),
|
|
246
260
|
}),
|
|
247
261
|
{ exitOnCtrlC: false, patchConsole: false }
|
|
248
262
|
);
|
|
@@ -274,7 +288,7 @@ function globalConfigTemplate(provider: string): string {
|
|
|
274
288
|
// Models are written as 'provider/model-id' so they resolve without a local node_modules.
|
|
275
289
|
// The key is read from ${envKey} in ~/.explorbot/.env
|
|
276
290
|
// Model ids are snapshotted from the recommendations of this Explorbot version.
|
|
277
|
-
// https://
|
|
291
|
+
// https://testomat.ai/docs/explorbot
|
|
278
292
|
const config = {
|
|
279
293
|
ai: {
|
|
280
294
|
${modelLines(provider)}
|
|
@@ -292,8 +306,47 @@ export default config;
|
|
|
292
306
|
`;
|
|
293
307
|
}
|
|
294
308
|
|
|
295
|
-
function
|
|
296
|
-
const
|
|
309
|
+
function printInitNextSteps(options: { url?: string; siteCommands?: NextStepCommand[] }): void {
|
|
310
|
+
const cli = getCliName();
|
|
311
|
+
const sections: NextStepSection[] = [
|
|
312
|
+
{
|
|
313
|
+
label: 'Try the demo on a sample app',
|
|
314
|
+
commands: [{ command: `${cli} explore https://todomvc.com/examples/react/dist/` }],
|
|
315
|
+
},
|
|
316
|
+
];
|
|
317
|
+
|
|
318
|
+
if (options.url) {
|
|
319
|
+
sections.push({
|
|
320
|
+
label: 'Explore your app',
|
|
321
|
+
commands: [
|
|
322
|
+
{ label: 'Whole site', command: `${cli} explore ${options.url}` },
|
|
323
|
+
{ label: 'One page', command: `${cli} explore /dashboard` },
|
|
324
|
+
],
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
if (options.siteCommands) sections.push({ label: 'Explore any site', commands: options.siteCommands });
|
|
329
|
+
|
|
330
|
+
sections.push(
|
|
331
|
+
{
|
|
332
|
+
label: 'Run it from VS Code',
|
|
333
|
+
commands: [{ command: 'https://testomat.ai/explorbot' }],
|
|
334
|
+
},
|
|
335
|
+
{
|
|
336
|
+
label: 'Docs',
|
|
337
|
+
commands: [{ command: 'https://testomat.ai/docs/explorbot' }],
|
|
338
|
+
}
|
|
339
|
+
);
|
|
340
|
+
|
|
341
|
+
printNextSteps(sections);
|
|
342
|
+
|
|
343
|
+
log('');
|
|
344
|
+
log('Add knowledge (logins, credentials) — created on first use in ./knowledge:');
|
|
345
|
+
tag('substep').log(chalk.yellow(`${cli} learn '*' 'to authorize use these credentials: admin@example.com / secret123'`));
|
|
346
|
+
tag('substep').log('You can use ${env.LOGIN} and ${env.PASSWORD} to reference environment variables.');
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function writeEnvKey(key: string, value: string, envPath = globalEnvPath()): void {
|
|
297
350
|
let content = '# AI provider API keys';
|
|
298
351
|
if (existsSync(envPath)) content = readFileSync(envPath, 'utf8').trimEnd();
|
|
299
352
|
|
|
@@ -306,6 +359,19 @@ function writeEnvKey(key: string, value: string): void {
|
|
|
306
359
|
writeFileSync(envPath, `${lines.join('\n').trimEnd()}\n`, 'utf8');
|
|
307
360
|
}
|
|
308
361
|
|
|
362
|
+
function printApiKeySteps(provider: string, envPath: string): void {
|
|
363
|
+
const { envKey, keyUrl } = PROVIDERS[provider];
|
|
364
|
+
tag('warning').log(`No API key yet — explorbot needs one to connect to ${provider}`);
|
|
365
|
+
tag('substep').log(`1. Create a key at ${keyUrl}`);
|
|
366
|
+
tag('substep').log(`2. Paste it into ${envPath}: ${envKey}=<your-key>`);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
type LocalSetup = {
|
|
370
|
+
provider: string;
|
|
371
|
+
apiKey: string;
|
|
372
|
+
url: string;
|
|
373
|
+
};
|
|
374
|
+
|
|
309
375
|
type InitCommandOptions = {
|
|
310
376
|
configPath?: string;
|
|
311
377
|
force?: boolean;
|
|
@@ -313,4 +379,5 @@ type InitCommandOptions = {
|
|
|
313
379
|
global?: boolean;
|
|
314
380
|
provider?: string;
|
|
315
381
|
apiKey?: string;
|
|
382
|
+
url?: string;
|
|
316
383
|
};
|