explorbot 0.5.8 → 0.5.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/explorbot-cli.ts +25 -4
- package/boat/api-tester/src/commands/init-command.ts +2 -0
- package/boat/doc-collector/bin/doc-collector-cli.ts +2 -1
- package/boat/prima/README.md +3 -0
- package/boat/prima/bin/prima-cli.ts +2 -1
- package/boat/prima/src/cli.ts +7 -1
- package/boat/prima/src/envelope.ts +18 -4
- package/boat/prima/src/help.ts +15 -4
- package/boat/prima/src/prima.ts +181 -25
- package/dist/bin/explorbot-cli.js +31 -4
- package/dist/boat/api-tester/bin/apibot-cli.js +2 -1
- package/dist/boat/api-tester/src/commands/init-command.js +2 -0
- package/dist/boat/doc-collector/bin/doc-collector-cli.js +2 -1
- package/dist/boat/prima/bin/prima-cli.js +2 -1
- package/dist/boat/prima/src/cli.js +6 -1
- package/dist/boat/prima/src/envelope.js +16 -3
- package/dist/boat/prima/src/help.js +14 -4
- package/dist/boat/prima/src/prima.js +191 -27
- package/dist/package.json +1 -1
- package/dist/rules/navigator/verification-actions.md +5 -3
- package/dist/rules/planner/ignored-pages.md +6 -0
- package/dist/rules/quartermaster/ux/accessibility.md +8 -0
- package/dist/rules/quartermaster/ux/charts.md +6 -0
- package/dist/rules/quartermaster/ux/copy.md +9 -0
- package/dist/rules/quartermaster/ux/forms.md +7 -0
- package/dist/rules/quartermaster/ux/interaction.md +7 -0
- package/dist/rules/quartermaster/ux/motion.md +5 -0
- package/dist/rules/quartermaster/ux/navigation.md +7 -0
- package/dist/rules/quartermaster/ux/readability.md +6 -0
- package/dist/rules/quartermaster/ux/responsive.md +7 -0
- package/dist/src/ai/judge.d.ts +3 -1
- package/dist/src/ai/judge.js +13 -3
- package/dist/src/ai/navigator.js +3 -2
- package/dist/src/ai/pilot.js +10 -4
- package/dist/src/ai/planner.js +9 -1
- package/dist/src/ai/provider.d.ts +4 -0
- package/dist/src/ai/provider.js +27 -5
- package/dist/src/ai/quartermaster.d.ts +31 -3
- package/dist/src/ai/quartermaster.js +262 -17
- package/dist/src/ai/tester.js +6 -3
- package/dist/src/commands/init-command.js +4 -0
- package/dist/src/commands/options/auth-option.d.ts +6 -0
- package/dist/src/commands/options/auth-option.js +14 -0
- package/dist/src/commands/options/index.d.ts +2 -0
- package/dist/src/commands/options/index.js +2 -0
- package/dist/src/config.js +1 -0
- package/dist/src/explorbot.d.ts +1 -1
- package/dist/src/explorbot.js +8 -13
- package/dist/src/explorer.js +1 -0
- package/dist/src/global-config.js +6 -2
- package/dist/src/utils/ux-probe.d.ts +25 -0
- package/dist/src/utils/ux-probe.js +155 -0
- package/docs/reference/commands.md +12 -0
- package/docs/workflow/agentic-usage.md +7 -0
- package/docs/workflow/knowledge.md +6 -0
- package/package.json +1 -1
- package/rules/navigator/verification-actions.md +5 -3
- package/rules/planner/ignored-pages.md +6 -0
- package/rules/quartermaster/ux/accessibility.md +8 -0
- package/rules/quartermaster/ux/charts.md +6 -0
- package/rules/quartermaster/ux/copy.md +9 -0
- package/rules/quartermaster/ux/forms.md +7 -0
- package/rules/quartermaster/ux/interaction.md +7 -0
- package/rules/quartermaster/ux/motion.md +5 -0
- package/rules/quartermaster/ux/navigation.md +7 -0
- package/rules/quartermaster/ux/readability.md +6 -0
- package/rules/quartermaster/ux/responsive.md +7 -0
- package/src/ai/judge.ts +11 -3
- package/src/ai/navigator.ts +2 -2
- package/src/ai/pilot.ts +10 -4
- package/src/ai/planner.ts +10 -1
- package/src/ai/provider.ts +29 -6
- package/src/ai/quartermaster.ts +300 -18
- package/src/ai/tester.ts +5 -3
- package/src/commands/init-command.ts +5 -0
- package/src/commands/options/auth-option.ts +15 -0
- package/src/commands/options/index.ts +2 -0
- package/src/config.ts +1 -0
- package/src/explorbot.ts +9 -13
- package/src/explorer.ts +1 -0
- package/src/global-config.ts +5 -2
- package/src/utils/ux-probe.ts +163 -0
|
@@ -1,25 +1,41 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import dedent from 'dedent';
|
|
3
4
|
import { z } from 'zod';
|
|
4
5
|
import { outputPath } from "../config.js";
|
|
6
|
+
import { ariaRefSelector, ariaRefSnapshot, parseAriaRefs } from "../utils/aria-ref.js";
|
|
7
|
+
import { compactAriaSnapshot } from "../utils/aria.js";
|
|
5
8
|
import { createDebug, tag } from "../utils/logger.js";
|
|
9
|
+
import { RulesLoader } from "../utils/rules-loader.js";
|
|
6
10
|
import { isCodeceptToolName } from "../utils/step-analyzer.js";
|
|
11
|
+
import { UX_FOCUS_ATTR, countRunningAnimations, measureUx, renderUxMeasurements, uxProbeArgs } from "../utils/ux-probe.js";
|
|
7
12
|
const debugLog = createDebug('explorbot:quartermaster');
|
|
13
|
+
const UX_RULES = ['ux/accessibility', 'ux/interaction', 'ux/responsive', 'ux/navigation', 'ux/copy', 'ux/forms', 'ux/readability', 'ux/motion', 'ux/charts'];
|
|
14
|
+
const UX_BREAKPOINTS = [768, 375];
|
|
15
|
+
const UX_FOCUS_SELECTOR = `[${UX_FOCUS_ATTR}]`;
|
|
16
|
+
const UX_SETTLE_MS = 400;
|
|
17
|
+
const UX_ARIA_CAP = 8000;
|
|
18
|
+
const UX_SCREENSHOT_TIMEOUT = 5000;
|
|
19
|
+
const UX_MAX_FINDINGS = 6;
|
|
20
|
+
const AGENT_NAME = 'quartermaster';
|
|
21
|
+
const FOCUS_CONTAINER_ROLES = ['main', 'region', 'form', 'dialog', 'alertdialog', 'navigation', 'complementary', 'article', 'table', 'grid', 'tabpanel', 'list', 'group', 'banner', 'contentinfo', 'search'];
|
|
8
22
|
export class Quartermaster {
|
|
9
23
|
provider;
|
|
10
24
|
model;
|
|
11
25
|
outputDir;
|
|
12
26
|
pageAnalyses = new Map();
|
|
13
27
|
unsubscribe = null;
|
|
14
|
-
explorer
|
|
28
|
+
explorer;
|
|
15
29
|
pendingAnalyses = [];
|
|
16
|
-
constructor(provider, options) {
|
|
30
|
+
constructor(provider, explorer, options) {
|
|
17
31
|
this.provider = provider;
|
|
32
|
+
this.explorer = explorer;
|
|
18
33
|
this.model = options?.model;
|
|
19
34
|
this.outputDir = outputPath('a11y');
|
|
20
35
|
}
|
|
21
|
-
start(
|
|
22
|
-
this.
|
|
36
|
+
start(stateManager) {
|
|
37
|
+
if (this.unsubscribe)
|
|
38
|
+
return;
|
|
23
39
|
mkdirSync(this.outputDir, { recursive: true });
|
|
24
40
|
this.unsubscribe = stateManager.onStateChange((event) => {
|
|
25
41
|
this.onPageChange(event);
|
|
@@ -45,24 +61,13 @@ export class Quartermaster {
|
|
|
45
61
|
this.pendingAnalyses.push(analysisPromise);
|
|
46
62
|
}
|
|
47
63
|
async runAxeAnalysis(state) {
|
|
48
|
-
const page = this.explorer
|
|
64
|
+
const page = this.explorer.page;
|
|
49
65
|
if (!page || !state.hash) {
|
|
50
66
|
debugLog('No page available for axe analysis');
|
|
51
67
|
return;
|
|
52
68
|
}
|
|
53
69
|
try {
|
|
54
|
-
const
|
|
55
|
-
const results = await new AxeBuilder({ page }).analyze();
|
|
56
|
-
const violations = results.violations.map((v) => ({
|
|
57
|
-
id: v.id,
|
|
58
|
-
impact: v.impact,
|
|
59
|
-
description: v.description,
|
|
60
|
-
helpUrl: v.helpUrl,
|
|
61
|
-
nodes: v.nodes.map((n) => ({
|
|
62
|
-
html: n.html,
|
|
63
|
-
target: n.target,
|
|
64
|
-
})),
|
|
65
|
-
}));
|
|
70
|
+
const violations = await this.axeViolations(page);
|
|
66
71
|
this.pageAnalyses.set(state.hash, {
|
|
67
72
|
url: state.url,
|
|
68
73
|
stateHash: state.hash,
|
|
@@ -110,6 +115,22 @@ export class Quartermaster {
|
|
|
110
115
|
return null;
|
|
111
116
|
}
|
|
112
117
|
}
|
|
118
|
+
async auditUx(result, context) {
|
|
119
|
+
const focus = await this.markUxFocus(result, context);
|
|
120
|
+
tag('step').log(`Auditing UX of ${focus.area}`);
|
|
121
|
+
try {
|
|
122
|
+
const evidence = await this.explorer.withPage((page) => this.collectUxEvidence(page));
|
|
123
|
+
return await this.writeUxReport(result, focus, evidence, context);
|
|
124
|
+
}
|
|
125
|
+
finally {
|
|
126
|
+
await this.explorer
|
|
127
|
+
.withPage((page) => page.evaluate((attr) => {
|
|
128
|
+
for (const el of Array.from(document.querySelectorAll(`[${attr}]`)))
|
|
129
|
+
el.removeAttribute(attr);
|
|
130
|
+
}, UX_FOCUS_ATTR))
|
|
131
|
+
.catch(() => { });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
113
134
|
async generateSemanticAnalysis(axeViolations, executions, initialState) {
|
|
114
135
|
const failedExecs = executions.filter((e) => !e.wasSuccessful);
|
|
115
136
|
if (failedExecs.length === 0 && axeViolations.length === 0)
|
|
@@ -200,4 +221,228 @@ Focus on what would confuse a real user or caused the agent to make mistakes.`;
|
|
|
200
221
|
}
|
|
201
222
|
return content;
|
|
202
223
|
}
|
|
224
|
+
async axeViolations(page, include) {
|
|
225
|
+
const { AxeBuilder } = await import('@axe-core/playwright');
|
|
226
|
+
let builder = new AxeBuilder({ page });
|
|
227
|
+
if (include)
|
|
228
|
+
builder = builder.include(include);
|
|
229
|
+
const results = await builder.analyze();
|
|
230
|
+
return results.violations.map((v) => ({
|
|
231
|
+
id: v.id,
|
|
232
|
+
impact: v.impact,
|
|
233
|
+
description: v.description,
|
|
234
|
+
helpUrl: v.helpUrl,
|
|
235
|
+
nodes: v.nodes.map((n) => ({
|
|
236
|
+
html: n.html,
|
|
237
|
+
target: n.target,
|
|
238
|
+
})),
|
|
239
|
+
}));
|
|
240
|
+
}
|
|
241
|
+
async markUxFocus(result, context) {
|
|
242
|
+
const mark = (locator) => locator
|
|
243
|
+
.first()
|
|
244
|
+
.evaluate((el, attr) => el.setAttribute(attr, ''), UX_FOCUS_ATTR)
|
|
245
|
+
.then(() => true)
|
|
246
|
+
.catch(() => false);
|
|
247
|
+
const region = result.overlay;
|
|
248
|
+
const locateRegion = (page) => {
|
|
249
|
+
if (region.root)
|
|
250
|
+
return page.locator(region.root);
|
|
251
|
+
if (region.xpath)
|
|
252
|
+
return page.locator(`xpath=${region.xpath}`);
|
|
253
|
+
const name = region.name || undefined;
|
|
254
|
+
return page.getByRole('dialog', { name }).or(page.getByRole('alertdialog', { name }));
|
|
255
|
+
};
|
|
256
|
+
const regionFocus = { area: region.name || `open ${region.type}`, reason: `it is the ${region.type} open on the page` };
|
|
257
|
+
if (!context.focus && region.isOpen && (await this.explorer.withPage((page) => mark(locateRegion(page)))))
|
|
258
|
+
return regionFocus;
|
|
259
|
+
if (context.focus || context.commands.length) {
|
|
260
|
+
const picked = await this.pickUxFocus(context, mark);
|
|
261
|
+
if (picked)
|
|
262
|
+
return picked;
|
|
263
|
+
}
|
|
264
|
+
if (region.isOpen && (await this.explorer.withPage((page) => mark(locateRegion(page)))))
|
|
265
|
+
return regionFocus;
|
|
266
|
+
let reason = 'nothing narrower is open or was touched by the session on this page';
|
|
267
|
+
if (context.focus)
|
|
268
|
+
reason = `"${context.focus}" matched no single container`;
|
|
269
|
+
return this.explorer.withPage(async (page) => {
|
|
270
|
+
if (await mark(page.locator('main, [role="main"]')))
|
|
271
|
+
return { area: 'main content', reason };
|
|
272
|
+
await mark(page.locator('body'));
|
|
273
|
+
return { area: 'whole page', reason: `${reason} and the page has no main landmark` };
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
async pickUxFocus(context, mark) {
|
|
277
|
+
const snapshot = await this.explorer.withPage(ariaRefSnapshot).catch(() => '');
|
|
278
|
+
const candidates = parseAriaRefs(snapshot).filter((entry) => FOCUS_CONTAINER_ROLES.includes(entry.role));
|
|
279
|
+
if (!candidates.length)
|
|
280
|
+
return null;
|
|
281
|
+
const hints = [];
|
|
282
|
+
if (context.focus)
|
|
283
|
+
hints.push(`The developer asked to review: ${context.focus}`);
|
|
284
|
+
if (context.commands.length)
|
|
285
|
+
hints.push('Commands the session ran on this page, latest last:', ...context.commands.map((command) => `- ${command}`));
|
|
286
|
+
const schema = z.object({
|
|
287
|
+
ref: z.enum(['none', ...candidates.map((entry) => entry.ref)]).describe('ref of the chosen container, or none'),
|
|
288
|
+
area: z.string().describe('the area named as a user would call it, a few words'),
|
|
289
|
+
});
|
|
290
|
+
const prompt = dedent `
|
|
291
|
+
Pick the page area a developer is working on, so a UX review covers it and nothing else.
|
|
292
|
+
Choose the smallest container that holds everything the hints point at.
|
|
293
|
+
Answer none when no listed container fits the hints.
|
|
294
|
+
|
|
295
|
+
<hints>
|
|
296
|
+
${hints.join('\n')}
|
|
297
|
+
</hints>
|
|
298
|
+
|
|
299
|
+
<containers>
|
|
300
|
+
${candidates.map((entry) => `- [${entry.ref}] ${entry.role} "${entry.name}"`).join('\n')}
|
|
301
|
+
</containers>
|
|
302
|
+
|
|
303
|
+
<page_aria>
|
|
304
|
+
${snapshot.slice(0, UX_ARIA_CAP)}
|
|
305
|
+
</page_aria>
|
|
306
|
+
`;
|
|
307
|
+
const response = await this.provider.generateObject([{ role: 'user', content: prompt }], schema, this.model, { agentName: AGENT_NAME }).catch(() => null);
|
|
308
|
+
const ref = response?.object?.ref;
|
|
309
|
+
if (!ref || ref === 'none')
|
|
310
|
+
return null;
|
|
311
|
+
if (!(await this.explorer.withPage((page) => mark(page.locator(ariaRefSelector(ref))))))
|
|
312
|
+
return null;
|
|
313
|
+
let reason = 'the session commands on this page act inside it';
|
|
314
|
+
if (context.focus)
|
|
315
|
+
reason = `it holds "${context.focus}"`;
|
|
316
|
+
return { area: response.object.area, reason };
|
|
317
|
+
}
|
|
318
|
+
async collectUxEvidence(page) {
|
|
319
|
+
const evidence = { breakpoints: [], axe: [], notes: [] };
|
|
320
|
+
evidence.axe = await this.axeViolations(page, UX_FOCUS_SELECTOR).catch((error) => {
|
|
321
|
+
evidence.notes.push(`axe could not run: ${error.message}`);
|
|
322
|
+
return [];
|
|
323
|
+
});
|
|
324
|
+
const ignoredMotion = await this.animationsIgnoringReducedMotion(page);
|
|
325
|
+
if (ignoredMotion)
|
|
326
|
+
evidence.notes.push(`${ignoredMotion} animations in the area keep running when reduced motion is requested`);
|
|
327
|
+
const width = await page.evaluate(() => window.innerWidth);
|
|
328
|
+
evidence.breakpoints.push(await this.captureUxBreakpoint(page));
|
|
329
|
+
const narrower = UX_BREAKPOINTS.filter((breakpoint) => breakpoint < width);
|
|
330
|
+
if (!narrower.length)
|
|
331
|
+
return evidence;
|
|
332
|
+
const viewport = page.viewportSize();
|
|
333
|
+
if (!viewport) {
|
|
334
|
+
evidence.notes.push('the browser window has no fixed viewport, so narrower widths were not measured');
|
|
335
|
+
return evidence;
|
|
336
|
+
}
|
|
337
|
+
try {
|
|
338
|
+
for (const breakpoint of narrower) {
|
|
339
|
+
await page.setViewportSize({ width: breakpoint, height: viewport.height });
|
|
340
|
+
await page.waitForTimeout(UX_SETTLE_MS);
|
|
341
|
+
if ((await page.evaluate(() => window.innerWidth)) !== breakpoint) {
|
|
342
|
+
evidence.notes.push(`resizing to ${breakpoint}px had no effect, so that width was not measured`);
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
evidence.breakpoints.push(await this.captureUxBreakpoint(page));
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
finally {
|
|
349
|
+
await page.setViewportSize(viewport);
|
|
350
|
+
}
|
|
351
|
+
return evidence;
|
|
352
|
+
}
|
|
353
|
+
async animationsIgnoringReducedMotion(page) {
|
|
354
|
+
if (!(await page.evaluate(countRunningAnimations, UX_FOCUS_SELECTOR)))
|
|
355
|
+
return 0;
|
|
356
|
+
try {
|
|
357
|
+
await page.emulateMedia({ reducedMotion: 'reduce' });
|
|
358
|
+
await page.waitForTimeout(UX_SETTLE_MS);
|
|
359
|
+
return await page.evaluate(countRunningAnimations, UX_FOCUS_SELECTOR);
|
|
360
|
+
}
|
|
361
|
+
finally {
|
|
362
|
+
await page.emulateMedia({ reducedMotion: null });
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
async captureUxBreakpoint(page) {
|
|
366
|
+
const focus = page.locator(UX_FOCUS_SELECTOR).first();
|
|
367
|
+
const measurements = await page.evaluate(measureUx, uxProbeArgs(UX_FOCUS_SELECTOR));
|
|
368
|
+
const aria = await focus.ariaSnapshot().catch(() => '');
|
|
369
|
+
const breakpoint = { measurements, aria: compactAriaSnapshot(aria, true).slice(0, UX_ARIA_CAP) };
|
|
370
|
+
if (!this.provider.hasVision())
|
|
371
|
+
return breakpoint;
|
|
372
|
+
const screenshot = await focus.screenshot({ timeout: UX_SCREENSHOT_TIMEOUT }).catch(() => null);
|
|
373
|
+
if (screenshot)
|
|
374
|
+
breakpoint.screenshot = screenshot.toString('base64');
|
|
375
|
+
return breakpoint;
|
|
376
|
+
}
|
|
377
|
+
async writeUxReport(result, focus, evidence, context) {
|
|
378
|
+
const url = result.url || '';
|
|
379
|
+
const rules = [RulesLoader.loadRules(AGENT_NAME, UX_RULES, url), this.provider.getSystemPromptForAgent(AGENT_NAME, url)].filter(Boolean).join('\n\n');
|
|
380
|
+
const system = dedent `
|
|
381
|
+
<role>
|
|
382
|
+
You review the UX of one area of a web page a developer is building right now.
|
|
383
|
+
The report goes back to that developer, so every finding must point at something they can find in that area.
|
|
384
|
+
</role>
|
|
385
|
+
|
|
386
|
+
<focus>
|
|
387
|
+
Area under review: ${focus.area}, chosen because ${focus.reason}.
|
|
388
|
+
Report only on elements inside it. Page-wide header, navigation and footer are out of scope unless they are the area under review.
|
|
389
|
+
</focus>
|
|
390
|
+
|
|
391
|
+
<scope>
|
|
392
|
+
You see one page of one app, not the app.
|
|
393
|
+
- Report that a text fails and why. Never propose replacement wording, terminology, tone or language: the app's vocabulary is unknown to you.
|
|
394
|
+
- Never recommend colors, fonts, spacing values, design tokens or visual style: the project owns its design system.
|
|
395
|
+
- Sizes, gaps, overflow and contrast come only from <measurements> and <axe>. Never estimate them from a screenshot.
|
|
396
|
+
- Report only what the evidence shows. A check the evidence cannot settle is skipped, not guessed.
|
|
397
|
+
</scope>
|
|
398
|
+
|
|
399
|
+
<rules>
|
|
400
|
+
Each rule set carries the priority its findings are reported under.
|
|
401
|
+
|
|
402
|
+
${rules}
|
|
403
|
+
</rules>
|
|
404
|
+
|
|
405
|
+
<output>
|
|
406
|
+
Markdown, no preamble.
|
|
407
|
+
|
|
408
|
+
## Focus
|
|
409
|
+
One line: the area reviewed and the widths covered.
|
|
410
|
+
|
|
411
|
+
Then one section per priority that has findings, in this order: ## Critical, ## High, ## Medium, ## Low. Omit empty sections.
|
|
412
|
+
Each finding is one bullet of one sentence, at most 30 words:
|
|
413
|
+
- **<element as the user sees it>** (<rule set>, <width or "all widths">): <evidence>, <what it costs the user>.
|
|
414
|
+
|
|
415
|
+
One bullet per cause: elements failing the same check share a bullet naming at most three of them.
|
|
416
|
+
Write at most ${UX_MAX_FINDINGS} bullets in total across all sections, most severe first; drop the rest.
|
|
417
|
+
No fix wording, no rule ids, no list of skipped checks, no summary, no tables.
|
|
418
|
+
When nothing fails, write the Focus section and the line: No UX issues found in this area.
|
|
419
|
+
</output>
|
|
420
|
+
`;
|
|
421
|
+
const sections = [`<page url="${url}" title="${result.title || ''}"></page>`];
|
|
422
|
+
if (context.commands.length)
|
|
423
|
+
sections.push(`<session_commands>\n${context.commands.join('\n')}\n</session_commands>`);
|
|
424
|
+
sections.push(`<axe>\n${evidence.axe.map((v) => `[${v.impact}] ${v.id}: ${v.description} — ${v.nodes[0]?.html.slice(0, 100) || ''}`).join('\n') || 'no violations'}\n</axe>`);
|
|
425
|
+
if (evidence.notes.length)
|
|
426
|
+
sections.push(`<notes>\n${evidence.notes.join('\n')}\n</notes>`);
|
|
427
|
+
for (const breakpoint of evidence.breakpoints) {
|
|
428
|
+
const aria = breakpoint.aria || 'the area is not rendered at this width';
|
|
429
|
+
sections.push(`<breakpoint width="${breakpoint.measurements.width}">\n<measurements>\n${renderUxMeasurements(breakpoint.measurements)}\n</measurements>\n<aria>\n${aria}\n</aria>\n</breakpoint>`);
|
|
430
|
+
}
|
|
431
|
+
const shots = evidence.breakpoints.filter((breakpoint) => breakpoint.screenshot);
|
|
432
|
+
if (!shots.length)
|
|
433
|
+
sections.push('No screenshots are attached: judge from ARIA and measurements only, and say in Focus that visual checks were not possible.');
|
|
434
|
+
const content = [{ type: 'text', text: sections.join('\n\n') }];
|
|
435
|
+
for (const shot of shots) {
|
|
436
|
+
content.push({ type: 'text', text: `Screenshot of the area at ${shot.measurements.width}px:` });
|
|
437
|
+
content.push({ type: 'file', mediaType: 'image/png', data: shot.screenshot });
|
|
438
|
+
}
|
|
439
|
+
let model = this.model || this.provider.getModelForAgent(AGENT_NAME);
|
|
440
|
+
if (shots.length)
|
|
441
|
+
model = this.provider.getVisionModel();
|
|
442
|
+
const response = await this.provider.chat([
|
|
443
|
+
{ role: 'system', content: system },
|
|
444
|
+
{ role: 'user', content },
|
|
445
|
+
], model, { agentName: AGENT_NAME });
|
|
446
|
+
return response.text;
|
|
447
|
+
}
|
|
203
448
|
}
|
package/dist/src/ai/tester.js
CHANGED
|
@@ -322,8 +322,9 @@ export class Tester extends TaskAgent {
|
|
|
322
322
|
return;
|
|
323
323
|
}
|
|
324
324
|
if (actionPerformed && !wasSuccessful) {
|
|
325
|
+
const noted = new Set(Object.values(task.notes).map((note) => note.message));
|
|
325
326
|
result?.toolExecutions
|
|
326
|
-
?.filter((execution) => !execution.wasSuccessful && execution.input?.explanation)
|
|
327
|
+
?.filter((execution) => !execution.wasSuccessful && execution.input?.explanation && !noted.has(execution.input.explanation))
|
|
327
328
|
.forEach((execution) => {
|
|
328
329
|
task.addNote(`Failed to ${execution.input.explanation} (${execution.toolName})`, TestResult.FAILED);
|
|
329
330
|
});
|
|
@@ -333,8 +334,10 @@ export class Tester extends TaskAgent {
|
|
|
333
334
|
return;
|
|
334
335
|
}
|
|
335
336
|
if (assertionPerformed) {
|
|
336
|
-
const
|
|
337
|
-
|
|
337
|
+
const passedVerifications = result?.toolExecutions?.filter((execution) => this.ASSERTION_TOOLS.includes(execution.toolName) && execution.wasSuccessful) || [];
|
|
338
|
+
for (const execution of passedVerifications)
|
|
339
|
+
task.addNote(execution.output?.message || '', TestResult.PASSED);
|
|
340
|
+
const message = passedVerifications.at(-1)?.output?.message || '';
|
|
338
341
|
if (wasSuccessful) {
|
|
339
342
|
conversation.addUserText(dedent `
|
|
340
343
|
Assertion "${message}" successfully passed!
|
|
@@ -288,6 +288,10 @@ function printInitNextSteps(options) {
|
|
|
288
288
|
}
|
|
289
289
|
if (options.siteCommands)
|
|
290
290
|
sections.push({ label: 'Explore any site', commands: options.siteCommands });
|
|
291
|
+
sections.push({
|
|
292
|
+
label: 'Log in for one run (kept in memory, never saved)',
|
|
293
|
+
commands: [{ command: `${cli} explore /dashboard --auth 'login: admin@example.com password: secret123'` }],
|
|
294
|
+
});
|
|
291
295
|
sections.push({
|
|
292
296
|
label: 'Run it from VS Code',
|
|
293
297
|
commands: [{ command: 'https://testomat.ai/explorbot' }],
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { KnowledgeTracker } from '../../knowledge-tracker.js';
|
|
2
|
+
import { registerSecret } from '../../utils/secrets.js';
|
|
3
|
+
import { BaseOption } from './base-option.js';
|
|
4
|
+
export class AuthOption extends BaseOption {
|
|
5
|
+
flags = '--auth <text>';
|
|
6
|
+
description = 'Auth details for this run only, not saved to disk, in free form (e.g. "login: admin password: secret" or "Bearer: <token>"). Falls back to EXPLORBOT_AUTH';
|
|
7
|
+
apply(options) {
|
|
8
|
+
const auth = options.auth || process.env.EXPLORBOT_AUTH;
|
|
9
|
+
if (!auth)
|
|
10
|
+
return;
|
|
11
|
+
registerSecret(auth);
|
|
12
|
+
KnowledgeTracker.appendSessionKnowledge(auth);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { AuthOption } from './auth-option.js';
|
|
1
2
|
import { DecisionModelOption } from './decision-model-option.js';
|
|
2
3
|
import { KnowledgeOption } from './knowledge-option.js';
|
|
3
4
|
import { WsOption } from './ws-option.js';
|
|
4
5
|
export { BaseOption } from './base-option.js';
|
|
6
|
+
export declare const authOption: AuthOption;
|
|
5
7
|
export declare const decisionModelOption: DecisionModelOption;
|
|
6
8
|
export declare const knowledgeOption: KnowledgeOption;
|
|
7
9
|
export declare const wsOption: WsOption;
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { AuthOption } from './auth-option.js';
|
|
1
2
|
import { DecisionModelOption } from './decision-model-option.js';
|
|
2
3
|
import { KnowledgeOption } from './knowledge-option.js';
|
|
3
4
|
import { WsOption } from './ws-option.js';
|
|
4
5
|
export { BaseOption } from './base-option.js';
|
|
6
|
+
export const authOption = new AuthOption();
|
|
5
7
|
export const decisionModelOption = new DecisionModelOption();
|
|
6
8
|
export const knowledgeOption = new KnowledgeOption();
|
|
7
9
|
export const wsOption = new WsOption();
|
package/dist/src/config.js
CHANGED
|
@@ -50,6 +50,7 @@ export const EXPLORBOT_ENV_VARS = [
|
|
|
50
50
|
{ name: 'EXPLORBOT_EPHEMERAL', description: 'Keep no state between runs — output goes to a fresh temp directory instead of the site dir' },
|
|
51
51
|
{ name: 'EXPLORBOT_KNOWLEDGE', description: 'Inline knowledge text, applied to every page' },
|
|
52
52
|
{ name: 'EXPLORBOT_KNOWLEDGE_FILE', description: 'Path to a knowledge markdown file' },
|
|
53
|
+
{ name: 'EXPLORBOT_AUTH', description: 'Auth details in free form (credentials, a token), kept in memory for the run and never written to disk; same as --auth', secret: true },
|
|
53
54
|
{ name: 'EXPLORBOT_SPEC', description: 'Docbot application spec directory or index.md, used as page knowledge' },
|
|
54
55
|
{ name: 'EXPLORBOT_API_SPEC', description: 'OpenAPI spec path for the API boat' },
|
|
55
56
|
{ name: 'EXPLORBOT_API_HEADERS', description: 'Headers sent with every API request, one "Name: value" per line', secret: true },
|
package/dist/src/explorbot.d.ts
CHANGED
|
@@ -97,7 +97,7 @@ export declare class ExplorBot {
|
|
|
97
97
|
agentTester(): Tester;
|
|
98
98
|
agentCaptain(): Captain;
|
|
99
99
|
agentExperienceCompactor(): ExperienceCompactor;
|
|
100
|
-
agentQuartermaster(): Quartermaster
|
|
100
|
+
agentQuartermaster(): Quartermaster;
|
|
101
101
|
agentHistorian(): Historian;
|
|
102
102
|
agentRerunner(): Rerunner;
|
|
103
103
|
agentDriller(): Driller;
|
package/dist/src/explorbot.js
CHANGED
|
@@ -111,6 +111,7 @@ export class ExplorBot {
|
|
|
111
111
|
if (this.options.optionalAi)
|
|
112
112
|
return this.bootstrapOptionalProvider();
|
|
113
113
|
this.provider = new AIProvider(this.config.ai);
|
|
114
|
+
this.provider.setJudge(this.judge());
|
|
114
115
|
await this.provider.validateConnection();
|
|
115
116
|
}
|
|
116
117
|
aiFailureReason() {
|
|
@@ -217,9 +218,11 @@ export class ExplorBot {
|
|
|
217
218
|
const tools = createAgentTools({ ...deps, researcher, navigator, withExperience: false });
|
|
218
219
|
return new Tester(deps, researcher, navigator, tools);
|
|
219
220
|
});
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
this.
|
|
221
|
+
if (this.config.ai?.agents?.quartermaster?.enabled === true) {
|
|
222
|
+
const quartermaster = this.agentQuartermaster();
|
|
223
|
+
quartermaster.start(this.stateManager());
|
|
224
|
+
this.agents.tester.setQuartermaster(quartermaster);
|
|
225
|
+
}
|
|
223
226
|
if (this.isHistorianEnabled())
|
|
224
227
|
this.agents.tester.setHistorian(this.agentHistorian());
|
|
225
228
|
this.agents.tester.setPilot(this.agentPilot());
|
|
@@ -240,16 +243,7 @@ export class ExplorBot {
|
|
|
240
243
|
return (this.agents.experienceCompactor ||= new ExperienceCompactor(this.provider, this.experienceTracker()));
|
|
241
244
|
}
|
|
242
245
|
agentQuartermaster() {
|
|
243
|
-
|
|
244
|
-
if (config?.enabled !== true)
|
|
245
|
-
return null;
|
|
246
|
-
if (!this.agents.quartermaster) {
|
|
247
|
-
this.agents.quartermaster = new Quartermaster(this.provider, {
|
|
248
|
-
model: config?.model,
|
|
249
|
-
});
|
|
250
|
-
this.agents.quartermaster.start(this.explorer, this.stateManager());
|
|
251
|
-
}
|
|
252
|
-
return this.agents.quartermaster;
|
|
246
|
+
return (this.agents.quartermaster ||= new Quartermaster(this.provider, this.explorer, { model: this.config.ai?.agents?.quartermaster?.model }));
|
|
253
247
|
}
|
|
254
248
|
agentHistorian() {
|
|
255
249
|
return (this.agents.historian ||= this.createAgent(({ ai, explorer, config, stateManager, playwrightRecorder }) => {
|
|
@@ -500,6 +494,7 @@ export class ExplorBot {
|
|
|
500
494
|
async bootstrapOptionalProvider() {
|
|
501
495
|
try {
|
|
502
496
|
const provider = new AIProvider(this.config.ai);
|
|
497
|
+
provider.setJudge(this.judge());
|
|
503
498
|
await provider.validateConnection();
|
|
504
499
|
this.provider = provider;
|
|
505
500
|
}
|
package/dist/src/explorer.js
CHANGED
|
@@ -74,6 +74,7 @@ class Explorer {
|
|
|
74
74
|
throw new Error('Playwright helper not available');
|
|
75
75
|
}
|
|
76
76
|
this.applyBasePathToHelper();
|
|
77
|
+
codeceptjs.store.visibleLocator = this.playwrightHelper.options?.visibleLocator;
|
|
77
78
|
await this.connectOrLaunchBrowser();
|
|
78
79
|
const hasSession = !this.options?.attachedBrowser && this.options?.session && existsSync(this.options.session);
|
|
79
80
|
await this.openContextPage();
|
|
@@ -13,6 +13,7 @@ import { pathToFileURL } from 'node:url';
|
|
|
13
13
|
import dedent from 'dedent';
|
|
14
14
|
const GLOBAL_CONFIG_NAMES = ['config.js', 'config.mjs', 'config.ts'];
|
|
15
15
|
const SITE_DIRS = ['knowledge', 'experience', 'output'];
|
|
16
|
+
const LOOPBACK_HOSTS = ['localhost', '127.0.0.1', '[::1]'];
|
|
16
17
|
export const EXPLORBOT_CONFIG_PATHS = ['explorbot.config.js', 'explorbot.config.mjs', 'explorbot.config.ts'];
|
|
17
18
|
export function globalDir() {
|
|
18
19
|
return join(os.homedir(), '.explorbot');
|
|
@@ -33,7 +34,10 @@ export function sitesDir() {
|
|
|
33
34
|
return join(globalDir(), 'sites');
|
|
34
35
|
}
|
|
35
36
|
export function siteFolderName(url) {
|
|
36
|
-
|
|
37
|
+
const parsed = new URL(url);
|
|
38
|
+
if (LOOPBACK_HOSTS.includes(parsed.hostname))
|
|
39
|
+
parsed.hostname = 'localhost';
|
|
40
|
+
return parsed.host.toLowerCase().replace(/[^a-z0-9._-]/g, '_');
|
|
37
41
|
}
|
|
38
42
|
export function listSites() {
|
|
39
43
|
if (!existsSync(sitesDir()))
|
|
@@ -74,7 +78,7 @@ function validateSiteConfig(config, configPath, baseUrl) {
|
|
|
74
78
|
`);
|
|
75
79
|
}
|
|
76
80
|
const declared = URL.parse(url)?.origin;
|
|
77
|
-
if (declared === baseUrl)
|
|
81
|
+
if (declared && siteFolderName(declared) === siteFolderName(baseUrl))
|
|
78
82
|
return;
|
|
79
83
|
throw new Error(dedent `
|
|
80
84
|
Site config declares a different site.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export declare const UX_FOCUS_ATTR = "data-explorbot-ux";
|
|
2
|
+
declare const UX_LIMITS: {
|
|
3
|
+
touchTarget: number;
|
|
4
|
+
pointerTarget: number;
|
|
5
|
+
touchWidth: number;
|
|
6
|
+
targetGap: number;
|
|
7
|
+
minFontSize: number;
|
|
8
|
+
minLineHeight: number;
|
|
9
|
+
maxEntries: number;
|
|
10
|
+
minOpacity: 0.1;
|
|
11
|
+
};
|
|
12
|
+
export declare function uxProbeArgs(scope: string): UxProbeArgs;
|
|
13
|
+
export declare function renderUxMeasurements(measurements: UxMeasurements): string;
|
|
14
|
+
export declare function measureUx({ scope, controlsSelector, limits }: UxProbeArgs): UxMeasurements;
|
|
15
|
+
export declare function countRunningAnimations(scope: string): number;
|
|
16
|
+
interface UxProbeArgs {
|
|
17
|
+
scope: string;
|
|
18
|
+
controlsSelector: string;
|
|
19
|
+
limits: typeof UX_LIMITS;
|
|
20
|
+
}
|
|
21
|
+
export interface UxMeasurements {
|
|
22
|
+
width: number;
|
|
23
|
+
checks: Record<string, string[]>;
|
|
24
|
+
}
|
|
25
|
+
export {};
|