explorbot 0.1.12 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +21 -21
- package/dist/bin/explorbot-cli.js +3 -3
- package/dist/package.json +4 -2
- package/dist/rules/researcher/container-rules.md +2 -0
- package/dist/src/action-result.js +2 -1
- package/dist/src/action.js +3 -8
- package/dist/src/ai/captain.js +0 -2
- package/dist/src/ai/conversation.js +20 -4
- package/dist/src/ai/driller.js +1108 -0
- package/dist/src/ai/historian/utils.js +8 -1
- package/dist/src/ai/pilot.js +214 -267
- package/dist/src/ai/provider.js +25 -12
- package/dist/src/ai/quartermaster.js +2 -2
- package/dist/src/ai/rules.js +5 -5
- package/dist/src/ai/session-analyst.js +122 -0
- package/dist/src/ai/tester.js +69 -22
- package/dist/src/ai/tools.js +19 -4
- package/dist/src/commands/base-command.js +6 -6
- package/dist/src/commands/drill-command.js +3 -2
- package/dist/src/commands/exit-command.js +1 -0
- package/dist/src/commands/explore-command.js +9 -2
- package/dist/src/components/AddRule.js +1 -1
- package/dist/src/components/StatusPane.js +6 -1
- package/dist/src/experience-tracker.js +9 -0
- package/dist/src/explorbot.js +48 -8
- package/dist/src/explorer.js +11 -13
- package/dist/src/reporter.js +105 -4
- package/dist/src/state-manager.js +4 -3
- package/dist/src/stats.js +7 -1
- package/dist/src/test-plan.js +47 -3
- package/dist/src/utils/aria.js +354 -529
- package/dist/src/utils/hooks-runner.js +2 -8
- package/dist/src/utils/html.js +371 -0
- package/dist/src/utils/unique-names.js +12 -1
- package/dist/src/utils/url-matcher.js +6 -1
- package/dist/src/utils/web-element.js +27 -24
- package/dist/src/utils/xpath.js +1 -1
- package/package.json +4 -2
- package/rules/researcher/container-rules.md +2 -0
- package/src/action-result.ts +2 -1
- package/src/action.ts +3 -10
- package/src/ai/captain.ts +0 -2
- package/src/ai/conversation.ts +21 -4
- package/src/ai/driller.ts +1194 -0
- package/src/ai/historian/utils.ts +8 -1
- package/src/ai/pilot.ts +215 -265
- package/src/ai/provider.ts +24 -12
- package/src/ai/quartermaster.ts +2 -2
- package/src/ai/rules.ts +5 -5
- package/src/ai/session-analyst.ts +139 -0
- package/src/ai/tester.ts +63 -20
- package/src/ai/tools.ts +18 -4
- package/src/commands/base-command.ts +6 -6
- package/src/commands/drill-command.ts +3 -2
- package/src/commands/exit-command.ts +1 -0
- package/src/commands/explore-command.ts +10 -2
- package/src/components/AddRule.tsx +1 -1
- package/src/components/StatusPane.tsx +6 -3
- package/src/config.ts +4 -0
- package/src/experience-tracker.ts +9 -0
- package/src/explorbot.ts +55 -10
- package/src/explorer.ts +10 -12
- package/src/reporter.ts +108 -4
- package/src/state-manager.ts +4 -3
- package/src/stats.ts +10 -1
- package/src/test-plan.ts +62 -3
- package/src/utils/aria.ts +367 -537
- package/src/utils/hooks-runner.ts +2 -6
- package/src/utils/html.ts +381 -0
- package/src/utils/unique-names.ts +13 -0
- package/src/utils/url-matcher.ts +5 -1
- package/src/utils/web-element.ts +31 -28
- package/src/utils/xpath.ts +1 -1
- package/dist/src/ai/bosun.js +0 -456
- package/src/ai/bosun.ts +0 -571
|
@@ -52,9 +52,12 @@ export const StatusPane: React.FC<{ onComplete?: () => void }> = ({ onComplete }
|
|
|
52
52
|
<Text bold>Usage</Text>
|
|
53
53
|
</Box>
|
|
54
54
|
<Row label="Time" value={Stats.getElapsedTime()} />
|
|
55
|
-
{tokenRows.map(([model, tokens]) =>
|
|
56
|
-
|
|
57
|
-
|
|
55
|
+
{tokenRows.map(([model, tokens]) => {
|
|
56
|
+
const cached = tokens.cached ?? 0;
|
|
57
|
+
const cachePct = tokens.input > 0 ? Math.round((cached / tokens.input) * 100) : 0;
|
|
58
|
+
const suffix = cached > 0 ? ` (${Stats.humanizeTokens(cached)} cached, ${cachePct}%)` : '';
|
|
59
|
+
return <Row key={model} label={model} value={`${Stats.humanizeTokens(tokens.total)} tokens${suffix}`} />;
|
|
60
|
+
})}
|
|
58
61
|
</>
|
|
59
62
|
)}
|
|
60
63
|
</Box>
|
package/src/config.ts
CHANGED
|
@@ -123,6 +123,7 @@ interface AgentsConfig {
|
|
|
123
123
|
researcher?: ResearcherAgentConfig;
|
|
124
124
|
planner?: PlannerAgentConfig;
|
|
125
125
|
pilot?: PilotAgentConfig;
|
|
126
|
+
driller?: AgentConfig;
|
|
126
127
|
'experience-compactor'?: AgentConfig;
|
|
127
128
|
captain?: AgentConfig;
|
|
128
129
|
quartermaster?: AgentConfig;
|
|
@@ -131,6 +132,7 @@ interface AgentsConfig {
|
|
|
131
132
|
chief?: AgentConfig;
|
|
132
133
|
curler?: AgentConfig;
|
|
133
134
|
rerunner?: RerunnerAgentConfig;
|
|
135
|
+
analyst?: AgentConfig;
|
|
134
136
|
}
|
|
135
137
|
|
|
136
138
|
interface AIConfig {
|
|
@@ -179,6 +181,8 @@ interface ActionConfig {
|
|
|
179
181
|
interface ReporterConfig {
|
|
180
182
|
enabled?: boolean;
|
|
181
183
|
html?: boolean;
|
|
184
|
+
markdown?: boolean;
|
|
185
|
+
runGroup?: string | null;
|
|
182
186
|
}
|
|
183
187
|
|
|
184
188
|
type ApiHookFn = (ctx: { headers: Record<string, string>; baseEndpoint: string }) => Promise<Record<string, string> | undefined> | Record<string, string> | undefined;
|
|
@@ -3,6 +3,7 @@ import { basename, dirname, join } from 'node:path';
|
|
|
3
3
|
import matter from 'gray-matter';
|
|
4
4
|
import { type Tokens, marked } from 'marked';
|
|
5
5
|
import type { ActionResult } from './action-result.js';
|
|
6
|
+
import { isNonReusableCode } from './ai/historian/utils.ts';
|
|
6
7
|
import { ConfigParser } from './config.js';
|
|
7
8
|
import { KnowledgeTracker } from './knowledge-tracker.js';
|
|
8
9
|
import type { WebPageState } from './state-manager.js';
|
|
@@ -166,6 +167,10 @@ export class ExperienceTracker {
|
|
|
166
167
|
writeAction(state: ActionResult, action: ActionInput): void {
|
|
167
168
|
if (this.disabled || this.isWritingDisabled(state)) return;
|
|
168
169
|
if (!action.code?.trim()) return;
|
|
170
|
+
if (isNonReusableCode(action.code)) {
|
|
171
|
+
debugLog('Skipping action with non-reusable code: %s', action.code);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
169
174
|
|
|
170
175
|
this.ensureExperienceFile(state);
|
|
171
176
|
const stateHash = state.getStateHash();
|
|
@@ -189,6 +194,10 @@ export class ExperienceTracker {
|
|
|
189
194
|
writeFlow(state: ActionResult, body: string, relatedUrls?: string[]): void {
|
|
190
195
|
if (this.disabled || this.isWritingDisabled(state)) return;
|
|
191
196
|
if (!body?.trim()) return;
|
|
197
|
+
if (isNonReusableCode(body)) {
|
|
198
|
+
debugLog('Skipping flow body with non-reusable code');
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
192
201
|
|
|
193
202
|
this.ensureExperienceFile(state);
|
|
194
203
|
const stateHash = state.getStateHash();
|
package/src/explorbot.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { existsSync, mkdirSync } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { ActionResult } from './action-result.ts';
|
|
4
|
-
import { Bosun } from './ai/bosun.ts';
|
|
5
4
|
import { Captain } from './ai/captain.ts';
|
|
5
|
+
import { Driller } from './ai/driller.ts';
|
|
6
6
|
import { ExperienceCompactor } from './ai/experience-compactor.ts';
|
|
7
7
|
import { Fisherman } from './ai/fisherman.ts';
|
|
8
8
|
import { Historian } from './ai/historian.ts';
|
|
@@ -13,6 +13,7 @@ import { AIProvider } from './ai/provider.ts';
|
|
|
13
13
|
import { Quartermaster } from './ai/quartermaster.ts';
|
|
14
14
|
import { Rerunner } from './ai/rerunner.ts';
|
|
15
15
|
import { Researcher } from './ai/researcher.ts';
|
|
16
|
+
import { SessionAnalyst } from './ai/session-analyst.ts';
|
|
16
17
|
import { Tester } from './ai/tester.ts';
|
|
17
18
|
import { createAgentTools } from './ai/tools.ts';
|
|
18
19
|
import { ApiClient } from './api/api-client.ts';
|
|
@@ -25,8 +26,9 @@ import Explorer from './explorer.ts';
|
|
|
25
26
|
import { KnowledgeTracker } from './knowledge-tracker.ts';
|
|
26
27
|
import { WebPageState } from './state-manager.ts';
|
|
27
28
|
import type { Suite } from './suite.ts';
|
|
28
|
-
import { Plan } from './test-plan.ts';
|
|
29
|
+
import { Plan, type Test } from './test-plan.ts';
|
|
29
30
|
import { setVerboseMode, tag } from './utils/logger.ts';
|
|
31
|
+
import { relativeToCwd } from './utils/next-steps.ts';
|
|
30
32
|
import { sanitizeFilename } from './utils/strings.ts';
|
|
31
33
|
|
|
32
34
|
export interface ExplorBotOptions {
|
|
@@ -55,6 +57,8 @@ export class ExplorBot {
|
|
|
55
57
|
lastPlanError: Error | null = null;
|
|
56
58
|
lastSavedPlanPath: string | null = null;
|
|
57
59
|
private agents: Record<string, any> = {};
|
|
60
|
+
private sessionPlans: Plan[] = [];
|
|
61
|
+
private lastReportedTestCount = 0;
|
|
58
62
|
|
|
59
63
|
constructor(options: ExplorBotOptions = {}) {
|
|
60
64
|
this.options = options;
|
|
@@ -284,15 +288,17 @@ export class ExplorBot {
|
|
|
284
288
|
return this.agents.rerunner;
|
|
285
289
|
}
|
|
286
290
|
|
|
287
|
-
|
|
288
|
-
return (this.agents.
|
|
289
|
-
const researcher = this.agentResearcher();
|
|
291
|
+
agentDriller(): Driller {
|
|
292
|
+
return (this.agents.driller ||= this.createAgent(({ ai, explorer }) => {
|
|
290
293
|
const navigator = this.agentNavigator();
|
|
291
|
-
|
|
292
|
-
return new Bosun(explorer, ai, researcher, navigator, tools);
|
|
294
|
+
return new Driller(explorer, ai, navigator);
|
|
293
295
|
}));
|
|
294
296
|
}
|
|
295
297
|
|
|
298
|
+
agentSessionAnalyst(): SessionAnalyst {
|
|
299
|
+
return (this.agents.sessionAnalyst ||= this.createAgent(({ ai }) => new SessionAnalyst(ai)));
|
|
300
|
+
}
|
|
301
|
+
|
|
296
302
|
agentFisherman(): Fisherman | null {
|
|
297
303
|
const fishermanConfig = this.config.ai?.agents?.fisherman;
|
|
298
304
|
const hasApiConfig = !!this.config.api;
|
|
@@ -365,7 +371,7 @@ export class ExplorBot {
|
|
|
365
371
|
}
|
|
366
372
|
this.lastPlanError = null;
|
|
367
373
|
try {
|
|
368
|
-
this.
|
|
374
|
+
this.setCurrentPlan(await planner.plan(feature, opts.style, opts.extend, opts.completedPlans));
|
|
369
375
|
} catch (err) {
|
|
370
376
|
this.lastPlanError = err instanceof Error ? err : new Error(String(err));
|
|
371
377
|
tag('warning').log(`Planning failed: ${this.lastPlanError.message}`);
|
|
@@ -436,11 +442,50 @@ export class ExplorBot {
|
|
|
436
442
|
throw new Error(`Plan file not found: ${planPath}`);
|
|
437
443
|
}
|
|
438
444
|
|
|
439
|
-
this.
|
|
440
|
-
return this.currentPlan
|
|
445
|
+
this.setCurrentPlan(Plan.fromMarkdown(planPath));
|
|
446
|
+
return this.currentPlan!;
|
|
441
447
|
}
|
|
442
448
|
|
|
443
449
|
setCurrentPlan(plan?: Plan): void {
|
|
444
450
|
this.currentPlan = plan;
|
|
451
|
+
if (plan && !this.sessionPlans.includes(plan)) {
|
|
452
|
+
this.sessionPlans.push(plan);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
getSessionTests(): Test[] {
|
|
457
|
+
return this.sessionPlans.flatMap((p) => p.tests.filter((t) => t.startTime != null));
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
async printSessionAnalysis(): Promise<void> {
|
|
461
|
+
const analystConfig = this.config.ai?.agents?.analyst;
|
|
462
|
+
if (analystConfig?.enabled === false) return;
|
|
463
|
+
|
|
464
|
+
const tests = this.getSessionTests();
|
|
465
|
+
if (tests.length === 0) return;
|
|
466
|
+
if (tests.length === this.lastReportedTestCount) return;
|
|
467
|
+
|
|
468
|
+
try {
|
|
469
|
+
const markdown = await this.agentSessionAnalyst().analyze(tests);
|
|
470
|
+
if (!markdown) {
|
|
471
|
+
this.lastReportedTestCount = tests.length;
|
|
472
|
+
return;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
tag('multiline').log(markdown);
|
|
476
|
+
|
|
477
|
+
const filePath = this.agentSessionAnalyst().writeReport(markdown);
|
|
478
|
+
tag('info').log(`Session report saved: ${relativeToCwd(filePath)}`);
|
|
479
|
+
|
|
480
|
+
const reporter = this.explorer?.getReporter();
|
|
481
|
+
if (reporter?.isEnabled()) {
|
|
482
|
+
await reporter.setRunDescription(markdown);
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
this.lastReportedTestCount = tests.length;
|
|
486
|
+
} catch (error) {
|
|
487
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
488
|
+
tag('warning').log(`Session analysis failed: ${message}`);
|
|
489
|
+
}
|
|
445
490
|
}
|
|
446
491
|
}
|
package/src/explorer.ts
CHANGED
|
@@ -19,8 +19,9 @@ import { PlaywrightRecorder } from './playwright-recorder.ts';
|
|
|
19
19
|
import { Reporter } from './reporter.ts';
|
|
20
20
|
import { StateManager } from './state-manager.js';
|
|
21
21
|
import { Test } from './test-plan.ts';
|
|
22
|
+
import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './utils/html.ts';
|
|
22
23
|
import { createDebug, log, tag } from './utils/logger.js';
|
|
23
|
-
import { WebElement
|
|
24
|
+
import { WebElement } from './utils/web-element.ts';
|
|
24
25
|
|
|
25
26
|
declare global {
|
|
26
27
|
namespace NodeJS {
|
|
@@ -337,11 +338,11 @@ class Explorer {
|
|
|
337
338
|
async getEidxInContainer(containerCss: string | null): Promise<string[]> {
|
|
338
339
|
const page = this.playwrightHelper.page;
|
|
339
340
|
try {
|
|
340
|
-
const selector = containerCss ? `${containerCss} [
|
|
341
|
+
const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`;
|
|
341
342
|
const elements = await page.locator(selector).all();
|
|
342
343
|
const result: string[] = [];
|
|
343
344
|
for (const el of elements) {
|
|
344
|
-
const attr = await el.getAttribute(
|
|
345
|
+
const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx);
|
|
345
346
|
if (attr) result.push(attr);
|
|
346
347
|
}
|
|
347
348
|
return result;
|
|
@@ -359,7 +360,7 @@ class Explorer {
|
|
|
359
360
|
const page = this.playwrightHelper.page;
|
|
360
361
|
const base = container ? page.locator(container) : page;
|
|
361
362
|
const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator);
|
|
362
|
-
return await el.first().getAttribute(
|
|
363
|
+
return await el.first().getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx);
|
|
363
364
|
} catch (error) {
|
|
364
365
|
if (this.isFatalBrowserError(error)) {
|
|
365
366
|
tag('warning').log(`getEidxByLocator: ${error instanceof Error ? error.message : error}`);
|
|
@@ -548,10 +549,7 @@ class Explorer {
|
|
|
548
549
|
if (!this.stateManager.getCurrentState()) return;
|
|
549
550
|
|
|
550
551
|
const lastScreenshot = ActionResult.fromState(this.stateManager.getCurrentState()!).screenshotFile;
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
const screenshotPath = outputPath('states', lastScreenshot);
|
|
554
|
-
test.addArtifact(screenshotPath);
|
|
552
|
+
test.setActiveNoteScreenshot(lastScreenshot);
|
|
555
553
|
};
|
|
556
554
|
|
|
557
555
|
const dialogHandler = (dialog: any) => {
|
|
@@ -751,20 +749,20 @@ export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: s
|
|
|
751
749
|
for (const [role, entries] of byRole) {
|
|
752
750
|
try {
|
|
753
751
|
const rawList = await page.getByRole(role).evaluateAll(
|
|
754
|
-
(domElements: Element[], [data, extractFnStr]: [Array<{ name: string; ref: string }>, string]) => {
|
|
752
|
+
(domElements: Element[], [data, extractFnStr, config]: [Array<{ name: string; ref: string }>, string, typeof ELEMENT_EXTRACTION_CONFIG]) => {
|
|
755
753
|
const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any;
|
|
756
754
|
const results: any[] = [];
|
|
757
755
|
let ariaIdx = 0;
|
|
758
756
|
for (const el of domElements) {
|
|
759
757
|
if (ariaIdx >= data.length) break;
|
|
760
|
-
el.setAttribute(
|
|
761
|
-
const elData = extract(el);
|
|
758
|
+
el.setAttribute(config.attrs.eidx, data[ariaIdx].ref);
|
|
759
|
+
const elData = extract(el, config);
|
|
762
760
|
if (elData) results.push(elData);
|
|
763
761
|
ariaIdx++;
|
|
764
762
|
}
|
|
765
763
|
return results;
|
|
766
764
|
},
|
|
767
|
-
[entries,
|
|
765
|
+
[entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]
|
|
768
766
|
);
|
|
769
767
|
for (const raw of rawList) {
|
|
770
768
|
elements.push(WebElement.fromRawData(raw, role));
|
package/src/reporter.ts
CHANGED
|
@@ -33,8 +33,21 @@ export class Reporter {
|
|
|
33
33
|
this.configureHtmlPipe();
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
if (this.reporterEnabled && config?.markdown) {
|
|
37
|
+
this.configureMarkdownPipe();
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (this.reporterEnabled) {
|
|
41
|
+
this.configureRunGroup(config?.runGroup);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
debugLog('Reporter initialized', {
|
|
45
|
+
enabled: this.reporterEnabled,
|
|
46
|
+
testomatio: Boolean(process.env.TESTOMATIO),
|
|
47
|
+
html: Boolean(process.env.TESTOMATIO_HTML_REPORT_SAVE),
|
|
48
|
+
markdown: Boolean(process.env.TESTOMATIO_MARKDOWN_REPORT_SAVE),
|
|
49
|
+
runGroup: process.env.TESTOMATIO_RUNGROUP_TITLE || null,
|
|
50
|
+
});
|
|
38
51
|
}
|
|
39
52
|
|
|
40
53
|
private buildTitle(): string {
|
|
@@ -56,7 +69,31 @@ export class Reporter {
|
|
|
56
69
|
private configureHtmlPipe(): void {
|
|
57
70
|
process.env.TESTOMATIO_HTML_REPORT_SAVE = '1';
|
|
58
71
|
process.env.TESTOMATIO_HTML_REPORT_FOLDER = outputPath('reports');
|
|
59
|
-
|
|
72
|
+
process.env.TESTOMATIO_HTML_FILENAME = `${Stats.sessionLabel()}.html`;
|
|
73
|
+
debugLog('HTML report pipe configured', {
|
|
74
|
+
folder: process.env.TESTOMATIO_HTML_REPORT_FOLDER,
|
|
75
|
+
filename: process.env.TESTOMATIO_HTML_FILENAME,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
private configureMarkdownPipe(): void {
|
|
80
|
+
process.env.TESTOMATIO_MARKDOWN_REPORT_SAVE = '1';
|
|
81
|
+
process.env.TESTOMATIO_MARKDOWN_REPORT_FOLDER = outputPath('reports');
|
|
82
|
+
process.env.TESTOMATIO_MARKDOWN_FILENAME = `${Stats.sessionLabel()}-tests.md`;
|
|
83
|
+
debugLog('Markdown report pipe configured', {
|
|
84
|
+
folder: process.env.TESTOMATIO_MARKDOWN_REPORT_FOLDER,
|
|
85
|
+
filename: process.env.TESTOMATIO_MARKDOWN_FILENAME,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
private configureRunGroup(runGroup: string | null | undefined): void {
|
|
90
|
+
if (process.env.TESTOMATIO_RUNGROUP_TITLE) return;
|
|
91
|
+
if (runGroup === null) return;
|
|
92
|
+
if (runGroup) {
|
|
93
|
+
process.env.TESTOMATIO_RUNGROUP_TITLE = runGroup;
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
process.env.TESTOMATIO_RUNGROUP_TITLE = `Explorbot ${new Date().toISOString().slice(0, 10)}`;
|
|
60
97
|
}
|
|
61
98
|
|
|
62
99
|
async startRun(): Promise<void> {
|
|
@@ -73,7 +110,7 @@ export class Reporter {
|
|
|
73
110
|
const timeoutMs = Number(process.env.TESTOMATIO_TIMEOUT_MS || '15000');
|
|
74
111
|
const timeoutPromise = new Promise<'timeout'>((resolve) => setTimeout(() => resolve('timeout'), timeoutMs));
|
|
75
112
|
|
|
76
|
-
const result = await Promise.race([this.client.createRun().then(() => 'success' as const), timeoutPromise]);
|
|
113
|
+
const result = await Promise.race([this.client.createRun({ configuration: { exploratory: true } }).then(() => 'success' as const), timeoutPromise]);
|
|
77
114
|
|
|
78
115
|
if (result === 'timeout') {
|
|
79
116
|
debugLog('Reporter run creation timed out');
|
|
@@ -108,6 +145,7 @@ export class Reporter {
|
|
|
108
145
|
message: note.message,
|
|
109
146
|
status: note.status,
|
|
110
147
|
screenshot: note.screenshot,
|
|
148
|
+
log: note.log,
|
|
111
149
|
}))
|
|
112
150
|
.sort((a, b) => a.startTime - b.startTime);
|
|
113
151
|
|
|
@@ -143,9 +181,18 @@ export class Reporter {
|
|
|
143
181
|
if (noteEntry.screenshot) {
|
|
144
182
|
step.artifacts = [outputPath('states', noteEntry.screenshot)];
|
|
145
183
|
}
|
|
184
|
+
if (noteEntry.log) {
|
|
185
|
+
step.log = noteEntry.log;
|
|
186
|
+
}
|
|
146
187
|
steps.push(step);
|
|
147
188
|
}
|
|
148
189
|
|
|
190
|
+
const verificationStep = this.buildVerificationStep(test, lastScreenshotFile);
|
|
191
|
+
if (verificationStep) {
|
|
192
|
+
steps.push(verificationStep);
|
|
193
|
+
return steps;
|
|
194
|
+
}
|
|
195
|
+
|
|
149
196
|
if (lastScreenshotFile && steps.length > 0) {
|
|
150
197
|
const lastStep = steps[steps.length - 1];
|
|
151
198
|
const screenshotPath = outputPath('states', lastScreenshotFile);
|
|
@@ -159,6 +206,39 @@ export class Reporter {
|
|
|
159
206
|
return steps;
|
|
160
207
|
}
|
|
161
208
|
|
|
209
|
+
private buildVerificationStep(test: Test, lastScreenshotFile?: string): Step | undefined {
|
|
210
|
+
const v = test.verification;
|
|
211
|
+
if (!v) return undefined;
|
|
212
|
+
|
|
213
|
+
const subSteps: Step[] = [];
|
|
214
|
+
if (v.message) subSteps.push({ category: 'framework', title: v.message, duration: 0 });
|
|
215
|
+
if (v.url) {
|
|
216
|
+
subSteps.push({
|
|
217
|
+
category: 'framework',
|
|
218
|
+
title: v.pageLabel ? `Navigated to ${v.pageLabel}` : 'Final page',
|
|
219
|
+
log: v.url,
|
|
220
|
+
duration: 0,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
for (const detail of v.details) {
|
|
224
|
+
subSteps.push({ category: 'framework', title: detail, duration: 0 });
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const screenshotFile = v.screenshot || lastScreenshotFile;
|
|
228
|
+
|
|
229
|
+
const step: Step = {
|
|
230
|
+
category: 'user',
|
|
231
|
+
title: 'Verification',
|
|
232
|
+
duration: 0,
|
|
233
|
+
status: v.status || 'none',
|
|
234
|
+
steps: subSteps.length > 0 ? subSteps : undefined,
|
|
235
|
+
};
|
|
236
|
+
if (screenshotFile) {
|
|
237
|
+
step.artifacts = [outputPath('states', screenshotFile)];
|
|
238
|
+
}
|
|
239
|
+
return step;
|
|
240
|
+
}
|
|
241
|
+
|
|
162
242
|
async reportTest(test: Test, meta?: ReporterMeta): Promise<void> {
|
|
163
243
|
await this.startRun();
|
|
164
244
|
|
|
@@ -229,6 +309,30 @@ export class Reporter {
|
|
|
229
309
|
return this.isRunStarted;
|
|
230
310
|
}
|
|
231
311
|
|
|
312
|
+
async setRunDescription(text: string): Promise<void> {
|
|
313
|
+
if (!this.isRunStarted) return;
|
|
314
|
+
if (!process.env.TESTOMATIO) return;
|
|
315
|
+
const runId = this.client.runId;
|
|
316
|
+
if (!runId) return;
|
|
317
|
+
|
|
318
|
+
const baseUrl = process.env.TESTOMATIO_URL || 'https://app.testomat.io';
|
|
319
|
+
const url = `${baseUrl}/api/reporter/${runId}`;
|
|
320
|
+
try {
|
|
321
|
+
const response = await fetch(url, {
|
|
322
|
+
method: 'PUT',
|
|
323
|
+
headers: { 'Content-Type': 'application/json' },
|
|
324
|
+
body: JSON.stringify({ api_key: process.env.TESTOMATIO, description: text }),
|
|
325
|
+
});
|
|
326
|
+
if (!response.ok) {
|
|
327
|
+
debugLog('Run description update failed:', response.status, response.statusText);
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
debugLog('Run description updated');
|
|
331
|
+
} catch (error) {
|
|
332
|
+
debugLog('Failed to update run description:', error);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
232
336
|
private extractLastNoteMessage(test: Test): string {
|
|
233
337
|
const notes = Object.values(test.notes);
|
|
234
338
|
if (notes.length === 0) return '';
|
package/src/state-manager.ts
CHANGED
|
@@ -142,8 +142,8 @@ export class StateManager {
|
|
|
142
142
|
|
|
143
143
|
/**
|
|
144
144
|
* Extract state path from full URL
|
|
145
|
-
* Removes domain, port, protocol
|
|
146
|
-
* Keeps path and hash: /path/to/page#section
|
|
145
|
+
* Removes domain, port, protocol
|
|
146
|
+
* Keeps path, query, and hash: /path/to/page?tab=users#section
|
|
147
147
|
*/
|
|
148
148
|
/**
|
|
149
149
|
* Update current state from ActionResult and record transition if state changed
|
|
@@ -549,7 +549,8 @@ export class StateManager {
|
|
|
549
549
|
export function normalizeUrl(url: string): string {
|
|
550
550
|
try {
|
|
551
551
|
const parsed = new URL(url, 'http://localhost');
|
|
552
|
-
|
|
552
|
+
const path = parsed.pathname.replace(/^\/+|\/+$/g, '');
|
|
553
|
+
return `${path}${parsed.search}${parsed.hash}`;
|
|
553
554
|
} catch {
|
|
554
555
|
return url.replace(/^\/+|\/+$/g, '');
|
|
555
556
|
}
|
package/src/stats.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
|
+
import { uniqExplorationName } from './utils/unique-names.ts';
|
|
2
|
+
|
|
1
3
|
interface TokenUsage {
|
|
2
4
|
input: number;
|
|
3
5
|
output: number;
|
|
4
6
|
total: number;
|
|
7
|
+
cached?: number;
|
|
5
8
|
}
|
|
6
9
|
|
|
7
10
|
export type ExplorbotMode = 'explore' | 'test' | 'freesail' | 'tui';
|
|
8
11
|
|
|
9
12
|
export class Stats {
|
|
10
13
|
static startTime = Date.now();
|
|
14
|
+
static sessionName = uniqExplorationName();
|
|
11
15
|
static researches = 0;
|
|
12
16
|
static tests = 0;
|
|
13
17
|
static plans = 0;
|
|
@@ -17,11 +21,12 @@ export class Stats {
|
|
|
17
21
|
|
|
18
22
|
static recordTokens(_agent: string, model: string, usage: TokenUsage): void {
|
|
19
23
|
if (!Stats.models[model]) {
|
|
20
|
-
Stats.models[model] = { input: 0, output: 0, total: 0 };
|
|
24
|
+
Stats.models[model] = { input: 0, output: 0, total: 0, cached: 0 };
|
|
21
25
|
}
|
|
22
26
|
Stats.models[model].input += usage.input;
|
|
23
27
|
Stats.models[model].output += usage.output;
|
|
24
28
|
Stats.models[model].total += usage.total;
|
|
29
|
+
Stats.models[model].cached = (Stats.models[model].cached ?? 0) + (usage.cached ?? 0);
|
|
25
30
|
}
|
|
26
31
|
|
|
27
32
|
static getElapsedTime(): string {
|
|
@@ -54,4 +59,8 @@ export class Stats {
|
|
|
54
59
|
const totalTokens = Object.values(Stats.models).reduce((sum, m) => sum + m.total, 0);
|
|
55
60
|
return totalTokens > 0;
|
|
56
61
|
}
|
|
62
|
+
|
|
63
|
+
static sessionLabel(): string {
|
|
64
|
+
return `${Stats.mode || 'session'}-${Stats.sessionName}`;
|
|
65
|
+
}
|
|
57
66
|
}
|
package/src/test-plan.ts
CHANGED
|
@@ -26,6 +26,7 @@ export interface Note {
|
|
|
26
26
|
startTime: number;
|
|
27
27
|
endTime: number;
|
|
28
28
|
screenshot?: string;
|
|
29
|
+
log?: string;
|
|
29
30
|
}
|
|
30
31
|
|
|
31
32
|
export class ActiveNote {
|
|
@@ -34,6 +35,7 @@ export class ActiveNote {
|
|
|
34
35
|
message: string;
|
|
35
36
|
status?: TestResultType;
|
|
36
37
|
screenshot?: string;
|
|
38
|
+
log?: string;
|
|
37
39
|
|
|
38
40
|
constructor(task: Task, message: string, status?: TestResultType) {
|
|
39
41
|
this.task = task;
|
|
@@ -73,6 +75,7 @@ export class Task {
|
|
|
73
75
|
steps: Record<string, StepData>;
|
|
74
76
|
states: WebPageState[];
|
|
75
77
|
startUrl: string;
|
|
78
|
+
verification?: Verification;
|
|
76
79
|
protected timestampCounter = 0;
|
|
77
80
|
private activeNote?: ActiveNote;
|
|
78
81
|
|
|
@@ -102,6 +105,7 @@ export class Task {
|
|
|
102
105
|
startTime: activeNote.getStartTime(),
|
|
103
106
|
endTime,
|
|
104
107
|
screenshot: activeNote.screenshot,
|
|
108
|
+
log: activeNote.log,
|
|
105
109
|
};
|
|
106
110
|
this.activeNote = undefined;
|
|
107
111
|
}
|
|
@@ -118,13 +122,28 @@ export class Task {
|
|
|
118
122
|
.join('\n');
|
|
119
123
|
}
|
|
120
124
|
|
|
121
|
-
addNote(message: string, status: TestResultType = null, screenshot?: string): void {
|
|
122
|
-
const isDuplicate = Object.values(this.notes).some((note) => note.message === message && note.status === status);
|
|
125
|
+
addNote(message: string, status: TestResultType = null, screenshot?: string, log?: string): void {
|
|
126
|
+
const isDuplicate = Object.values(this.notes).some((note) => note.message === message && note.status === status && note.log === log);
|
|
123
127
|
if (isDuplicate) return;
|
|
124
128
|
|
|
125
129
|
const now = performance.now();
|
|
126
130
|
const timestamp = `${now}_${this.timestampCounter++}`;
|
|
127
|
-
this.notes[timestamp] = { message, status, startTime: now, endTime: now, screenshot };
|
|
131
|
+
this.notes[timestamp] = { message, status, startTime: now, endTime: now, screenshot, log };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
addUrlNote(state: UrlNoteState, prevState?: { title?: string; h1?: string; h2?: string }): void {
|
|
135
|
+
const fullUrl = state.fullUrl || state.url;
|
|
136
|
+
if (!fullUrl) return;
|
|
137
|
+
|
|
138
|
+
let label: string | undefined;
|
|
139
|
+
if (state.title && state.title !== prevState?.title) label = state.title;
|
|
140
|
+
else if (state.h1 && state.h1 !== prevState?.h1) label = state.h1;
|
|
141
|
+
else if (state.h2 && state.h2 !== prevState?.h2) label = state.h2;
|
|
142
|
+
else label = state.title || state.h1 || state.h2;
|
|
143
|
+
|
|
144
|
+
if (!label) return;
|
|
145
|
+
|
|
146
|
+
this.addNote(`Navigated to ${label}`, TestResult.PASSED, state.screenshotFile, fullUrl);
|
|
128
147
|
}
|
|
129
148
|
|
|
130
149
|
addState(state: WebPageState): void {
|
|
@@ -136,6 +155,28 @@ export class Task {
|
|
|
136
155
|
this.steps[timestamp] = { text, duration, status, error, log, artifacts, noteStartTime: this.activeNote?.getStartTime() };
|
|
137
156
|
}
|
|
138
157
|
|
|
158
|
+
setActiveNoteScreenshot(screenshotFile?: string): void {
|
|
159
|
+
if (!this.activeNote || !screenshotFile) return;
|
|
160
|
+
this.activeNote.screenshot = screenshotFile;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
setVerification(message: string, status: TestResultType, state?: UrlNoteState): void {
|
|
164
|
+
this.verification ||= { message: '', status: null, details: [] };
|
|
165
|
+
this.verification.message = message;
|
|
166
|
+
this.verification.status = status;
|
|
167
|
+
if (!state) return;
|
|
168
|
+
if (state.screenshotFile) this.verification.screenshot = state.screenshotFile;
|
|
169
|
+
const fullUrl = state.fullUrl || state.url;
|
|
170
|
+
if (fullUrl) this.verification.url = fullUrl;
|
|
171
|
+
this.verification.pageLabel = state.title || state.h1 || state.h2 || undefined;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
addVerificationDetail(detail: string): void {
|
|
175
|
+
if (!detail) return;
|
|
176
|
+
this.verification ||= { message: '', status: null, details: [] };
|
|
177
|
+
this.verification.details.push(detail);
|
|
178
|
+
}
|
|
179
|
+
|
|
139
180
|
getLog(): Array<{ type: 'step' | 'note' | 'artifact'; content: string; timestamp: number }> {
|
|
140
181
|
const merged: Record<string, { type: 'step' | 'note' | 'artifact'; content: string }> = {};
|
|
141
182
|
|
|
@@ -442,3 +483,21 @@ export class Plan {
|
|
|
442
483
|
return planToAiContext(this, options);
|
|
443
484
|
}
|
|
444
485
|
}
|
|
486
|
+
|
|
487
|
+
interface Verification {
|
|
488
|
+
message: string;
|
|
489
|
+
status: TestResultType;
|
|
490
|
+
screenshot?: string;
|
|
491
|
+
url?: string;
|
|
492
|
+
pageLabel?: string;
|
|
493
|
+
details: string[];
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
interface UrlNoteState {
|
|
497
|
+
url?: string;
|
|
498
|
+
fullUrl?: string;
|
|
499
|
+
title?: string;
|
|
500
|
+
h1?: string;
|
|
501
|
+
h2?: string;
|
|
502
|
+
screenshotFile?: string;
|
|
503
|
+
}
|