explorbot 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/bin/explorbot-cli.ts +29 -7
- package/boat/api-tester/src/config.ts +45 -3
- package/boat/doc-collector/src/ai/documentarian.ts +8 -2
- package/boat/doc-collector/src/ai/tools.ts +29 -16
- package/boat/doc-collector/src/cli.ts +2 -6
- package/boat/doc-collector/src/config.ts +2 -0
- package/boat/doc-collector/src/docbot.ts +42 -14
- package/boat/doc-collector/src/interaction-screenshots.ts +160 -0
- package/boat/doc-collector/src/screenshots.ts +24 -16
- package/dist/bin/explorbot-cli.js +26 -6
- package/dist/boat/api-tester/src/config.js +43 -4
- package/dist/boat/doc-collector/src/ai/documentarian.js +7 -2
- package/dist/boat/doc-collector/src/ai/tools.js +21 -15
- package/dist/boat/doc-collector/src/cli.js +2 -5
- package/dist/boat/doc-collector/src/config.js +1 -0
- package/dist/boat/doc-collector/src/docbot.js +39 -13
- package/dist/boat/doc-collector/src/interaction-screenshots.js +156 -0
- package/dist/boat/doc-collector/src/screenshots.js +25 -16
- package/dist/models.json +30 -0
- package/dist/package.json +8 -2
- package/dist/src/action.d.ts +11 -3
- package/dist/src/action.js +22 -5
- package/dist/src/ai/agent.d.ts +17 -0
- package/dist/src/ai/captain/idle-mode.js +1 -1
- package/dist/src/ai/captain/test-mode.js +1 -1
- package/dist/src/ai/captain/web-mode.js +18 -22
- package/dist/src/ai/captain.js +13 -26
- package/dist/src/ai/driller.d.ts +2 -13
- package/dist/src/ai/driller.js +23 -38
- package/dist/src/ai/historian/codeceptjs.js +1 -1
- package/dist/src/ai/historian/playwright.d.ts +3 -2
- package/dist/src/ai/historian/playwright.js +1 -1
- package/dist/src/ai/historian/screencast.d.ts +2 -1
- package/dist/src/ai/historian/screencast.js +2 -2
- package/dist/src/ai/historian.d.ts +5 -4
- package/dist/src/ai/navigator.d.ts +6 -2
- package/dist/src/ai/navigator.js +26 -22
- package/dist/src/ai/pilot.d.ts +13 -6
- package/dist/src/ai/pilot.js +49 -44
- package/dist/src/ai/planner.d.ts +3 -2
- package/dist/src/ai/planner.js +19 -17
- package/dist/src/ai/provider.js +0 -1
- package/dist/src/ai/quartermaster.d.ts +3 -2
- package/dist/src/ai/quartermaster.js +4 -4
- package/dist/src/ai/rerunner.d.ts +2 -11
- package/dist/src/ai/rerunner.js +9 -22
- package/dist/src/ai/researcher/coordinates.js +4 -3
- package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
- package/dist/src/ai/researcher/deep-analysis.js +6 -6
- package/dist/src/ai/researcher/locators.d.ts +9 -2
- package/dist/src/ai/researcher/locators.js +65 -8
- package/dist/src/ai/researcher/sections.d.ts +2 -0
- package/dist/src/ai/researcher/sections.js +2 -2
- package/dist/src/ai/researcher.js +29 -54
- package/dist/src/ai/task-agent.d.ts +18 -5
- package/dist/src/ai/task-agent.js +35 -6
- package/dist/src/ai/tester.d.ts +6 -12
- package/dist/src/ai/tester.js +37 -51
- package/dist/src/ai/tools.d.ts +11 -10
- package/dist/src/ai/tools.js +24 -36
- package/dist/src/command-handler.js +1 -1
- package/dist/src/commands/context-aria-command.js +1 -1
- package/dist/src/commands/context-command.js +7 -5
- package/dist/src/commands/context-data-command.js +2 -2
- package/dist/src/commands/context-experience-command.js +2 -2
- package/dist/src/commands/context-html-command.js +2 -2
- package/dist/src/commands/context-knowledge-command.js +1 -1
- package/dist/src/commands/drill-command.js +1 -1
- package/dist/src/commands/explore-command.js +4 -3
- package/dist/src/commands/freesail-command.js +2 -2
- package/dist/src/commands/learn-command.js +4 -4
- package/dist/src/commands/path-command.js +1 -1
- package/dist/src/commands/research-command.js +1 -1
- package/dist/src/commands/test-command.js +1 -1
- package/dist/src/components/App.js +1 -1
- package/dist/src/config.d.ts +17 -0
- package/dist/src/config.js +144 -10
- package/dist/src/experience-tracker.js +1 -1
- package/dist/src/explorbot.d.ts +17 -8
- package/dist/src/explorbot.js +74 -36
- package/dist/src/explorer.d.ts +67 -91
- package/dist/src/explorer.js +327 -567
- package/dist/src/state-manager.d.ts +6 -1
- package/dist/src/state-manager.js +1 -0
- package/dist/src/utils/hooks-runner.js +2 -4
- package/dist/src/utils/test-files.js +1 -1
- package/dist/src/utils/web-annotate.d.ts +5 -0
- package/dist/src/utils/web-annotate.js +58 -0
- package/dist/src/utils/web-eidx.d.ts +2 -0
- package/dist/src/utils/web-eidx.js +20 -0
- package/dist/src/utils/web-element.d.ts +1 -0
- package/dist/src/utils/web-element.js +29 -0
- package/models.json +30 -0
- package/package.json +8 -2
- package/src/action.ts +25 -5
- package/src/ai/agent.ts +20 -0
- package/src/ai/captain/idle-mode.ts +1 -1
- package/src/ai/captain/test-mode.ts +1 -1
- package/src/ai/captain/web-mode.ts +18 -23
- package/src/ai/captain.ts +12 -25
- package/src/ai/driller.ts +50 -67
- package/src/ai/historian/codeceptjs.ts +1 -1
- package/src/ai/historian/playwright.ts +4 -3
- package/src/ai/historian/screencast.ts +4 -3
- package/src/ai/historian.ts +3 -2
- package/src/ai/navigator.ts +30 -25
- package/src/ai/pilot.ts +55 -47
- package/src/ai/planner.ts +22 -20
- package/src/ai/provider.ts +0 -1
- package/src/ai/quartermaster.ts +5 -4
- package/src/ai/rerunner.ts +10 -29
- package/src/ai/researcher/coordinates.ts +4 -3
- package/src/ai/researcher/deep-analysis.ts +8 -6
- package/src/ai/researcher/locators.ts +66 -9
- package/src/ai/researcher/sections.ts +4 -2
- package/src/ai/researcher.ts +32 -61
- package/src/ai/task-agent.ts +45 -11
- package/src/ai/tester.ts +40 -57
- package/src/ai/tools.ts +32 -53
- package/src/command-handler.ts +1 -1
- package/src/commands/context-aria-command.ts +1 -1
- package/src/commands/context-command.ts +7 -5
- package/src/commands/context-data-command.ts +2 -2
- package/src/commands/context-experience-command.ts +2 -2
- package/src/commands/context-html-command.ts +2 -2
- package/src/commands/context-knowledge-command.ts +1 -1
- package/src/commands/drill-command.ts +1 -1
- package/src/commands/explore-command.ts +4 -3
- package/src/commands/freesail-command.ts +2 -2
- package/src/commands/learn-command.ts +4 -4
- package/src/commands/path-command.ts +1 -1
- package/src/commands/research-command.ts +1 -1
- package/src/commands/test-command.ts +1 -1
- package/src/components/App.tsx +1 -1
- package/src/config.ts +173 -11
- package/src/experience-tracker.ts +1 -1
- package/src/explorbot.ts +78 -39
- package/src/explorer.ts +388 -626
- package/src/state-manager.ts +7 -1
- package/src/utils/hooks-runner.ts +2 -4
- package/src/utils/test-files.ts +1 -1
- package/src/utils/web-annotate.ts +64 -0
- package/src/utils/web-eidx.ts +21 -0
- package/src/utils/web-element.ts +30 -0
package/src/ai/pilot.ts
CHANGED
|
@@ -2,8 +2,11 @@ import { tool } from 'ai';
|
|
|
2
2
|
import dedent from 'dedent';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { ActionResult } from '../action-result.ts';
|
|
5
|
+
import type { RequestStore } from '../api/request-store.ts';
|
|
5
6
|
import { ConfigParser } from '../config.ts';
|
|
6
7
|
import type Explorer from '../explorer.ts';
|
|
8
|
+
import type { PlaywrightRecorder } from '../playwright-recorder.ts';
|
|
9
|
+
import type { StateManager } from '../state-manager.ts';
|
|
7
10
|
import { type Test, TestResult } from '../test-plan.ts';
|
|
8
11
|
import { collectInteractiveNodes, detectFocusArea, extractFocusedElement } from '../utils/aria.ts';
|
|
9
12
|
import { ErrorPageError } from '../utils/error-page.ts';
|
|
@@ -11,7 +14,7 @@ import { createDebug, tag } from '../utils/logger.ts';
|
|
|
11
14
|
|
|
12
15
|
const debugLog = createDebug('explorbot:pilot');
|
|
13
16
|
import { truncateJson } from '../utils/strings.ts';
|
|
14
|
-
import type { Agent } from './agent.ts';
|
|
17
|
+
import type { Agent, AgentDeps } from './agent.ts';
|
|
15
18
|
import type { Conversation } from './conversation.ts';
|
|
16
19
|
import type { Fisherman } from './fisherman.ts';
|
|
17
20
|
import type { Navigator } from './navigator.ts';
|
|
@@ -30,13 +33,19 @@ export class Pilot implements Agent {
|
|
|
30
33
|
private conversation: Conversation | null = null;
|
|
31
34
|
private researcher: Researcher;
|
|
32
35
|
private explorer: Explorer;
|
|
36
|
+
private stateManager: StateManager;
|
|
37
|
+
private requestStore: RequestStore;
|
|
38
|
+
private playwrightRecorder: PlaywrightRecorder;
|
|
33
39
|
private fisherman: Fisherman | null = null;
|
|
34
40
|
|
|
35
|
-
constructor(
|
|
36
|
-
this.provider =
|
|
41
|
+
constructor(deps: AgentDeps, agentTools: any, researcher: Researcher) {
|
|
42
|
+
this.provider = deps.ai;
|
|
37
43
|
this.agentTools = agentTools;
|
|
38
44
|
this.researcher = researcher;
|
|
39
|
-
this.explorer = explorer;
|
|
45
|
+
this.explorer = deps.explorer;
|
|
46
|
+
this.stateManager = deps.stateManager;
|
|
47
|
+
this.requestStore = deps.requestStore;
|
|
48
|
+
this.playwrightRecorder = deps.playwrightRecorder;
|
|
40
49
|
}
|
|
41
50
|
|
|
42
51
|
setFisherman(fisherman: Fisherman): void {
|
|
@@ -91,7 +100,7 @@ export class Pilot implements Agent {
|
|
|
91
100
|
let screenshotState: ActionResult | null = null;
|
|
92
101
|
if (type === 'finish' && this.provider.hasVision()) {
|
|
93
102
|
try {
|
|
94
|
-
screenshotState = await this.explorer.
|
|
103
|
+
screenshotState = await this.explorer.capture({ screenshot: true });
|
|
95
104
|
if (screenshotState.screenshot) {
|
|
96
105
|
visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || '';
|
|
97
106
|
}
|
|
@@ -154,7 +163,7 @@ export class Pilot implements Agent {
|
|
|
154
163
|
const messages = [
|
|
155
164
|
{
|
|
156
165
|
role: 'system' as const,
|
|
157
|
-
content: this.buildVerdictSystemPrompt(
|
|
166
|
+
content: this.buildVerdictSystemPrompt(task),
|
|
158
167
|
},
|
|
159
168
|
{ role: 'user' as const, content: userContent },
|
|
160
169
|
];
|
|
@@ -175,7 +184,7 @@ export class Pilot implements Agent {
|
|
|
175
184
|
tag('substep').log(`Pilot requesting verification: ${result.requestVerification}`);
|
|
176
185
|
const verifyResult = await navigator.verifyState(result.requestVerification, currentState).catch(() => null);
|
|
177
186
|
if (verifyResult?.verified && verifyResult.assertionSteps?.length) {
|
|
178
|
-
this.
|
|
187
|
+
this.playwrightRecorder.recordVerification(verifyResult.assertionSteps);
|
|
179
188
|
}
|
|
180
189
|
}
|
|
181
190
|
|
|
@@ -301,10 +310,8 @@ export class Pilot implements Agent {
|
|
|
301
310
|
}
|
|
302
311
|
}
|
|
303
312
|
|
|
304
|
-
private buildSharedEvidenceRules(
|
|
313
|
+
private buildSharedEvidenceRules(): string {
|
|
305
314
|
return dedent`
|
|
306
|
-
SCENARIO: ${task.scenario}
|
|
307
|
-
|
|
308
315
|
EVIDENCE PRIORITY (strict):
|
|
309
316
|
1) Final observable state proving the scenario goal
|
|
310
317
|
2) verify()/see() results in the LAST few actions before stop/finish
|
|
@@ -337,6 +344,12 @@ export class Pilot implements Agent {
|
|
|
337
344
|
|
|
338
345
|
Expected results are MILESTONES, not the goal. Never fail because a milestone (toast, icon, styling)
|
|
339
346
|
didn't match if the scenario goal IS accomplished.
|
|
347
|
+
`;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
private buildTaskContext(task: Test): string {
|
|
351
|
+
return dedent`
|
|
352
|
+
SCENARIO: ${task.scenario}
|
|
340
353
|
|
|
341
354
|
${this.buildDeletionScope(task)}
|
|
342
355
|
|
|
@@ -351,7 +364,7 @@ export class Pilot implements Agent {
|
|
|
351
364
|
iteration's work, but server-side side effects (records created, forms submitted) persist.
|
|
352
365
|
Unnecessary resets create duplicate data and infinite loops.
|
|
353
366
|
|
|
354
|
-
${this.buildSharedEvidenceRules(
|
|
367
|
+
${this.buildSharedEvidenceRules()}
|
|
355
368
|
|
|
356
369
|
DECISION:
|
|
357
370
|
- "allow": current page cannot host the scenario, irrecoverable error, or no path back.
|
|
@@ -368,17 +381,19 @@ export class Pilot implements Agent {
|
|
|
368
381
|
to verify, how to record. Do not suggest repeating actions that already succeeded.
|
|
369
382
|
If progress is blocked only because the page lacks target data for the scenario, prefer precondition()
|
|
370
383
|
over repeated UI attempts.
|
|
384
|
+
|
|
385
|
+
${this.buildTaskContext(task)}
|
|
371
386
|
`;
|
|
372
387
|
}
|
|
373
388
|
|
|
374
|
-
private buildVerdictSystemPrompt(
|
|
389
|
+
private buildVerdictSystemPrompt(task: Test): string {
|
|
375
390
|
return dedent`
|
|
376
|
-
You are Pilot — final decision maker for test pass/fail.
|
|
377
|
-
|
|
391
|
+
You are Pilot — final decision maker for test pass/fail. Review the evidence and commit to a
|
|
392
|
+
verdict; "continue" only when evidence is genuinely insufficient.
|
|
378
393
|
|
|
379
394
|
${capabilityGroundingRule}
|
|
380
395
|
|
|
381
|
-
${this.buildSharedEvidenceRules(
|
|
396
|
+
${this.buildSharedEvidenceRules()}
|
|
382
397
|
|
|
383
398
|
DECISION:
|
|
384
399
|
- "pass": scenario goal is fully accomplished. Set requestVerification to a one-sentence claim about
|
|
@@ -397,6 +412,8 @@ export class Pilot implements Agent {
|
|
|
397
412
|
reason field: one short sentence, maximum 120 characters. Do NOT restate the decision
|
|
398
413
|
("scenario goal achieved/not achieved"). State what happened: what was verified, what failed,
|
|
399
414
|
or what evidence was found.
|
|
415
|
+
|
|
416
|
+
${this.buildTaskContext(task)}
|
|
400
417
|
`;
|
|
401
418
|
}
|
|
402
419
|
|
|
@@ -474,9 +491,6 @@ export class Pilot implements Agent {
|
|
|
474
491
|
.slice(-this.stepsToReview);
|
|
475
492
|
const actionsContext = this.formatActions(toolCalls);
|
|
476
493
|
|
|
477
|
-
this.conversation.cleanupTag('page_summary', '...trimmed...', 1);
|
|
478
|
-
this.conversation.cleanupTag('recent_actions', '...trimmed...', 2);
|
|
479
|
-
|
|
480
494
|
return this.sendToPilot(
|
|
481
495
|
dedent`
|
|
482
496
|
Navigated to new page.
|
|
@@ -498,7 +512,8 @@ export class Pilot implements Agent {
|
|
|
498
512
|
|
|
499
513
|
First: evaluate whether this navigation makes sense for the scenario goal. If the page is unrelated, instruct Tester to back() or reset(). Then plan next steps.
|
|
500
514
|
`,
|
|
501
|
-
'pilot.reviewNewPage'
|
|
515
|
+
'pilot.reviewNewPage',
|
|
516
|
+
{ task }
|
|
502
517
|
);
|
|
503
518
|
}
|
|
504
519
|
|
|
@@ -515,8 +530,6 @@ export class Pilot implements Agent {
|
|
|
515
530
|
const actionsContext = this.formatActions(toolCalls);
|
|
516
531
|
const stateContext = this.buildStateContext(currentState);
|
|
517
532
|
|
|
518
|
-
this.conversation.cleanupTag('recent_actions', '...trimmed...', 2);
|
|
519
|
-
|
|
520
533
|
const hasFailures = toolCalls.length === 0 || toolCalls.some((t) => !t.wasSuccessful);
|
|
521
534
|
|
|
522
535
|
const text = await this.sendToPilot(
|
|
@@ -554,7 +567,7 @@ export class Pilot implements Agent {
|
|
|
554
567
|
return `CHECKED: ${checked.length > 0 ? checked.join(', ') : 'none'}\nREMAINING: ${remaining.length > 0 ? remaining.join(', ') : 'none'}`;
|
|
555
568
|
}
|
|
556
569
|
|
|
557
|
-
private async sendToPilot(userText: string, functionId: string, opts: { tools?: boolean; maxToolRoundtrips?: number; task
|
|
570
|
+
private async sendToPilot(userText: string, functionId: string, opts: { tools?: boolean; maxToolRoundtrips?: number; task: Test }): Promise<string> {
|
|
558
571
|
debugLog(`sendToPilot: ${functionId}, tools: ${!!opts.tools}, roundtrips: ${opts.maxToolRoundtrips ?? 0}`);
|
|
559
572
|
|
|
560
573
|
let finalUserText = userText;
|
|
@@ -565,19 +578,14 @@ export class Pilot implements Agent {
|
|
|
565
578
|
}
|
|
566
579
|
}
|
|
567
580
|
this.conversation!.addUserText(finalUserText);
|
|
568
|
-
let tools: any;
|
|
569
|
-
if (opts.tools) {
|
|
570
|
-
tools = this.pickPlanningTools();
|
|
571
|
-
}
|
|
572
581
|
|
|
573
|
-
|
|
574
|
-
tools = { ...tools, ...this.buildPreconditionTool(opts.task) };
|
|
575
|
-
}
|
|
582
|
+
const tools = { ...this.pickPlanningTools(), ...this.buildPreconditionTool(opts.task) };
|
|
576
583
|
|
|
577
584
|
const result = await this.provider.invokeConversation(this.conversation!, tools, {
|
|
578
585
|
maxToolRoundtrips: opts.maxToolRoundtrips ?? 0,
|
|
586
|
+
toolChoice: opts.tools ? 'auto' : 'none',
|
|
579
587
|
agentName: 'pilot',
|
|
580
|
-
stopWhen:
|
|
588
|
+
stopWhen: () => opts.task.hasFinished,
|
|
581
589
|
telemetry: { functionId },
|
|
582
590
|
});
|
|
583
591
|
const text = result?.response?.text || '';
|
|
@@ -596,9 +604,9 @@ export class Pilot implements Agent {
|
|
|
596
604
|
}
|
|
597
605
|
|
|
598
606
|
private getExperienceToc(): string {
|
|
599
|
-
const state = this.
|
|
607
|
+
const state = this.stateManager.getCurrentState();
|
|
600
608
|
if (!state) return '';
|
|
601
|
-
return this.
|
|
609
|
+
return this.stateManager.getExperienceTracker().renderExperienceTocFor(ActionResult.fromState(state));
|
|
602
610
|
}
|
|
603
611
|
|
|
604
612
|
private pickPlanningTools() {
|
|
@@ -661,7 +669,7 @@ export class Pilot implements Agent {
|
|
|
661
669
|
private async checkDataAvailability(task: Test, requestedData: string, fishermanReason: string | undefined): Promise<string | null> {
|
|
662
670
|
if (!this.provider.hasVision()) return null;
|
|
663
671
|
|
|
664
|
-
const screenshotState = await this.explorer.
|
|
672
|
+
const screenshotState = await this.explorer.capture({ screenshot: true }).catch(() => null);
|
|
665
673
|
if (!screenshotState?.screenshot) return null;
|
|
666
674
|
|
|
667
675
|
const question = dedent`
|
|
@@ -713,8 +721,8 @@ export class Pilot implements Agent {
|
|
|
713
721
|
lines.push('modal: none');
|
|
714
722
|
}
|
|
715
723
|
|
|
716
|
-
|
|
717
|
-
|
|
724
|
+
const tabs = this.stateManager.otherTabs;
|
|
725
|
+
if (tabs.length > 0) {
|
|
718
726
|
lines.push(`other tabs: ${tabs.length} (${tabs.map((t) => `${t.url} - ${t.title}`).join(', ')})`);
|
|
719
727
|
} else {
|
|
720
728
|
lines.push('other tabs: none');
|
|
@@ -737,7 +745,7 @@ export class Pilot implements Agent {
|
|
|
737
745
|
lines.push('console errors: none');
|
|
738
746
|
}
|
|
739
747
|
|
|
740
|
-
const failedRequests = this.
|
|
748
|
+
const failedRequests = this.requestStore.getFailedRequests();
|
|
741
749
|
if (failedRequests.length > 0) {
|
|
742
750
|
const sample = failedRequests
|
|
743
751
|
.slice(-5)
|
|
@@ -818,7 +826,7 @@ export class Pilot implements Agent {
|
|
|
818
826
|
|
|
819
827
|
private formatSessionLog(testerConversation: Conversation): string {
|
|
820
828
|
const executions = testerConversation.getToolExecutions().filter((t) => !META_TOOLS.includes(t.toolName));
|
|
821
|
-
const stateHistory = this.
|
|
829
|
+
const stateHistory = this.stateManager.getStateHistory();
|
|
822
830
|
|
|
823
831
|
const initialUrl = stateHistory[0]?.toState?.url || '';
|
|
824
832
|
let currentUrl = initialUrl;
|
|
@@ -975,16 +983,6 @@ export class Pilot implements Agent {
|
|
|
975
983
|
return dedent`
|
|
976
984
|
You are Pilot — a supervisor that detects problems and intervenes only when needed.
|
|
977
985
|
|
|
978
|
-
SCENARIO: ${task.scenario}
|
|
979
|
-
START URL: ${initialState.url}
|
|
980
|
-
PAGE: ${initialState.title || ''} | ${initialState.h1 || ''}
|
|
981
|
-
|
|
982
|
-
EXPECTED RESULTS:
|
|
983
|
-
${task.expected.map((e) => `- ${e}`).join('\n')}
|
|
984
|
-
|
|
985
|
-
PLANNED STEPS:
|
|
986
|
-
${stepsText}
|
|
987
|
-
|
|
988
986
|
Your job: plan, review new pages, detect stuck patterns, suggest concrete next steps. Track which
|
|
989
987
|
expectations are checked. When things go well, encourage briefly and let Tester continue. The current
|
|
990
988
|
page is usually richer than the page summary lists — prefer exploring it before navigating away.
|
|
@@ -1054,6 +1052,16 @@ export class Pilot implements Agent {
|
|
|
1054
1052
|
NEXT: <specific actionable instruction for Tester>
|
|
1055
1053
|
|
|
1056
1054
|
Keep user-facing reasons concise: one short sentence, maximum 120 characters, evidence only, no repeated verdict wording.
|
|
1055
|
+
|
|
1056
|
+
SCENARIO: ${task.scenario}
|
|
1057
|
+
START URL: ${initialState.url}
|
|
1058
|
+
PAGE: ${initialState.title || ''} | ${initialState.h1 || ''}
|
|
1059
|
+
|
|
1060
|
+
EXPECTED RESULTS:
|
|
1061
|
+
${task.expected.map((e) => `- ${e}`).join('\n')}
|
|
1062
|
+
|
|
1063
|
+
PLANNED STEPS:
|
|
1064
|
+
${stepsText}
|
|
1057
1065
|
`;
|
|
1058
1066
|
}
|
|
1059
1067
|
}
|
package/src/ai/planner.ts
CHANGED
|
@@ -14,7 +14,7 @@ import { createDebug, tag } from '../utils/logger.js';
|
|
|
14
14
|
import { jsonToTable } from '../utils/markdown-parser.ts';
|
|
15
15
|
import { mdq } from '../utils/markdown-query.js';
|
|
16
16
|
import { planToCompactAiContext } from '../utils/test-plan-markdown.ts';
|
|
17
|
-
import type { Agent } from './agent.js';
|
|
17
|
+
import type { Agent, AgentDeps } from './agent.js';
|
|
18
18
|
import { Conversation } from './conversation.ts';
|
|
19
19
|
import type { Fisherman } from './fisherman.ts';
|
|
20
20
|
import { WithSessionDedup } from './planner/session-dedup.ts';
|
|
@@ -63,13 +63,13 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
63
63
|
researcher: Researcher;
|
|
64
64
|
private fisherman: Fisherman | null = null;
|
|
65
65
|
|
|
66
|
-
constructor(
|
|
66
|
+
constructor(deps: AgentDeps, researcher: Researcher) {
|
|
67
67
|
super();
|
|
68
|
-
this.explorer = explorer;
|
|
69
|
-
this.provider =
|
|
68
|
+
this.explorer = deps.explorer;
|
|
69
|
+
this.provider = deps.ai;
|
|
70
70
|
this.researcher = researcher;
|
|
71
|
-
this.stateManager =
|
|
72
|
-
this.experienceTracker =
|
|
71
|
+
this.stateManager = deps.stateManager;
|
|
72
|
+
this.experienceTracker = deps.stateManager.getExperienceTracker();
|
|
73
73
|
}
|
|
74
74
|
|
|
75
75
|
setFisherman(fisherman: Fisherman): void {
|
|
@@ -374,16 +374,10 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
374
374
|
${dataProtectionRules}
|
|
375
375
|
${fileUploadRule}
|
|
376
376
|
</rules>
|
|
377
|
-
|
|
378
|
-
${this.buildApproach(style)}
|
|
379
|
-
|
|
380
|
-
<context>
|
|
381
|
-
URL: ${state.url || 'Unknown'}
|
|
382
|
-
Title: ${state.title || 'Unknown'}
|
|
383
|
-
</context>
|
|
384
377
|
`;
|
|
385
378
|
|
|
386
379
|
conversation.addUserText(planningPrompt);
|
|
380
|
+
conversation.addUserText(this.getTasksMessage());
|
|
387
381
|
const currentState = this.stateManager.getCurrentState();
|
|
388
382
|
const research = await this.researcher.research(currentState || state, {
|
|
389
383
|
deep: true,
|
|
@@ -417,6 +411,15 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
417
411
|
</page_research>
|
|
418
412
|
`);
|
|
419
413
|
|
|
414
|
+
conversation.addUserText(dedent`
|
|
415
|
+
${this.buildApproach(style)}
|
|
416
|
+
|
|
417
|
+
<context>
|
|
418
|
+
URL: ${state.url || 'Unknown'}
|
|
419
|
+
Title: ${state.title || 'Unknown'}
|
|
420
|
+
</context>
|
|
421
|
+
`);
|
|
422
|
+
|
|
420
423
|
if (this.fisherman) {
|
|
421
424
|
await this.fisherman.ensureReady(state.url);
|
|
422
425
|
if (this.fisherman.isAvailable()) {
|
|
@@ -568,8 +571,11 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
568
571
|
`);
|
|
569
572
|
}
|
|
570
573
|
|
|
571
|
-
|
|
572
|
-
|
|
574
|
+
return conversation;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
private getTasksMessage(): string {
|
|
578
|
+
return dedent`
|
|
573
579
|
<task>
|
|
574
580
|
Provide testing scenarios as structured data with the following requirements:
|
|
575
581
|
1. Create a short, descriptive plan name that summarizes what will be tested (e.g., "User Authentication Testing", "Product Catalog Navigation", "Form Validation Tests")
|
|
@@ -615,12 +621,8 @@ export class Planner extends PlannerBase implements Agent {
|
|
|
615
621
|
- Do not wrap text in ** or * quotes, ( or ) brackets.
|
|
616
622
|
- Avoid using emojis or special characters.
|
|
617
623
|
7. Only tests that can be tested from web UI should be proposed.
|
|
618
|
-
|
|
624
|
+
8. For a fresh plan propose at least ${this.MIN_TASKS} tests; when expanding an existing plan return ONLY new scenarios not in the tested list, and an empty array if none remain.
|
|
619
625
|
</task>
|
|
620
626
|
`;
|
|
621
|
-
|
|
622
|
-
conversation.addUserText(tasksMessage);
|
|
623
|
-
|
|
624
|
-
return conversation;
|
|
625
627
|
}
|
|
626
628
|
}
|
package/src/ai/provider.ts
CHANGED
package/src/ai/quartermaster.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { join } from 'node:path';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import type { ActionResult } from '../action-result.ts';
|
|
5
5
|
import { outputPath } from '../config.ts';
|
|
6
|
+
import type Explorer from '../explorer.ts';
|
|
6
7
|
import type { StateManager, StateTransition, WebPageState } from '../state-manager.ts';
|
|
7
8
|
import type { Task } from '../test-plan.ts';
|
|
8
9
|
import { createDebug, tag } from '../utils/logger.ts';
|
|
@@ -18,7 +19,7 @@ export class Quartermaster {
|
|
|
18
19
|
private outputDir: string;
|
|
19
20
|
private pageAnalyses: Map<string, PageAnalysis> = new Map();
|
|
20
21
|
private unsubscribe: (() => void) | null = null;
|
|
21
|
-
private
|
|
22
|
+
private explorer: Explorer | null = null;
|
|
22
23
|
private pendingAnalyses: Promise<void>[] = [];
|
|
23
24
|
|
|
24
25
|
constructor(provider: Provider, options?: { model?: any }) {
|
|
@@ -28,8 +29,8 @@ export class Quartermaster {
|
|
|
28
29
|
this.outputDir = outputPath('a11y');
|
|
29
30
|
}
|
|
30
31
|
|
|
31
|
-
start(
|
|
32
|
-
this.
|
|
32
|
+
start(explorer: Explorer, stateManager: StateManager): void {
|
|
33
|
+
this.explorer = explorer;
|
|
33
34
|
mkdirSync(this.outputDir, { recursive: true });
|
|
34
35
|
|
|
35
36
|
this.unsubscribe = stateManager.onStateChange((event) => {
|
|
@@ -60,7 +61,7 @@ export class Quartermaster {
|
|
|
60
61
|
}
|
|
61
62
|
|
|
62
63
|
private async runAxeAnalysis(state: WebPageState): Promise<void> {
|
|
63
|
-
const page = this.
|
|
64
|
+
const page = this.explorer?.page;
|
|
64
65
|
if (!page || !state.hash) {
|
|
65
66
|
debugLog('No page available for axe analysis');
|
|
66
67
|
return;
|
package/src/ai/rerunner.ts
CHANGED
|
@@ -12,16 +12,13 @@ import figureSet from 'figures';
|
|
|
12
12
|
import { z } from 'zod';
|
|
13
13
|
import { ActionResult } from '../action-result.ts';
|
|
14
14
|
import { setActivity } from '../activity.ts';
|
|
15
|
-
import type { ExperienceTracker } from '../experience-tracker.ts';
|
|
16
|
-
import type Explorer from '../explorer.ts';
|
|
17
|
-
import type { KnowledgeTracker } from '../knowledge-tracker.ts';
|
|
18
15
|
import { Stats } from '../stats.ts';
|
|
19
16
|
import { Task, Test, TestResult } from '../test-plan.ts';
|
|
20
17
|
import { formatHeadings } from '../utils/context-formatter.ts';
|
|
21
18
|
import { createDebug, tag } from '../utils/logger.ts';
|
|
22
19
|
import { loop } from '../utils/loop.ts';
|
|
23
20
|
import { RulesLoader } from '../utils/rules-loader.ts';
|
|
24
|
-
import type { Agent } from './agent.ts';
|
|
21
|
+
import type { Agent, AgentDeps } from './agent.ts';
|
|
25
22
|
import { toolExecutionLabel } from './conversation.ts';
|
|
26
23
|
import type { Navigator } from './navigator.ts';
|
|
27
24
|
import { Provider } from './provider.ts';
|
|
@@ -35,17 +32,13 @@ export class Rerunner extends TaskAgent implements Agent {
|
|
|
35
32
|
protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form'];
|
|
36
33
|
emoji = '🔄';
|
|
37
34
|
|
|
38
|
-
private explorer: Explorer;
|
|
39
|
-
private provider: Provider;
|
|
40
35
|
private agentTools: any;
|
|
41
36
|
private healedSteps: Array<{ original: string; healed: string }> = [];
|
|
42
37
|
private traceDir = '';
|
|
43
38
|
private static pluginsWired = false;
|
|
44
39
|
|
|
45
|
-
constructor(
|
|
46
|
-
super();
|
|
47
|
-
this.explorer = explorer;
|
|
48
|
-
this.provider = provider;
|
|
40
|
+
constructor(deps: AgentDeps, agentTools?: any) {
|
|
41
|
+
super(deps);
|
|
49
42
|
this.agentTools = agentTools;
|
|
50
43
|
}
|
|
51
44
|
|
|
@@ -53,20 +46,8 @@ export class Rerunner extends TaskAgent implements Agent {
|
|
|
53
46
|
throw new Error('Rerunner does not use Navigator');
|
|
54
47
|
}
|
|
55
48
|
|
|
56
|
-
protected getExperienceTracker(): ExperienceTracker {
|
|
57
|
-
return this.explorer.getStateManager().getExperienceTracker();
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
protected getKnowledgeTracker(): KnowledgeTracker {
|
|
61
|
-
return this.explorer.getKnowledgeTracker();
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
protected getProvider(): Provider {
|
|
65
|
-
return this.provider;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
49
|
private get rerunnerConfig(): Record<string, any> {
|
|
69
|
-
return (this.
|
|
50
|
+
return (this.config.ai?.agents?.rerunner as any) || {};
|
|
70
51
|
}
|
|
71
52
|
|
|
72
53
|
private get healLimit(): number {
|
|
@@ -325,7 +306,7 @@ export class Rerunner extends TaskAgent implements Agent {
|
|
|
325
306
|
});
|
|
326
307
|
|
|
327
308
|
const healTask = new Task(`Heal: ${failedCode}`);
|
|
328
|
-
const codeceptTools = createCodeceptJSTools(this.
|
|
309
|
+
const codeceptTools = createCodeceptJSTools(this.toolDeps, healTask);
|
|
329
310
|
|
|
330
311
|
let healed = false;
|
|
331
312
|
let healedCommand = '';
|
|
@@ -345,9 +326,9 @@ export class Rerunner extends TaskAgent implements Agent {
|
|
|
345
326
|
healTask.addNote(note);
|
|
346
327
|
tag('substep').log(note);
|
|
347
328
|
}
|
|
348
|
-
const action = this.explorer.
|
|
329
|
+
const action = this.explorer.action();
|
|
349
330
|
await action.execute(`I.wait(${seconds})`);
|
|
350
|
-
const state = this.
|
|
331
|
+
const state = this.stateManager.getCurrentState();
|
|
351
332
|
const ar = state ? ActionResult.fromState(state) : null;
|
|
352
333
|
return {
|
|
353
334
|
success: true,
|
|
@@ -438,8 +419,8 @@ export class Rerunner extends TaskAgent implements Agent {
|
|
|
438
419
|
}
|
|
439
420
|
|
|
440
421
|
private getHealSystemPrompt(): string {
|
|
441
|
-
const customRules = this.provider.getSystemPromptForAgent('rerunner', this.
|
|
442
|
-
const currentUrl = this.
|
|
422
|
+
const customRules = this.provider.getSystemPromptForAgent('rerunner', this.stateManager.getCurrentState()?.url) || '';
|
|
423
|
+
const currentUrl = this.stateManager.getCurrentState()?.url || '';
|
|
443
424
|
const approach = RulesLoader.loadRules('rerunner', ['healing-approach'], currentUrl);
|
|
444
425
|
|
|
445
426
|
return dedent`
|
|
@@ -474,7 +455,7 @@ export class Rerunner extends TaskAgent implements Agent {
|
|
|
474
455
|
}
|
|
475
456
|
|
|
476
457
|
private getHealUserPrompt(failedCode: string, error: Error): string {
|
|
477
|
-
const state = this.
|
|
458
|
+
const state = this.stateManager.getCurrentState();
|
|
478
459
|
const actionResult = state ? ActionResult.fromState(state) : null;
|
|
479
460
|
|
|
480
461
|
const headings = formatHeadings(state || {});
|
|
@@ -4,6 +4,7 @@ import type { ActionResult } from '../../action-result.js';
|
|
|
4
4
|
import type Explorer from '../../explorer.ts';
|
|
5
5
|
import { tag } from '../../utils/logger.js';
|
|
6
6
|
import { mdq } from '../../utils/markdown-query.ts';
|
|
7
|
+
import { eidxByLocator } from '../../utils/web-eidx.ts';
|
|
7
8
|
import { WebElement } from '../../utils/web-element.ts';
|
|
8
9
|
import type { Provider } from '../provider.js';
|
|
9
10
|
import { type Constructor, debugLog } from './mixin.ts';
|
|
@@ -81,7 +82,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
81
82
|
}
|
|
82
83
|
|
|
83
84
|
async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise<number> {
|
|
84
|
-
return this.explorer.
|
|
85
|
+
return this.explorer.withPage((page) => visuallyAnnotateContainers(page, opts?.containers || []));
|
|
85
86
|
}
|
|
86
87
|
|
|
87
88
|
private async _analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult> {
|
|
@@ -177,7 +178,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
177
178
|
let eidx = el.eidx || null;
|
|
178
179
|
if (!eidx) {
|
|
179
180
|
const locator = el.css || el.xpath || (el.aria ? `role=${el.aria.role}[name="${el.aria.text}"]` : null);
|
|
180
|
-
if (locator) eidx = await this.explorer.
|
|
181
|
+
if (locator) eidx = await this.explorer.withPage((page) => eidxByLocator(page, locator, section.containerCss));
|
|
181
182
|
}
|
|
182
183
|
if (!eidx) continue;
|
|
183
184
|
|
|
@@ -202,7 +203,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
202
203
|
}
|
|
203
204
|
if (eidxWithoutCoords.length === 0) return;
|
|
204
205
|
|
|
205
|
-
const webElements = await this.explorer.
|
|
206
|
+
const webElements = await this.explorer.withPage((page) => WebElement.fromEidxList(page, eidxWithoutCoords));
|
|
206
207
|
if (webElements.length === 0) return;
|
|
207
208
|
|
|
208
209
|
const rectMap = new Map(webElements.map((w) => [w.eidx!, w]));
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import dedent from 'dedent';
|
|
2
2
|
import { ActionResult, type Diff } from '../../action-result.js';
|
|
3
|
+
import type { ExplorbotConfig } from '../../config.ts';
|
|
3
4
|
import { executionController } from '../../execution-controller.ts';
|
|
4
5
|
import type Explorer from '../../explorer.ts';
|
|
5
6
|
import type { StateManager } from '../../state-manager.js';
|
|
@@ -21,13 +22,14 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
21
22
|
declare explorer: Explorer;
|
|
22
23
|
declare provider: Provider;
|
|
23
24
|
declare stateManager: StateManager;
|
|
25
|
+
declare config: ExplorbotConfig;
|
|
24
26
|
declare actionResult: ActionResult | undefined;
|
|
25
27
|
|
|
26
28
|
async performDeepAnalysis(state: WebPageState, result: ResearchResult): Promise<void> {
|
|
27
29
|
tag('info').log('Starting deep analysis of expandable elements');
|
|
28
30
|
await (this as any).navigateTo(state.fullUrl || state.url);
|
|
29
31
|
|
|
30
|
-
const maxClicks = (this.
|
|
32
|
+
const maxClicks = (this.config.ai?.agents?.researcher as any)?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS;
|
|
31
33
|
|
|
32
34
|
const expandedSections: string[] = [];
|
|
33
35
|
const navigationLinks: Array<{ code: string; url: string }> = [];
|
|
@@ -358,10 +360,10 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
358
360
|
const isCoordinateClick = el.commands[0].startsWith('I.clickXY(');
|
|
359
361
|
if (!isCoordinateClick) {
|
|
360
362
|
const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo(');
|
|
361
|
-
await this.explorer.
|
|
363
|
+
await this.explorer.action().attempt(hoverCmd);
|
|
362
364
|
await new Promise((r) => setTimeout(r, 500));
|
|
363
365
|
|
|
364
|
-
await this.explorer.
|
|
366
|
+
await this.explorer.capture();
|
|
365
367
|
const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState()!);
|
|
366
368
|
const hoverDiff = await hoverAR.diff(previousState);
|
|
367
369
|
const hoverHtmlSize = hoverDiff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0);
|
|
@@ -401,7 +403,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
401
403
|
const previousState = ActionResult.fromState(this.stateManager.getCurrentState()!);
|
|
402
404
|
|
|
403
405
|
let clickCode: string | null = null;
|
|
404
|
-
const action = this.explorer.
|
|
406
|
+
const action = this.explorer.action();
|
|
405
407
|
for (const cmd of commands) {
|
|
406
408
|
if (await action.attempt(cmd, undefined)) {
|
|
407
409
|
clickCode = cmd;
|
|
@@ -417,7 +419,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
417
419
|
|
|
418
420
|
let diff: Diff;
|
|
419
421
|
try {
|
|
420
|
-
await this.explorer.
|
|
422
|
+
await this.explorer.capture();
|
|
421
423
|
const currAR = ActionResult.fromState(this.stateManager.getCurrentState()!);
|
|
422
424
|
diff = await currAR.diff(previousState);
|
|
423
425
|
} catch (err) {
|
|
@@ -448,7 +450,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
448
450
|
private async _restorePageState(url: string, originalAria: string): Promise<void> {
|
|
449
451
|
try {
|
|
450
452
|
await (this as any).cancelInUi();
|
|
451
|
-
await this.explorer.
|
|
453
|
+
await this.explorer.capture();
|
|
452
454
|
const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || '';
|
|
453
455
|
if (!diffAriaSnapshots(originalAria, currentAria).text) return;
|
|
454
456
|
} catch (err) {
|