explorbot 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/bin/explorbot-cli.ts +29 -7
- package/boat/api-tester/src/config.ts +45 -3
- package/boat/doc-collector/src/ai/documentarian.ts +8 -2
- package/boat/doc-collector/src/ai/tools.ts +29 -16
- package/boat/doc-collector/src/cli.ts +2 -6
- package/boat/doc-collector/src/config.ts +2 -0
- package/boat/doc-collector/src/docbot.ts +42 -14
- package/boat/doc-collector/src/interaction-screenshots.ts +160 -0
- package/boat/doc-collector/src/screenshots.ts +24 -16
- package/dist/bin/explorbot-cli.js +26 -6
- package/dist/boat/api-tester/src/config.js +43 -4
- package/dist/boat/doc-collector/src/ai/documentarian.js +7 -2
- package/dist/boat/doc-collector/src/ai/tools.js +21 -15
- package/dist/boat/doc-collector/src/cli.js +2 -5
- package/dist/boat/doc-collector/src/config.js +1 -0
- package/dist/boat/doc-collector/src/docbot.js +39 -13
- package/dist/boat/doc-collector/src/interaction-screenshots.js +156 -0
- package/dist/boat/doc-collector/src/screenshots.js +25 -16
- package/dist/models.json +30 -0
- package/dist/package.json +8 -2
- package/dist/src/action.d.ts +11 -3
- package/dist/src/action.js +22 -5
- package/dist/src/ai/agent.d.ts +17 -0
- package/dist/src/ai/captain/idle-mode.js +1 -1
- package/dist/src/ai/captain/test-mode.js +1 -1
- package/dist/src/ai/captain/web-mode.js +18 -22
- package/dist/src/ai/captain.js +13 -26
- package/dist/src/ai/driller.d.ts +2 -13
- package/dist/src/ai/driller.js +23 -38
- package/dist/src/ai/historian/codeceptjs.js +1 -1
- package/dist/src/ai/historian/playwright.d.ts +3 -2
- package/dist/src/ai/historian/playwright.js +1 -1
- package/dist/src/ai/historian/screencast.d.ts +2 -1
- package/dist/src/ai/historian/screencast.js +2 -2
- package/dist/src/ai/historian.d.ts +5 -4
- package/dist/src/ai/navigator.d.ts +6 -2
- package/dist/src/ai/navigator.js +26 -22
- package/dist/src/ai/pilot.d.ts +13 -6
- package/dist/src/ai/pilot.js +49 -44
- package/dist/src/ai/planner.d.ts +3 -2
- package/dist/src/ai/planner.js +19 -17
- package/dist/src/ai/provider.js +0 -1
- package/dist/src/ai/quartermaster.d.ts +3 -2
- package/dist/src/ai/quartermaster.js +4 -4
- package/dist/src/ai/rerunner.d.ts +2 -11
- package/dist/src/ai/rerunner.js +9 -22
- package/dist/src/ai/researcher/coordinates.js +4 -3
- package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
- package/dist/src/ai/researcher/deep-analysis.js +6 -6
- package/dist/src/ai/researcher/locators.d.ts +9 -2
- package/dist/src/ai/researcher/locators.js +65 -8
- package/dist/src/ai/researcher/sections.d.ts +2 -0
- package/dist/src/ai/researcher/sections.js +2 -2
- package/dist/src/ai/researcher.js +29 -54
- package/dist/src/ai/task-agent.d.ts +18 -5
- package/dist/src/ai/task-agent.js +35 -6
- package/dist/src/ai/tester.d.ts +6 -12
- package/dist/src/ai/tester.js +37 -51
- package/dist/src/ai/tools.d.ts +11 -10
- package/dist/src/ai/tools.js +24 -36
- package/dist/src/command-handler.js +1 -1
- package/dist/src/commands/context-aria-command.js +1 -1
- package/dist/src/commands/context-command.js +7 -5
- package/dist/src/commands/context-data-command.js +2 -2
- package/dist/src/commands/context-experience-command.js +2 -2
- package/dist/src/commands/context-html-command.js +2 -2
- package/dist/src/commands/context-knowledge-command.js +1 -1
- package/dist/src/commands/drill-command.js +1 -1
- package/dist/src/commands/explore-command.js +4 -3
- package/dist/src/commands/freesail-command.js +2 -2
- package/dist/src/commands/learn-command.js +4 -4
- package/dist/src/commands/path-command.js +1 -1
- package/dist/src/commands/research-command.js +1 -1
- package/dist/src/commands/test-command.js +1 -1
- package/dist/src/components/App.js +1 -1
- package/dist/src/config.d.ts +17 -0
- package/dist/src/config.js +144 -10
- package/dist/src/experience-tracker.js +1 -1
- package/dist/src/explorbot.d.ts +17 -8
- package/dist/src/explorbot.js +74 -36
- package/dist/src/explorer.d.ts +67 -91
- package/dist/src/explorer.js +327 -567
- package/dist/src/state-manager.d.ts +6 -1
- package/dist/src/state-manager.js +1 -0
- package/dist/src/utils/hooks-runner.js +2 -4
- package/dist/src/utils/test-files.js +1 -1
- package/dist/src/utils/web-annotate.d.ts +5 -0
- package/dist/src/utils/web-annotate.js +58 -0
- package/dist/src/utils/web-eidx.d.ts +2 -0
- package/dist/src/utils/web-eidx.js +20 -0
- package/dist/src/utils/web-element.d.ts +1 -0
- package/dist/src/utils/web-element.js +29 -0
- package/models.json +30 -0
- package/package.json +8 -2
- package/src/action.ts +25 -5
- package/src/ai/agent.ts +20 -0
- package/src/ai/captain/idle-mode.ts +1 -1
- package/src/ai/captain/test-mode.ts +1 -1
- package/src/ai/captain/web-mode.ts +18 -23
- package/src/ai/captain.ts +12 -25
- package/src/ai/driller.ts +50 -67
- package/src/ai/historian/codeceptjs.ts +1 -1
- package/src/ai/historian/playwright.ts +4 -3
- package/src/ai/historian/screencast.ts +4 -3
- package/src/ai/historian.ts +3 -2
- package/src/ai/navigator.ts +30 -25
- package/src/ai/pilot.ts +55 -47
- package/src/ai/planner.ts +22 -20
- package/src/ai/provider.ts +0 -1
- package/src/ai/quartermaster.ts +5 -4
- package/src/ai/rerunner.ts +10 -29
- package/src/ai/researcher/coordinates.ts +4 -3
- package/src/ai/researcher/deep-analysis.ts +8 -6
- package/src/ai/researcher/locators.ts +66 -9
- package/src/ai/researcher/sections.ts +4 -2
- package/src/ai/researcher.ts +32 -61
- package/src/ai/task-agent.ts +45 -11
- package/src/ai/tester.ts +40 -57
- package/src/ai/tools.ts +32 -53
- package/src/command-handler.ts +1 -1
- package/src/commands/context-aria-command.ts +1 -1
- package/src/commands/context-command.ts +7 -5
- package/src/commands/context-data-command.ts +2 -2
- package/src/commands/context-experience-command.ts +2 -2
- package/src/commands/context-html-command.ts +2 -2
- package/src/commands/context-knowledge-command.ts +1 -1
- package/src/commands/drill-command.ts +1 -1
- package/src/commands/explore-command.ts +4 -3
- package/src/commands/freesail-command.ts +2 -2
- package/src/commands/learn-command.ts +4 -4
- package/src/commands/path-command.ts +1 -1
- package/src/commands/research-command.ts +1 -1
- package/src/commands/test-command.ts +1 -1
- package/src/components/App.tsx +1 -1
- package/src/config.ts +173 -11
- package/src/experience-tracker.ts +1 -1
- package/src/explorbot.ts +78 -39
- package/src/explorer.ts +388 -626
- package/src/state-manager.ts +7 -1
- package/src/utils/hooks-runner.ts +2 -4
- package/src/utils/test-files.ts +1 -1
- package/src/utils/web-annotate.ts +64 -0
- package/src/utils/web-eidx.ts +21 -0
- package/src/utils/web-element.ts +30 -0
package/dist/src/ai/pilot.js
CHANGED
|
@@ -20,12 +20,18 @@ export class Pilot {
|
|
|
20
20
|
conversation = null;
|
|
21
21
|
researcher;
|
|
22
22
|
explorer;
|
|
23
|
+
stateManager;
|
|
24
|
+
requestStore;
|
|
25
|
+
playwrightRecorder;
|
|
23
26
|
fisherman = null;
|
|
24
|
-
constructor(
|
|
25
|
-
this.provider =
|
|
27
|
+
constructor(deps, agentTools, researcher) {
|
|
28
|
+
this.provider = deps.ai;
|
|
26
29
|
this.agentTools = agentTools;
|
|
27
30
|
this.researcher = researcher;
|
|
28
|
-
this.explorer = explorer;
|
|
31
|
+
this.explorer = deps.explorer;
|
|
32
|
+
this.stateManager = deps.stateManager;
|
|
33
|
+
this.requestStore = deps.requestStore;
|
|
34
|
+
this.playwrightRecorder = deps.playwrightRecorder;
|
|
29
35
|
}
|
|
30
36
|
setFisherman(fisherman) {
|
|
31
37
|
this.fisherman = fisherman;
|
|
@@ -71,7 +77,7 @@ export class Pilot {
|
|
|
71
77
|
let screenshotState = null;
|
|
72
78
|
if (type === 'finish' && this.provider.hasVision()) {
|
|
73
79
|
try {
|
|
74
|
-
screenshotState = await this.explorer.
|
|
80
|
+
screenshotState = await this.explorer.capture({ screenshot: true });
|
|
75
81
|
if (screenshotState.screenshot) {
|
|
76
82
|
visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || '';
|
|
77
83
|
}
|
|
@@ -130,7 +136,7 @@ export class Pilot {
|
|
|
130
136
|
const messages = [
|
|
131
137
|
{
|
|
132
138
|
role: 'system',
|
|
133
|
-
content: this.buildVerdictSystemPrompt(
|
|
139
|
+
content: this.buildVerdictSystemPrompt(task),
|
|
134
140
|
},
|
|
135
141
|
{ role: 'user', content: userContent },
|
|
136
142
|
];
|
|
@@ -148,7 +154,7 @@ export class Pilot {
|
|
|
148
154
|
tag('substep').log(`Pilot requesting verification: ${result.requestVerification}`);
|
|
149
155
|
const verifyResult = await navigator.verifyState(result.requestVerification, currentState).catch(() => null);
|
|
150
156
|
if (verifyResult?.verified && verifyResult.assertionSteps?.length) {
|
|
151
|
-
this.
|
|
157
|
+
this.playwrightRecorder.recordVerification(verifyResult.assertionSteps);
|
|
152
158
|
}
|
|
153
159
|
}
|
|
154
160
|
tag('info').log(`Pilot: ${result.decision} - ${result.reason}`);
|
|
@@ -258,10 +264,8 @@ export class Pilot {
|
|
|
258
264
|
return true;
|
|
259
265
|
}
|
|
260
266
|
}
|
|
261
|
-
buildSharedEvidenceRules(
|
|
267
|
+
buildSharedEvidenceRules() {
|
|
262
268
|
return dedent `
|
|
263
|
-
SCENARIO: ${task.scenario}
|
|
264
|
-
|
|
265
269
|
EVIDENCE PRIORITY (strict):
|
|
266
270
|
1) Final observable state proving the scenario goal
|
|
267
271
|
2) verify()/see() results in the LAST few actions before stop/finish
|
|
@@ -294,6 +298,11 @@ export class Pilot {
|
|
|
294
298
|
|
|
295
299
|
Expected results are MILESTONES, not the goal. Never fail because a milestone (toast, icon, styling)
|
|
296
300
|
didn't match if the scenario goal IS accomplished.
|
|
301
|
+
`;
|
|
302
|
+
}
|
|
303
|
+
buildTaskContext(task) {
|
|
304
|
+
return dedent `
|
|
305
|
+
SCENARIO: ${task.scenario}
|
|
297
306
|
|
|
298
307
|
${this.buildDeletionScope(task)}
|
|
299
308
|
|
|
@@ -307,7 +316,7 @@ export class Pilot {
|
|
|
307
316
|
iteration's work, but server-side side effects (records created, forms submitted) persist.
|
|
308
317
|
Unnecessary resets create duplicate data and infinite loops.
|
|
309
318
|
|
|
310
|
-
${this.buildSharedEvidenceRules(
|
|
319
|
+
${this.buildSharedEvidenceRules()}
|
|
311
320
|
|
|
312
321
|
DECISION:
|
|
313
322
|
- "allow": current page cannot host the scenario, irrecoverable error, or no path back.
|
|
@@ -324,16 +333,18 @@ export class Pilot {
|
|
|
324
333
|
to verify, how to record. Do not suggest repeating actions that already succeeded.
|
|
325
334
|
If progress is blocked only because the page lacks target data for the scenario, prefer precondition()
|
|
326
335
|
over repeated UI attempts.
|
|
336
|
+
|
|
337
|
+
${this.buildTaskContext(task)}
|
|
327
338
|
`;
|
|
328
339
|
}
|
|
329
|
-
buildVerdictSystemPrompt(
|
|
340
|
+
buildVerdictSystemPrompt(task) {
|
|
330
341
|
return dedent `
|
|
331
|
-
You are Pilot — final decision maker for test pass/fail.
|
|
332
|
-
|
|
342
|
+
You are Pilot — final decision maker for test pass/fail. Review the evidence and commit to a
|
|
343
|
+
verdict; "continue" only when evidence is genuinely insufficient.
|
|
333
344
|
|
|
334
345
|
${capabilityGroundingRule}
|
|
335
346
|
|
|
336
|
-
${this.buildSharedEvidenceRules(
|
|
347
|
+
${this.buildSharedEvidenceRules()}
|
|
337
348
|
|
|
338
349
|
DECISION:
|
|
339
350
|
- "pass": scenario goal is fully accomplished. Set requestVerification to a one-sentence claim about
|
|
@@ -352,6 +363,8 @@ export class Pilot {
|
|
|
352
363
|
reason field: one short sentence, maximum 120 characters. Do NOT restate the decision
|
|
353
364
|
("scenario goal achieved/not achieved"). State what happened: what was verified, what failed,
|
|
354
365
|
or what evidence was found.
|
|
366
|
+
|
|
367
|
+
${this.buildTaskContext(task)}
|
|
355
368
|
`;
|
|
356
369
|
}
|
|
357
370
|
async planTest(task, currentState) {
|
|
@@ -418,8 +431,6 @@ export class Pilot {
|
|
|
418
431
|
.filter((t) => t.wasSuccessful)
|
|
419
432
|
.slice(-this.stepsToReview);
|
|
420
433
|
const actionsContext = this.formatActions(toolCalls);
|
|
421
|
-
this.conversation.cleanupTag('page_summary', '...trimmed...', 1);
|
|
422
|
-
this.conversation.cleanupTag('recent_actions', '...trimmed...', 2);
|
|
423
434
|
return this.sendToPilot(dedent `
|
|
424
435
|
Navigated to new page.
|
|
425
436
|
START URL: ${task.startUrl}
|
|
@@ -439,7 +450,7 @@ export class Pilot {
|
|
|
439
450
|
${this.formatExpectations(task)}
|
|
440
451
|
|
|
441
452
|
First: evaluate whether this navigation makes sense for the scenario goal. If the page is unrelated, instruct Tester to back() or reset(). Then plan next steps.
|
|
442
|
-
`, 'pilot.reviewNewPage');
|
|
453
|
+
`, 'pilot.reviewNewPage', { task });
|
|
443
454
|
}
|
|
444
455
|
async analyzeProgress(task, currentState, testerConversation) {
|
|
445
456
|
tag('substep').log('Pilot analyzing progress...');
|
|
@@ -451,7 +462,6 @@ export class Pilot {
|
|
|
451
462
|
const toolCalls = testerConversation.getToolExecutions().slice(-this.stepsToReview);
|
|
452
463
|
const actionsContext = this.formatActions(toolCalls);
|
|
453
464
|
const stateContext = this.buildStateContext(currentState);
|
|
454
|
-
this.conversation.cleanupTag('recent_actions', '...trimmed...', 2);
|
|
455
465
|
const hasFailures = toolCalls.length === 0 || toolCalls.some((t) => !t.wasSuccessful);
|
|
456
466
|
const text = await this.sendToPilot(dedent `
|
|
457
467
|
START URL: ${task.startUrl}
|
|
@@ -479,7 +489,7 @@ export class Pilot {
|
|
|
479
489
|
const remaining = task.getRemainingExpectations();
|
|
480
490
|
return `CHECKED: ${checked.length > 0 ? checked.join(', ') : 'none'}\nREMAINING: ${remaining.length > 0 ? remaining.join(', ') : 'none'}`;
|
|
481
491
|
}
|
|
482
|
-
async sendToPilot(userText, functionId, opts
|
|
492
|
+
async sendToPilot(userText, functionId, opts) {
|
|
483
493
|
debugLog(`sendToPilot: ${functionId}, tools: ${!!opts.tools}, roundtrips: ${opts.maxToolRoundtrips ?? 0}`);
|
|
484
494
|
let finalUserText = userText;
|
|
485
495
|
if (opts.tools) {
|
|
@@ -489,17 +499,12 @@ export class Pilot {
|
|
|
489
499
|
}
|
|
490
500
|
}
|
|
491
501
|
this.conversation.addUserText(finalUserText);
|
|
492
|
-
|
|
493
|
-
if (opts.tools) {
|
|
494
|
-
tools = this.pickPlanningTools();
|
|
495
|
-
}
|
|
496
|
-
if (opts.tools && opts.task) {
|
|
497
|
-
tools = { ...tools, ...this.buildPreconditionTool(opts.task) };
|
|
498
|
-
}
|
|
502
|
+
const tools = { ...this.pickPlanningTools(), ...this.buildPreconditionTool(opts.task) };
|
|
499
503
|
const result = await this.provider.invokeConversation(this.conversation, tools, {
|
|
500
504
|
maxToolRoundtrips: opts.maxToolRoundtrips ?? 0,
|
|
505
|
+
toolChoice: opts.tools ? 'auto' : 'none',
|
|
501
506
|
agentName: 'pilot',
|
|
502
|
-
stopWhen:
|
|
507
|
+
stopWhen: () => opts.task.hasFinished,
|
|
503
508
|
telemetry: { functionId },
|
|
504
509
|
});
|
|
505
510
|
const text = result?.response?.text || '';
|
|
@@ -518,10 +523,10 @@ export class Pilot {
|
|
|
518
523
|
`;
|
|
519
524
|
}
|
|
520
525
|
getExperienceToc() {
|
|
521
|
-
const state = this.
|
|
526
|
+
const state = this.stateManager.getCurrentState();
|
|
522
527
|
if (!state)
|
|
523
528
|
return '';
|
|
524
|
-
return this.
|
|
529
|
+
return this.stateManager.getExperienceTracker().renderExperienceTocFor(ActionResult.fromState(state));
|
|
525
530
|
}
|
|
526
531
|
pickPlanningTools() {
|
|
527
532
|
const { see, context, verify, research, getVisitedStates, xpathCheck, learnExperience, askUser } = this.agentTools ?? {};
|
|
@@ -589,7 +594,7 @@ export class Pilot {
|
|
|
589
594
|
async checkDataAvailability(task, requestedData, fishermanReason) {
|
|
590
595
|
if (!this.provider.hasVision())
|
|
591
596
|
return null;
|
|
592
|
-
const screenshotState = await this.explorer.
|
|
597
|
+
const screenshotState = await this.explorer.capture({ screenshot: true }).catch(() => null);
|
|
593
598
|
if (!screenshotState?.screenshot)
|
|
594
599
|
return null;
|
|
595
600
|
const question = dedent `
|
|
@@ -635,8 +640,8 @@ export class Pilot {
|
|
|
635
640
|
else {
|
|
636
641
|
lines.push('modal: none');
|
|
637
642
|
}
|
|
638
|
-
|
|
639
|
-
|
|
643
|
+
const tabs = this.stateManager.otherTabs;
|
|
644
|
+
if (tabs.length > 0) {
|
|
640
645
|
lines.push(`other tabs: ${tabs.length} (${tabs.map((t) => `${t.url} - ${t.title}`).join(', ')})`);
|
|
641
646
|
}
|
|
642
647
|
else {
|
|
@@ -658,7 +663,7 @@ export class Pilot {
|
|
|
658
663
|
else {
|
|
659
664
|
lines.push('console errors: none');
|
|
660
665
|
}
|
|
661
|
-
const failedRequests = this.
|
|
666
|
+
const failedRequests = this.requestStore.getFailedRequests();
|
|
662
667
|
if (failedRequests.length > 0) {
|
|
663
668
|
const sample = failedRequests
|
|
664
669
|
.slice(-5)
|
|
@@ -733,7 +738,7 @@ export class Pilot {
|
|
|
733
738
|
}
|
|
734
739
|
formatSessionLog(testerConversation) {
|
|
735
740
|
const executions = testerConversation.getToolExecutions().filter((t) => !META_TOOLS.includes(t.toolName));
|
|
736
|
-
const stateHistory = this.
|
|
741
|
+
const stateHistory = this.stateManager.getStateHistory();
|
|
737
742
|
const initialUrl = stateHistory[0]?.toState?.url || '';
|
|
738
743
|
let currentUrl = initialUrl;
|
|
739
744
|
const groups = new Map();
|
|
@@ -875,16 +880,6 @@ export class Pilot {
|
|
|
875
880
|
return dedent `
|
|
876
881
|
You are Pilot — a supervisor that detects problems and intervenes only when needed.
|
|
877
882
|
|
|
878
|
-
SCENARIO: ${task.scenario}
|
|
879
|
-
START URL: ${initialState.url}
|
|
880
|
-
PAGE: ${initialState.title || ''} | ${initialState.h1 || ''}
|
|
881
|
-
|
|
882
|
-
EXPECTED RESULTS:
|
|
883
|
-
${task.expected.map((e) => `- ${e}`).join('\n')}
|
|
884
|
-
|
|
885
|
-
PLANNED STEPS:
|
|
886
|
-
${stepsText}
|
|
887
|
-
|
|
888
883
|
Your job: plan, review new pages, detect stuck patterns, suggest concrete next steps. Track which
|
|
889
884
|
expectations are checked. When things go well, encourage briefly and let Tester continue. The current
|
|
890
885
|
page is usually richer than the page summary lists — prefer exploring it before navigating away.
|
|
@@ -954,6 +949,16 @@ export class Pilot {
|
|
|
954
949
|
NEXT: <specific actionable instruction for Tester>
|
|
955
950
|
|
|
956
951
|
Keep user-facing reasons concise: one short sentence, maximum 120 characters, evidence only, no repeated verdict wording.
|
|
952
|
+
|
|
953
|
+
SCENARIO: ${task.scenario}
|
|
954
|
+
START URL: ${initialState.url}
|
|
955
|
+
PAGE: ${initialState.title || ''} | ${initialState.h1 || ''}
|
|
956
|
+
|
|
957
|
+
EXPECTED RESULTS:
|
|
958
|
+
${task.expected.map((e) => `- ${e}`).join('\n')}
|
|
959
|
+
|
|
960
|
+
PLANNED STEPS:
|
|
961
|
+
${stepsText}
|
|
957
962
|
`;
|
|
958
963
|
}
|
|
959
964
|
}
|
package/dist/src/ai/planner.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type Explorer from '../explorer.js';
|
|
|
4
4
|
import type { StateManager } from '../state-manager.js';
|
|
5
5
|
import { Suite } from '../suite.js';
|
|
6
6
|
import { Plan, Test } from '../test-plan.js';
|
|
7
|
-
import type { Agent } from './agent.js';
|
|
7
|
+
import type { Agent, AgentDeps } from './agent.js';
|
|
8
8
|
import { Conversation } from './conversation.js';
|
|
9
9
|
import type { Fisherman } from './fisherman.js';
|
|
10
10
|
import type { Provider } from './provider.js';
|
|
@@ -58,7 +58,7 @@ export declare class Planner extends PlannerBase implements Agent {
|
|
|
58
58
|
lastSuite: Suite | null;
|
|
59
59
|
researcher: Researcher;
|
|
60
60
|
fisherman: Fisherman | null;
|
|
61
|
-
constructor(
|
|
61
|
+
constructor(deps: AgentDeps, researcher: Researcher);
|
|
62
62
|
setFisherman(fisherman: Fisherman): void;
|
|
63
63
|
get sectionOrder(): string[];
|
|
64
64
|
getDefaultStartUrl(state: {
|
|
@@ -75,5 +75,6 @@ export declare class Planner extends PlannerBase implements Agent {
|
|
|
75
75
|
cleanExperienceFlows(text: string): string | null;
|
|
76
76
|
buildApproach(style?: string): string;
|
|
77
77
|
buildConversation(state: ActionResult, style?: string, parentPlan?: Plan, feature?: string): Promise<Conversation>;
|
|
78
|
+
getTasksMessage(): string;
|
|
78
79
|
}
|
|
79
80
|
export {};
|
package/dist/src/ai/planner.js
CHANGED
|
@@ -49,13 +49,13 @@ export class Planner extends PlannerBase {
|
|
|
49
49
|
lastSuite = null;
|
|
50
50
|
researcher;
|
|
51
51
|
fisherman = null;
|
|
52
|
-
constructor(
|
|
52
|
+
constructor(deps, researcher) {
|
|
53
53
|
super();
|
|
54
|
-
this.explorer = explorer;
|
|
55
|
-
this.provider =
|
|
54
|
+
this.explorer = deps.explorer;
|
|
55
|
+
this.provider = deps.ai;
|
|
56
56
|
this.researcher = researcher;
|
|
57
|
-
this.stateManager =
|
|
58
|
-
this.experienceTracker =
|
|
57
|
+
this.stateManager = deps.stateManager;
|
|
58
|
+
this.experienceTracker = deps.stateManager.getExperienceTracker();
|
|
59
59
|
}
|
|
60
60
|
setFisherman(fisherman) {
|
|
61
61
|
this.fisherman = fisherman;
|
|
@@ -334,15 +334,9 @@ export class Planner extends PlannerBase {
|
|
|
334
334
|
${dataProtectionRules}
|
|
335
335
|
${fileUploadRule}
|
|
336
336
|
</rules>
|
|
337
|
-
|
|
338
|
-
${this.buildApproach(style)}
|
|
339
|
-
|
|
340
|
-
<context>
|
|
341
|
-
URL: ${state.url || 'Unknown'}
|
|
342
|
-
Title: ${state.title || 'Unknown'}
|
|
343
|
-
</context>
|
|
344
337
|
`;
|
|
345
338
|
conversation.addUserText(planningPrompt);
|
|
339
|
+
conversation.addUserText(this.getTasksMessage());
|
|
346
340
|
const currentState = this.stateManager.getCurrentState();
|
|
347
341
|
const research = await this.researcher.research(currentState || state, {
|
|
348
342
|
deep: true,
|
|
@@ -373,6 +367,14 @@ export class Planner extends PlannerBase {
|
|
|
373
367
|
|
|
374
368
|
${plannerResearch}
|
|
375
369
|
</page_research>
|
|
370
|
+
`);
|
|
371
|
+
conversation.addUserText(dedent `
|
|
372
|
+
${this.buildApproach(style)}
|
|
373
|
+
|
|
374
|
+
<context>
|
|
375
|
+
URL: ${state.url || 'Unknown'}
|
|
376
|
+
Title: ${state.title || 'Unknown'}
|
|
377
|
+
</context>
|
|
376
378
|
`);
|
|
377
379
|
if (this.fisherman) {
|
|
378
380
|
await this.fisherman.ensureReady(state.url);
|
|
@@ -517,8 +519,10 @@ export class Planner extends PlannerBase {
|
|
|
517
519
|
</parent_page_context>
|
|
518
520
|
`);
|
|
519
521
|
}
|
|
520
|
-
|
|
521
|
-
|
|
522
|
+
return conversation;
|
|
523
|
+
}
|
|
524
|
+
getTasksMessage() {
|
|
525
|
+
return dedent `
|
|
522
526
|
<task>
|
|
523
527
|
Provide testing scenarios as structured data with the following requirements:
|
|
524
528
|
1. Create a short, descriptive plan name that summarizes what will be tested (e.g., "User Authentication Testing", "Product Catalog Navigation", "Form Validation Tests")
|
|
@@ -564,10 +568,8 @@ export class Planner extends PlannerBase {
|
|
|
564
568
|
- Do not wrap text in ** or * quotes, ( or ) brackets.
|
|
565
569
|
- Avoid using emojis or special characters.
|
|
566
570
|
7. Only tests that can be tested from web UI should be proposed.
|
|
567
|
-
|
|
571
|
+
8. For a fresh plan propose at least ${this.MIN_TASKS} tests; when expanding an existing plan return ONLY new scenarios not in the tested list, and an empty array if none remain.
|
|
568
572
|
</task>
|
|
569
573
|
`;
|
|
570
|
-
conversation.addUserText(tasksMessage);
|
|
571
|
-
return conversation;
|
|
572
574
|
}
|
|
573
575
|
}
|
package/dist/src/ai/provider.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ActionResult } from '../action-result.js';
|
|
2
|
+
import type Explorer from '../explorer.js';
|
|
2
3
|
import type { StateManager, StateTransition, WebPageState } from '../state-manager.js';
|
|
3
4
|
import type { Task } from '../test-plan.js';
|
|
4
5
|
import type { Conversation, ToolExecution } from './conversation.js';
|
|
@@ -9,12 +10,12 @@ export declare class Quartermaster {
|
|
|
9
10
|
outputDir: string;
|
|
10
11
|
pageAnalyses: Map<string, PageAnalysis>;
|
|
11
12
|
unsubscribe: (() => void) | null;
|
|
12
|
-
|
|
13
|
+
explorer: Explorer | null;
|
|
13
14
|
pendingAnalyses: Promise<void>[];
|
|
14
15
|
constructor(provider: Provider, options?: {
|
|
15
16
|
model?: any;
|
|
16
17
|
});
|
|
17
|
-
start(
|
|
18
|
+
start(explorer: Explorer, stateManager: StateManager): void;
|
|
18
19
|
stop(): void;
|
|
19
20
|
onPageChange(event: StateTransition): void;
|
|
20
21
|
runAxeAnalysis(state: WebPageState): Promise<void>;
|
|
@@ -11,15 +11,15 @@ export class Quartermaster {
|
|
|
11
11
|
outputDir;
|
|
12
12
|
pageAnalyses = new Map();
|
|
13
13
|
unsubscribe = null;
|
|
14
|
-
|
|
14
|
+
explorer = null;
|
|
15
15
|
pendingAnalyses = [];
|
|
16
16
|
constructor(provider, options) {
|
|
17
17
|
this.provider = provider;
|
|
18
18
|
this.model = options?.model;
|
|
19
19
|
this.outputDir = outputPath('a11y');
|
|
20
20
|
}
|
|
21
|
-
start(
|
|
22
|
-
this.
|
|
21
|
+
start(explorer, stateManager) {
|
|
22
|
+
this.explorer = explorer;
|
|
23
23
|
mkdirSync(this.outputDir, { recursive: true });
|
|
24
24
|
this.unsubscribe = stateManager.onStateChange((event) => {
|
|
25
25
|
this.onPageChange(event);
|
|
@@ -45,7 +45,7 @@ export class Quartermaster {
|
|
|
45
45
|
this.pendingAnalyses.push(analysisPromise);
|
|
46
46
|
}
|
|
47
47
|
async runAxeAnalysis(state) {
|
|
48
|
-
const page = this.
|
|
48
|
+
const page = this.explorer?.page;
|
|
49
49
|
if (!page || !state.hash) {
|
|
50
50
|
debugLog('No page available for axe analysis');
|
|
51
51
|
return;
|
|
@@ -1,16 +1,10 @@
|
|
|
1
|
-
import type { ExperienceTracker } from '../experience-tracker.js';
|
|
2
|
-
import type Explorer from '../explorer.js';
|
|
3
|
-
import type { KnowledgeTracker } from '../knowledge-tracker.js';
|
|
4
1
|
import { Test } from '../test-plan.js';
|
|
5
|
-
import type { Agent } from './agent.js';
|
|
2
|
+
import type { Agent, AgentDeps } from './agent.js';
|
|
6
3
|
import type { Navigator } from './navigator.js';
|
|
7
|
-
import { Provider } from './provider.js';
|
|
8
4
|
import { TaskAgent } from './task-agent.js';
|
|
9
5
|
export declare class Rerunner extends TaskAgent implements Agent {
|
|
10
6
|
readonly ACTION_TOOLS: string[];
|
|
11
7
|
emoji: string;
|
|
12
|
-
explorer: Explorer;
|
|
13
|
-
provider: Provider;
|
|
14
8
|
agentTools: any;
|
|
15
9
|
healedSteps: Array<{
|
|
16
10
|
original: string;
|
|
@@ -18,11 +12,8 @@ export declare class Rerunner extends TaskAgent implements Agent {
|
|
|
18
12
|
}>;
|
|
19
13
|
traceDir: string;
|
|
20
14
|
static pluginsWired: boolean;
|
|
21
|
-
constructor(
|
|
15
|
+
constructor(deps: AgentDeps, agentTools?: any);
|
|
22
16
|
getNavigator(): Navigator;
|
|
23
|
-
getExperienceTracker(): ExperienceTracker;
|
|
24
|
-
getKnowledgeTracker(): KnowledgeTracker;
|
|
25
|
-
getProvider(): Provider;
|
|
26
17
|
get rerunnerConfig(): Record<string, any>;
|
|
27
18
|
get healLimit(): number;
|
|
28
19
|
get healMaxIterations(): number;
|
package/dist/src/ai/rerunner.js
CHANGED
|
@@ -26,32 +26,19 @@ const debugLog = createDebug('explorbot:rerunner');
|
|
|
26
26
|
export class Rerunner extends TaskAgent {
|
|
27
27
|
ACTION_TOOLS = ['click', 'pressKey', 'form'];
|
|
28
28
|
emoji = '🔄';
|
|
29
|
-
explorer;
|
|
30
|
-
provider;
|
|
31
29
|
agentTools;
|
|
32
30
|
healedSteps = [];
|
|
33
31
|
traceDir = '';
|
|
34
32
|
static pluginsWired = false;
|
|
35
|
-
constructor(
|
|
36
|
-
super();
|
|
37
|
-
this.explorer = explorer;
|
|
38
|
-
this.provider = provider;
|
|
33
|
+
constructor(deps, agentTools) {
|
|
34
|
+
super(deps);
|
|
39
35
|
this.agentTools = agentTools;
|
|
40
36
|
}
|
|
41
37
|
getNavigator() {
|
|
42
38
|
throw new Error('Rerunner does not use Navigator');
|
|
43
39
|
}
|
|
44
|
-
getExperienceTracker() {
|
|
45
|
-
return this.explorer.getStateManager().getExperienceTracker();
|
|
46
|
-
}
|
|
47
|
-
getKnowledgeTracker() {
|
|
48
|
-
return this.explorer.getKnowledgeTracker();
|
|
49
|
-
}
|
|
50
|
-
getProvider() {
|
|
51
|
-
return this.provider;
|
|
52
|
-
}
|
|
53
40
|
get rerunnerConfig() {
|
|
54
|
-
return this.
|
|
41
|
+
return this.config.ai?.agents?.rerunner || {};
|
|
55
42
|
}
|
|
56
43
|
get healLimit() {
|
|
57
44
|
return this.rerunnerConfig.healLimit ?? 3;
|
|
@@ -282,7 +269,7 @@ export class Rerunner extends TaskAgent {
|
|
|
282
269
|
},
|
|
283
270
|
});
|
|
284
271
|
const healTask = new Task(`Heal: ${failedCode}`);
|
|
285
|
-
const codeceptTools = createCodeceptJSTools(this.
|
|
272
|
+
const codeceptTools = createCodeceptJSTools(this.toolDeps, healTask);
|
|
286
273
|
let healed = false;
|
|
287
274
|
let healedCommand = '';
|
|
288
275
|
const tools = {
|
|
@@ -300,9 +287,9 @@ export class Rerunner extends TaskAgent {
|
|
|
300
287
|
healTask.addNote(note);
|
|
301
288
|
tag('substep').log(note);
|
|
302
289
|
}
|
|
303
|
-
const action = this.explorer.
|
|
290
|
+
const action = this.explorer.action();
|
|
304
291
|
await action.execute(`I.wait(${seconds})`);
|
|
305
|
-
const state = this.
|
|
292
|
+
const state = this.stateManager.getCurrentState();
|
|
306
293
|
const ar = state ? ActionResult.fromState(state) : null;
|
|
307
294
|
return {
|
|
308
295
|
success: true,
|
|
@@ -383,8 +370,8 @@ export class Rerunner extends TaskAgent {
|
|
|
383
370
|
};
|
|
384
371
|
}
|
|
385
372
|
getHealSystemPrompt() {
|
|
386
|
-
const customRules = this.provider.getSystemPromptForAgent('rerunner', this.
|
|
387
|
-
const currentUrl = this.
|
|
373
|
+
const customRules = this.provider.getSystemPromptForAgent('rerunner', this.stateManager.getCurrentState()?.url) || '';
|
|
374
|
+
const currentUrl = this.stateManager.getCurrentState()?.url || '';
|
|
388
375
|
const approach = RulesLoader.loadRules('rerunner', ['healing-approach'], currentUrl);
|
|
389
376
|
return dedent `
|
|
390
377
|
<role>
|
|
@@ -417,7 +404,7 @@ export class Rerunner extends TaskAgent {
|
|
|
417
404
|
`;
|
|
418
405
|
}
|
|
419
406
|
getHealUserPrompt(failedCode, error) {
|
|
420
|
-
const state = this.
|
|
407
|
+
const state = this.stateManager.getCurrentState();
|
|
421
408
|
const actionResult = state ? ActionResult.fromState(state) : null;
|
|
422
409
|
const headings = formatHeadings(state || {});
|
|
423
410
|
return dedent `
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import dedent from 'dedent';
|
|
2
2
|
import { tag } from '../../utils/logger.js';
|
|
3
3
|
import { mdq } from "../../utils/markdown-query.js";
|
|
4
|
+
import { eidxByLocator } from "../../utils/web-eidx.js";
|
|
4
5
|
import { WebElement } from "../../utils/web-element.js";
|
|
5
6
|
import { debugLog } from "./mixin.js";
|
|
6
7
|
import { parseResearchSections } from "./parser.js";
|
|
@@ -67,7 +68,7 @@ export function WithCoordinates(Base) {
|
|
|
67
68
|
return this._analyzeScreenshotForVisualProps();
|
|
68
69
|
}
|
|
69
70
|
async visuallyAnnotateElements(opts) {
|
|
70
|
-
return this.explorer.
|
|
71
|
+
return this.explorer.withPage((page) => visuallyAnnotateContainers(page, opts?.containers || []));
|
|
71
72
|
}
|
|
72
73
|
async _analyzeScreenshotForVisualProps() {
|
|
73
74
|
const elements = new Map();
|
|
@@ -160,7 +161,7 @@ export function WithCoordinates(Base) {
|
|
|
160
161
|
if (!eidx) {
|
|
161
162
|
const locator = el.css || el.xpath || (el.aria ? `role=${el.aria.role}[name="${el.aria.text}"]` : null);
|
|
162
163
|
if (locator)
|
|
163
|
-
eidx = await this.explorer.
|
|
164
|
+
eidx = await this.explorer.withPage((page) => eidxByLocator(page, locator, section.containerCss));
|
|
164
165
|
}
|
|
165
166
|
if (!eidx)
|
|
166
167
|
continue;
|
|
@@ -187,7 +188,7 @@ export function WithCoordinates(Base) {
|
|
|
187
188
|
}
|
|
188
189
|
if (eidxWithoutCoords.length === 0)
|
|
189
190
|
return;
|
|
190
|
-
const webElements = await this.explorer.
|
|
191
|
+
const webElements = await this.explorer.withPage((page) => WebElement.fromEidxList(page, eidxWithoutCoords));
|
|
191
192
|
if (webElements.length === 0)
|
|
192
193
|
return;
|
|
193
194
|
const rectMap = new Map(webElements.map((w) => [w.eidx, w]));
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ActionResult, type Diff } from '../../action-result.js';
|
|
2
|
+
import type { ExplorbotConfig } from '../../config.js';
|
|
2
3
|
import type Explorer from '../../explorer.js';
|
|
3
4
|
import type { StateManager } from '../../state-manager.js';
|
|
4
5
|
import { WebPageState } from '../../state-manager.js';
|
|
@@ -11,6 +12,7 @@ export declare function WithDeepAnalysis<T extends Constructor>(Base: T): {
|
|
|
11
12
|
explorer: Explorer;
|
|
12
13
|
provider: Provider;
|
|
13
14
|
stateManager: StateManager;
|
|
15
|
+
config: ExplorbotConfig;
|
|
14
16
|
actionResult: ActionResult | undefined;
|
|
15
17
|
performDeepAnalysis(state: WebPageState, result: ResearchResult): Promise<void>;
|
|
16
18
|
researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise<string | null>;
|
|
@@ -14,7 +14,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
14
14
|
async performDeepAnalysis(state, result) {
|
|
15
15
|
tag('info').log('Starting deep analysis of expandable elements');
|
|
16
16
|
await this.navigateTo(state.fullUrl || state.url);
|
|
17
|
-
const maxClicks = this.
|
|
17
|
+
const maxClicks = this.config.ai?.agents?.researcher?.maxExpandableClicks ?? DEFAULT_MAX_EXPANDABLE_CLICKS;
|
|
18
18
|
const expandedSections = [];
|
|
19
19
|
const navigationLinks = [];
|
|
20
20
|
let verifiedCodes = [];
|
|
@@ -311,9 +311,9 @@ export function WithDeepAnalysis(Base) {
|
|
|
311
311
|
const isCoordinateClick = el.commands[0].startsWith('I.clickXY(');
|
|
312
312
|
if (!isCoordinateClick) {
|
|
313
313
|
const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo(');
|
|
314
|
-
await this.explorer.
|
|
314
|
+
await this.explorer.action().attempt(hoverCmd);
|
|
315
315
|
await new Promise((r) => setTimeout(r, 500));
|
|
316
|
-
await this.explorer.
|
|
316
|
+
await this.explorer.capture();
|
|
317
317
|
const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState());
|
|
318
318
|
const hoverDiff = await hoverAR.diff(previousState);
|
|
319
319
|
const hoverHtmlSize = hoverDiff.htmlParts.reduce((sum, p) => sum + p.subtree.length, 0);
|
|
@@ -351,7 +351,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
351
351
|
async _executeAndAnalyze(commands, description, state, originalAria, alreadyExpanded) {
|
|
352
352
|
const previousState = ActionResult.fromState(this.stateManager.getCurrentState());
|
|
353
353
|
let clickCode = null;
|
|
354
|
-
const action = this.explorer.
|
|
354
|
+
const action = this.explorer.action();
|
|
355
355
|
for (const cmd of commands) {
|
|
356
356
|
if (await action.attempt(cmd, undefined)) {
|
|
357
357
|
clickCode = cmd;
|
|
@@ -365,7 +365,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
365
365
|
await new Promise((r) => setTimeout(r, 500));
|
|
366
366
|
let diff;
|
|
367
367
|
try {
|
|
368
|
-
await this.explorer.
|
|
368
|
+
await this.explorer.capture();
|
|
369
369
|
const currAR = ActionResult.fromState(this.stateManager.getCurrentState());
|
|
370
370
|
diff = await currAR.diff(previousState);
|
|
371
371
|
}
|
|
@@ -394,7 +394,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
394
394
|
async _restorePageState(url, originalAria) {
|
|
395
395
|
try {
|
|
396
396
|
await this.cancelInUi();
|
|
397
|
-
await this.explorer.
|
|
397
|
+
await this.explorer.capture();
|
|
398
398
|
const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || '';
|
|
399
399
|
if (!diffAriaSnapshots(originalAria, currentAria).text)
|
|
400
400
|
return;
|
|
@@ -12,9 +12,13 @@ export declare function WithLocators<T extends Constructor>(Base: T): {
|
|
|
12
12
|
explorer: Explorer;
|
|
13
13
|
provider: Provider;
|
|
14
14
|
actionResult: ActionResult | undefined;
|
|
15
|
-
testLocators(locators: Locator[]
|
|
15
|
+
testLocators(locators: Locator[], opts?: {
|
|
16
|
+
scope?: boolean;
|
|
17
|
+
}): Promise<void>;
|
|
16
18
|
fixBrokenSections(result: ResearchResult, conversation: Conversation): Promise<void>;
|
|
17
19
|
backfillBrokenLocators(result: ResearchResult): Promise<void>;
|
|
20
|
+
resolveContainers(result: ResearchResult): Promise<Locator[]>;
|
|
21
|
+
recoverContainerFromChildren(result: ResearchResult, css: string): Promise<boolean>;
|
|
18
22
|
validateContainers(result: ResearchResult): Promise<void>;
|
|
19
23
|
updateSectionContainer(result: ResearchResult, section: ReturnType<typeof parseResearchSections>[0], newCss: string | null): void;
|
|
20
24
|
};
|
|
@@ -30,7 +34,10 @@ export interface Locator {
|
|
|
30
34
|
pwLocator: string | null;
|
|
31
35
|
}
|
|
32
36
|
export interface LocatorMethods {
|
|
33
|
-
testLocators(locators: Locator[]
|
|
37
|
+
testLocators(locators: Locator[], opts?: {
|
|
38
|
+
scope?: boolean;
|
|
39
|
+
}): Promise<void>;
|
|
34
40
|
fixBrokenSections(result: ResearchResult, conversation: Conversation): Promise<void>;
|
|
35
41
|
backfillBrokenLocators(result: ResearchResult): Promise<void>;
|
|
42
|
+
resolveContainers(result: ResearchResult): Promise<Locator[]>;
|
|
36
43
|
}
|