explorbot 0.1.20 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/explorbot-cli.ts +54 -25
- package/boat/doc-collector/src/ai/documentarian.ts +259 -84
- package/boat/doc-collector/src/ai/tools.ts +544 -0
- package/boat/doc-collector/src/cli.ts +1 -0
- package/boat/doc-collector/src/config.ts +2 -0
- package/boat/doc-collector/src/docbot.ts +64 -5
- package/boat/doc-collector/src/docs-renderer.ts +56 -2
- package/dist/bin/explorbot-cli.js +30 -4
- package/dist/boat/doc-collector/src/ai/documentarian.js +220 -71
- package/dist/boat/doc-collector/src/ai/tools.js +415 -0
- package/dist/boat/doc-collector/src/cli.js +1 -0
- package/dist/boat/doc-collector/src/config.js +1 -0
- package/dist/boat/doc-collector/src/docbot.js +57 -5
- package/dist/boat/doc-collector/src/docs-renderer.js +46 -0
- package/dist/package.json +3 -3
- package/dist/src/action-result.js +12 -0
- package/dist/src/ai/navigator.js +179 -43
- package/dist/src/ai/pilot.js +6 -0
- package/dist/src/commands/index.js +2 -0
- package/dist/src/commands/init-command.js +9 -0
- package/dist/src/commands/plans-command.js +83 -0
- package/dist/src/commands/test-command.js +14 -8
- package/dist/src/playwright-recorder.js +19 -5
- package/dist/src/reporter.js +5 -7
- package/package.json +3 -3
- package/src/action-result.ts +11 -0
- package/src/ai/navigator.ts +183 -46
- package/src/ai/pilot.ts +6 -0
- package/src/commands/index.ts +2 -0
- package/src/commands/init-command.ts +9 -0
- package/src/commands/plans-command.ts +99 -0
- package/src/commands/test-command.ts +15 -8
- package/src/config.ts +2 -0
- package/src/playwright-recorder.ts +19 -5
- package/src/reporter.ts +4 -7
package/src/ai/navigator.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { tool } from 'ai';
|
|
1
2
|
import dedent from 'dedent';
|
|
3
|
+
import { z } from 'zod';
|
|
2
4
|
import { ActionResult } from '../action-result.js';
|
|
3
5
|
import type Action from '../action.ts';
|
|
4
6
|
import { ExperienceTracker, renderExperienceToc } from '../experience-tracker.js';
|
|
@@ -80,6 +82,14 @@ class Navigator implements Agent {
|
|
|
80
82
|
this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
|
|
81
83
|
}
|
|
82
84
|
|
|
85
|
+
private get verifyAttempts(): number {
|
|
86
|
+
return this.explorer.getConfig().ai?.agents?.navigator?.verifyAttempts ?? 3;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
private get verifyTimeout(): number {
|
|
90
|
+
return this.explorer.getConfig().ai?.agents?.navigator?.verifyTimeout ?? 1500;
|
|
91
|
+
}
|
|
92
|
+
|
|
83
93
|
private getBaseOrigin(): string | null {
|
|
84
94
|
const baseUrl = this.explorer.getConfig().playwright.url;
|
|
85
95
|
try {
|
|
@@ -238,14 +248,34 @@ class Navigator implements Agent {
|
|
|
238
248
|
const conversation = this.provider.startConversation(this.systemPrompt, 'navigator');
|
|
239
249
|
conversation.addUserText(prompt);
|
|
240
250
|
|
|
241
|
-
|
|
251
|
+
let stopReason: string | null = null;
|
|
252
|
+
const tools = {
|
|
253
|
+
stop: tool({
|
|
254
|
+
description: dedent`
|
|
255
|
+
Stop the navigation because no locator or strategy change can reach the goal.
|
|
256
|
+
Use this when reaching the goal requires something only the user can supply or that the
|
|
257
|
+
page cannot grant from the current state — for example: an authentication failure you
|
|
258
|
+
cannot guess past, a captcha or human-verification step, a permission the test cannot
|
|
259
|
+
satisfy, a piece of data not present in the available knowledge / hint context, or a
|
|
260
|
+
blocking error or dialog you cannot dismiss.
|
|
261
|
+
Do NOT use this for locator or strategy problems — for those, emit new code blocks instead.
|
|
262
|
+
`,
|
|
263
|
+
inputSchema: z.object({
|
|
264
|
+
reason: z.string().describe('Short user-facing explanation. Quote what you observed (alert text, dialog title, status message, validation note) and name what is missing or required.'),
|
|
265
|
+
}),
|
|
266
|
+
execute: async ({ reason }) => {
|
|
267
|
+
stopReason = reason;
|
|
268
|
+
return { success: true, message: 'Recorded. Navigator will stop and surface the reason.' };
|
|
269
|
+
},
|
|
270
|
+
}),
|
|
271
|
+
};
|
|
242
272
|
|
|
243
273
|
let codeBlocks: string[] = [];
|
|
244
274
|
let htmlContextAdded = false;
|
|
245
275
|
let codeBlockIndex = 0;
|
|
246
276
|
let totalAttempts = 0;
|
|
247
277
|
const progressBlocks: string[] = [];
|
|
248
|
-
const batchFailures: Array<{ code: string; error: string }> = [];
|
|
278
|
+
const batchFailures: Array<{ code: string; error: string; ariaChanges?: string | null; urlAfter?: string }> = [];
|
|
249
279
|
|
|
250
280
|
let resolved = false;
|
|
251
281
|
await loop(
|
|
@@ -253,6 +283,12 @@ class Navigator implements Agent {
|
|
|
253
283
|
if (codeBlocks.length === 0) {
|
|
254
284
|
const result = await this.provider.invokeConversation(conversation, tools);
|
|
255
285
|
if (!result) return;
|
|
286
|
+
if (stopReason) {
|
|
287
|
+
tag('error').log(`Navigator stopped: ${stopReason}`);
|
|
288
|
+
resolved = false;
|
|
289
|
+
stop();
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
256
292
|
const aiResponse = result?.response?.text;
|
|
257
293
|
debugLog('AI:', aiResponse?.split('\n')[0]);
|
|
258
294
|
debugLog('Received AI response:', aiResponse?.length ?? 0, 'characters');
|
|
@@ -274,14 +310,38 @@ class Navigator implements Agent {
|
|
|
274
310
|
tag('substep').log('Feeding failures back to AI for a new batch...');
|
|
275
311
|
let contextMsg = 'Previous solutions did not work. Analyze the failures and try DIFFERENT strategies (not syntactic variants of the same locator).\n\n';
|
|
276
312
|
if (batchFailures.length > 0) {
|
|
277
|
-
const lines = batchFailures
|
|
313
|
+
const lines = batchFailures
|
|
314
|
+
.map((f) => {
|
|
315
|
+
const head = `- \`${f.code.split('\n')[0]}\` → ${f.error}`;
|
|
316
|
+
if (!f.ariaChanges) return head;
|
|
317
|
+
const trimmed = f.ariaChanges.split('\n').slice(0, 12).join('\n ');
|
|
318
|
+
return `${head}\n • ARIA changes after the action:\n ${trimmed}`;
|
|
319
|
+
})
|
|
320
|
+
.join('\n');
|
|
278
321
|
contextMsg += `<previous_failures>\n${lines}\n</previous_failures>\n\n`;
|
|
279
322
|
}
|
|
280
323
|
if (!htmlContextAdded) {
|
|
281
324
|
htmlContextAdded = true;
|
|
282
325
|
contextMsg += `Full HTML context:\n\n<page_html>\n${await actionResult.combinedHtml()}\n</page_html>\n\n`;
|
|
283
326
|
}
|
|
284
|
-
|
|
327
|
+
const pageReacted = batchFailures.some((f) => f.ariaChanges);
|
|
328
|
+
if (pageReacted) {
|
|
329
|
+
contextMsg += dedent`
|
|
330
|
+
Some steps in the previous batch did not throw, but the URL did not change to the expected target and the page changed in other ways — the ARIA diff for each such step is listed in <previous_failures> above.
|
|
331
|
+
|
|
332
|
+
Read those diffs and judge what each step actually triggered. Different action types produce different reactions; the diff is your only evidence of what happened. A diff might show, for example: a new alert / alertdialog / status / validation message appearing near a field or at page level; a modal, dialog, or wizard step opening; a banner, toast, or notification region appearing; a section expanding or collapsing; a tab or accordion switching content. A diff might also be empty or unrelated to the step — that is also a signal.
|
|
333
|
+
|
|
334
|
+
Choose exactly ONE path based on what the diffs actually show — do not assume the previous step submitted any particular kind of data:
|
|
335
|
+
|
|
336
|
+
A. The diff indicates the application requires something only the user can supply — for example: an authentication failure you cannot guess past, a captcha, a permission the test cannot satisfy, or knowledge that is not present in the provided context. Call the stop() tool and quote what you saw in the diff and what is needed.
|
|
337
|
+
|
|
338
|
+
B. The diff indicates the next step is something you can perform from the existing knowledge / hint context — for example: re-emit a step with a value that exists in the knowledge but was used incorrectly; dismiss an unexpected modal; accept a confirmation; take a follow-up step the page now requires. Emit code blocks for that next step. Do NOT change the locator of a step that already produced a reaction.
|
|
339
|
+
|
|
340
|
+
C. The diff is empty or unrelated to your step — the action likely missed its target. Propose a different locator strategy.
|
|
341
|
+
`;
|
|
342
|
+
} else {
|
|
343
|
+
contextMsg += 'Propose new solutions. If errors mention "intercepts pointer events" or timeouts on visible elements, an overlay is blocking — dismiss it first (Escape, click outside, Close button) before retrying the original action.';
|
|
344
|
+
}
|
|
285
345
|
conversation.addUserText(contextMsg);
|
|
286
346
|
codeBlocks = [];
|
|
287
347
|
batchFailures.length = 0;
|
|
@@ -292,7 +352,8 @@ class Navigator implements Agent {
|
|
|
292
352
|
|
|
293
353
|
await this.explorer.switchToMainFrame();
|
|
294
354
|
|
|
295
|
-
const
|
|
355
|
+
const prevActionResult = action.actionResult ?? actionResult;
|
|
356
|
+
const prevHash = prevActionResult.getStateHash();
|
|
296
357
|
|
|
297
358
|
debugLog(`Attempting resolution: ${codeBlock}`);
|
|
298
359
|
const attemptOk = await action.attempt(codeBlock, message);
|
|
@@ -328,6 +389,22 @@ class Navigator implements Agent {
|
|
|
328
389
|
resolved = urlMatches && stateChanged;
|
|
329
390
|
|
|
330
391
|
if (!resolved && attemptOk) {
|
|
392
|
+
let ariaChanges: string | null = null;
|
|
393
|
+
if (freshState.getStateHash() !== prevHash) {
|
|
394
|
+
try {
|
|
395
|
+
const diff = await freshState.diff(prevActionResult);
|
|
396
|
+
await diff.calculate();
|
|
397
|
+
ariaChanges = diff.ariaChanged;
|
|
398
|
+
} catch (err) {
|
|
399
|
+
debugLog('Failed to compute pageDiff for failed URL verification:', err);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
batchFailures.push({
|
|
403
|
+
code: codeBlock,
|
|
404
|
+
error: `URL did not change (still ${freshState.url})`,
|
|
405
|
+
ariaChanges,
|
|
406
|
+
urlAfter: freshState.url,
|
|
407
|
+
});
|
|
331
408
|
tag('warning').log(`URL verification failed: expected ${expectedUrl}, got ${freshState.url}`);
|
|
332
409
|
}
|
|
333
410
|
if (freshState.getStateHash() !== prevHash && (attemptOk || urlMatches)) {
|
|
@@ -380,12 +457,15 @@ class Navigator implements Agent {
|
|
|
380
457
|
}
|
|
381
458
|
}
|
|
382
459
|
|
|
383
|
-
if (!resolved &&
|
|
460
|
+
if (!resolved && stopReason) {
|
|
461
|
+
tag('error').log(`Navigator stopped: ${stopReason}`);
|
|
462
|
+
} else if (!resolved && totalAttempts > 0) {
|
|
384
463
|
tag('error').log(`Navigation failed after ${totalAttempts} attempts`);
|
|
385
464
|
}
|
|
386
465
|
|
|
387
466
|
if (!resolved && isInteractive()) {
|
|
388
|
-
const
|
|
467
|
+
const stopLine = stopReason ? `Navigator stopped: ${stopReason}\n` : '';
|
|
468
|
+
const userInput = await pause(`${stopLine}Navigator failed to resolve. Current: ${action.stateManager.getCurrentState()?.url}\n` + `Target: ${expectedUrl ?? '(none)'}\nEnter CodeceptJS commands (or press Enter to skip):`);
|
|
389
469
|
|
|
390
470
|
if (userInput?.trim()) {
|
|
391
471
|
resolved = await action.attempt(userInput, message);
|
|
@@ -551,6 +631,12 @@ class Navigator implements Agent {
|
|
|
551
631
|
tag('info').log('AI Navigator verifying state at', actionResult.url);
|
|
552
632
|
debugLog('Verification message:', message);
|
|
553
633
|
|
|
634
|
+
const cachedVerification = actionResult.getVerification(message);
|
|
635
|
+
if (cachedVerification !== null) {
|
|
636
|
+
tag('substep').log(`Reusing cached verification: ${cachedVerification ? 'PASS' : 'FAIL'}`);
|
|
637
|
+
return { verified: cachedVerification, successfulCodes: [], assertionSteps: [], totalAttempted: 0 };
|
|
638
|
+
}
|
|
639
|
+
|
|
554
640
|
let knowledge = '';
|
|
555
641
|
let experience = '';
|
|
556
642
|
|
|
@@ -573,6 +659,21 @@ class Navigator implements Agent {
|
|
|
573
659
|
}
|
|
574
660
|
}
|
|
575
661
|
|
|
662
|
+
const priorVerifications = Object.entries(actionResult.verifications ?? {});
|
|
663
|
+
let verificationContext = '';
|
|
664
|
+
if (priorVerifications.length > 0) {
|
|
665
|
+
const lines = priorVerifications.map(([claim, passed]) => `- "${claim}" → ${passed ? 'passed' : 'failed'}`).join('\n');
|
|
666
|
+
verificationContext = dedent`
|
|
667
|
+
<already_verified>
|
|
668
|
+
These claims were already checked on this page:
|
|
669
|
+
${lines}
|
|
670
|
+
|
|
671
|
+
If the claim to verify has the same meaning as one above (even if worded differently), do NOT write any assertion code.
|
|
672
|
+
Respond with a single line and nothing else: ALREADY_VERIFIED: <exact text of the matching claim>
|
|
673
|
+
</already_verified>
|
|
674
|
+
`;
|
|
675
|
+
}
|
|
676
|
+
|
|
576
677
|
const prompt = dedent`
|
|
577
678
|
<message>
|
|
578
679
|
${message}
|
|
@@ -586,11 +687,13 @@ class Navigator implements Agent {
|
|
|
586
687
|
</page_html>
|
|
587
688
|
</page>
|
|
588
689
|
|
|
690
|
+
${verificationContext}
|
|
691
|
+
|
|
589
692
|
<task>
|
|
590
693
|
Identify what assertion the user wants to verify on the page.
|
|
591
|
-
Propose
|
|
694
|
+
Propose 2-3 strong, distinct CodeceptJS assertion code blocks that each directly prove the claim.
|
|
592
695
|
Use only data from the <page> context to plan the verification.
|
|
593
|
-
|
|
696
|
+
Prefer the fewest, most specific assertions over many variants of the same locator.
|
|
594
697
|
|
|
595
698
|
IMPORTANT: Each code block must verify the SPECIFIC claim in the message, not just a generic aspect of it.
|
|
596
699
|
Bad: I.seeElement({"role":"button","aria-pressed":"true"}) — matches ANY button, not the specific one
|
|
@@ -612,6 +715,7 @@ class Navigator implements Agent {
|
|
|
612
715
|
const conversation = this.provider.startConversation(this.systemPrompt, 'navigator');
|
|
613
716
|
conversation.addUserText(prompt);
|
|
614
717
|
|
|
718
|
+
let alreadyVerified = false;
|
|
615
719
|
const tools = this.buildExperienceTools();
|
|
616
720
|
|
|
617
721
|
let codeBlocks: string[] = [];
|
|
@@ -619,57 +723,90 @@ class Navigator implements Agent {
|
|
|
619
723
|
const assertionSteps: Array<{ name: string; args: any[] }> = [];
|
|
620
724
|
|
|
621
725
|
const action = this.explorer.createAction();
|
|
726
|
+
let failures = 0;
|
|
622
727
|
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
const result = await this.provider.invokeConversation(conversation, tools);
|
|
627
|
-
if (!result) return;
|
|
628
|
-
const aiResponse = result?.response?.text;
|
|
629
|
-
debugLog('Received AI response:', aiResponse?.length ?? 0, 'characters');
|
|
630
|
-
tag('step').log('Verifying assertion...');
|
|
631
|
-
codeBlocks = extractCodeBlocks(aiResponse ?? '');
|
|
632
|
-
}
|
|
728
|
+
const page = this.explorer.playwrightHelper?.page;
|
|
729
|
+
const originalTimeout = this.explorer.playwrightHelper?.options?.timeout ?? 3000;
|
|
730
|
+
page?.setDefaultTimeout(this.verifyTimeout);
|
|
633
731
|
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
}
|
|
732
|
+
try {
|
|
733
|
+
await loop(
|
|
734
|
+
async ({ stop, iteration }) => {
|
|
735
|
+
if (codeBlocks.length === 0) {
|
|
736
|
+
const result = await this.provider.invokeConversation(conversation, tools);
|
|
737
|
+
if (!result) return;
|
|
738
|
+
const aiResponse = result?.response?.text ?? '';
|
|
739
|
+
debugLog('Received AI response:', aiResponse.length, 'characters');
|
|
740
|
+
tag('step').log('Verifying assertion...');
|
|
741
|
+
|
|
742
|
+
if (this.checkAlreadyVerified(aiResponse, actionResult)) {
|
|
743
|
+
alreadyVerified = true;
|
|
744
|
+
stop();
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
637
747
|
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
stop();
|
|
641
|
-
return;
|
|
642
|
-
}
|
|
748
|
+
codeBlocks = extractCodeBlocks(aiResponse);
|
|
749
|
+
}
|
|
643
750
|
|
|
644
|
-
|
|
751
|
+
if (codeBlocks.length === 0) {
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
645
754
|
|
|
646
|
-
|
|
755
|
+
const codeBlock = codeBlocks[iteration - 1];
|
|
756
|
+
if (!codeBlock) {
|
|
757
|
+
stop();
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
647
760
|
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
761
|
+
await this.explorer.switchToMainFrame();
|
|
762
|
+
|
|
763
|
+
const verified = await action.attempt(codeBlock, message, false);
|
|
764
|
+
|
|
765
|
+
if (verified) {
|
|
766
|
+
tag('success').log('Verification passed');
|
|
767
|
+
successfulCodes.push(codeBlock);
|
|
768
|
+
assertionSteps.push(...action.assertionSteps);
|
|
769
|
+
} else {
|
|
770
|
+
failures++;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
const target = Math.min(codeBlocks.length, this.verifyAttempts);
|
|
774
|
+
const majorityNeeded = Math.floor(target / 2) + 1;
|
|
775
|
+
if (successfulCodes.length >= majorityNeeded || failures > target - majorityNeeded) {
|
|
776
|
+
stop();
|
|
777
|
+
}
|
|
661
778
|
},
|
|
662
|
-
|
|
663
|
-
|
|
779
|
+
{
|
|
780
|
+
maxAttempts: this.verifyAttempts,
|
|
781
|
+
observability: {
|
|
782
|
+
agent: 'navigator',
|
|
783
|
+
},
|
|
784
|
+
catch: async (error) => {
|
|
785
|
+
debugLog(error);
|
|
786
|
+
},
|
|
787
|
+
}
|
|
788
|
+
);
|
|
789
|
+
} finally {
|
|
790
|
+
page?.setDefaultTimeout(originalTimeout);
|
|
791
|
+
}
|
|
664
792
|
|
|
665
|
-
const totalAttempted = Math.min(codeBlocks.length, this.
|
|
666
|
-
const
|
|
793
|
+
const totalAttempted = Math.min(codeBlocks.length, this.verifyAttempts);
|
|
794
|
+
const majorityNeeded = Math.floor(totalAttempted / 2) + 1;
|
|
795
|
+
let verified = successfulCodes.length >= majorityNeeded;
|
|
796
|
+
if (alreadyVerified) verified = true;
|
|
667
797
|
|
|
668
798
|
actionResult.addVerification(message, verified);
|
|
669
799
|
this.explorer.getStateManager().updateState(actionResult);
|
|
670
800
|
|
|
671
801
|
return { verified, successfulCodes, assertionSteps, totalAttempted };
|
|
672
802
|
}
|
|
803
|
+
|
|
804
|
+
private checkAlreadyVerified(aiResponse: string, actionResult: ActionResult): boolean {
|
|
805
|
+
const verifiedMatch = aiResponse.match(/ALREADY_VERIFIED:\s*(.+)/i);
|
|
806
|
+
if (!verifiedMatch) return false;
|
|
807
|
+
const claim = verifiedMatch[1].trim().replace(/^["']|["']$/g, '');
|
|
808
|
+
return actionResult.getVerification(claim) === true;
|
|
809
|
+
}
|
|
673
810
|
}
|
|
674
811
|
|
|
675
812
|
export { Navigator };
|
package/src/ai/pilot.ts
CHANGED
|
@@ -322,6 +322,12 @@ export class Pilot implements Agent {
|
|
|
322
322
|
- "Delete X" → X must be gone. Clicking delete is NOT enough.
|
|
323
323
|
- "Edit X" → updated value must be persisted (visible in list/detail). Opening edit is NOT enough; redirect after save with the new value visible IS enough.
|
|
324
324
|
- Negative tests ("without a name", "invalid", "duplicate", "unauthorized") → success means the system PREVENTED the action with validation/error.
|
|
325
|
+
- Navigation-prefixed titles ("Access/Open/Go to X to <do Y>") → the goal is <do Y>; reaching X is
|
|
326
|
+
only a milestone. A satisfied milestone (tab active, panel/prompt visible, list shown) is NEVER a pass
|
|
327
|
+
if <do Y> did not occur this run.
|
|
328
|
+
- If the page reveals the goal cannot be performed here — required control absent, integration not
|
|
329
|
+
connected, or only a setup/connect/empty-state prompt is shown — vote "skipped" (prerequisites unmet),
|
|
330
|
+
never "pass".
|
|
325
331
|
|
|
326
332
|
PROVENANCE: the entity you cite as proof must appear by name in <notes> or
|
|
327
333
|
<session_log> tool inputs for THIS run. Name absent from tester activity = stale
|
package/src/commands/index.ts
CHANGED
|
@@ -26,6 +26,7 @@ import { PlanEditCommand } from './plan-edit-command.js';
|
|
|
26
26
|
import { PlanLoadCommand } from './plan-load-command.js';
|
|
27
27
|
import { PlanReloadCommand } from './plan-reload-command.js';
|
|
28
28
|
import { PlanSaveCommand } from './plan-save-command.js';
|
|
29
|
+
import { PlansCommand } from './plans-command.js';
|
|
29
30
|
import { RerunCommand } from './rerun-command.js';
|
|
30
31
|
import { ResearchCommand } from './research-command.js';
|
|
31
32
|
import { RunsCommand } from './runs-command.js';
|
|
@@ -48,6 +49,7 @@ const commandClasses: CommandClass[] = [
|
|
|
48
49
|
PlanCommand,
|
|
49
50
|
PlanSaveCommand,
|
|
50
51
|
PlanLoadCommand,
|
|
52
|
+
PlansCommand,
|
|
51
53
|
PlanReloadCommand,
|
|
52
54
|
PlanClearCommand,
|
|
53
55
|
PlanEditCommand,
|
|
@@ -30,6 +30,15 @@ const config = {
|
|
|
30
30
|
// agentic model for decision making
|
|
31
31
|
agenticModel: openrouter('minimax/minimax-m2.5:nitro'),
|
|
32
32
|
},
|
|
33
|
+
|
|
34
|
+
reporter: {
|
|
35
|
+
// Save a local HTML report after each run.
|
|
36
|
+
html: true,
|
|
37
|
+
// Save a local markdown report after each run.
|
|
38
|
+
markdown: true,
|
|
39
|
+
// Group runs by title in Testomat.io / HTML reports. Defaults to today's date — customize or remove.
|
|
40
|
+
runGroup: new Date().toISOString().slice(0, 10),
|
|
41
|
+
},
|
|
33
42
|
};
|
|
34
43
|
|
|
35
44
|
export default config;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { existsSync, readdirSync, statSync } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { Plan } from '../test-plan.js';
|
|
4
|
+
import { getCliName } from '../utils/cli-name.js';
|
|
5
|
+
import { tag } from '../utils/logger.js';
|
|
6
|
+
import { relativeToCwd } from '../utils/next-steps.js';
|
|
7
|
+
import { BaseCommand } from './base-command.js';
|
|
8
|
+
|
|
9
|
+
export class PlansCommand extends BaseCommand {
|
|
10
|
+
name = 'plans';
|
|
11
|
+
description = 'List saved plans and show their test scenarios';
|
|
12
|
+
options = [{ flags: '--from-plan <file>', description: 'Plan file to show' }];
|
|
13
|
+
|
|
14
|
+
async execute(args: string): Promise<void> {
|
|
15
|
+
const { opts, args: remaining } = this.parseArgs(args);
|
|
16
|
+
const files = this.getPlanFiles();
|
|
17
|
+
const target = String(opts.fromPlan || remaining[0] || '').trim();
|
|
18
|
+
|
|
19
|
+
if (!target) {
|
|
20
|
+
this.printPlans(files);
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const file = this.resolvePlanFile(target, files);
|
|
25
|
+
const plan = Plan.fromMarkdown(file.path);
|
|
26
|
+
this.printPlanDetails(plan, file);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
private getPlanFiles(): PlanFile[] {
|
|
30
|
+
const plansDir = this.explorBot.getPlansDir();
|
|
31
|
+
if (!existsSync(plansDir)) return [];
|
|
32
|
+
|
|
33
|
+
return readdirSync(plansDir)
|
|
34
|
+
.filter((file) => file.endsWith('.md'))
|
|
35
|
+
.map((file) => {
|
|
36
|
+
const filePath = path.join(plansDir, file);
|
|
37
|
+
const stat = statSync(filePath);
|
|
38
|
+
return {
|
|
39
|
+
name: file,
|
|
40
|
+
path: filePath,
|
|
41
|
+
modifiedAt: stat.mtimeMs,
|
|
42
|
+
};
|
|
43
|
+
})
|
|
44
|
+
.sort((left, right) => right.modifiedAt - left.modifiedAt);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
private printPlans(files: PlanFile[]): void {
|
|
48
|
+
if (files.length === 0) {
|
|
49
|
+
tag('info').log(`No saved plans found in ${relativeToCwd(this.explorBot.getPlansDir())}`);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
tag('info').log('Saved plans:');
|
|
54
|
+
for (let i = 0; i < files.length; i++) {
|
|
55
|
+
const file = files[i];
|
|
56
|
+
const plan = Plan.fromMarkdown(file.path);
|
|
57
|
+
tag('info').log(`${i + 1}. ${plan.title} (${plan.tests.length} tests) - ${file.name}`);
|
|
58
|
+
}
|
|
59
|
+
tag('info').log('');
|
|
60
|
+
tag('info').log(`View plan tests: ${getCliName()} plans <number>`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private printPlanDetails(plan: Plan, file: PlanFile): void {
|
|
64
|
+
tag('info').log(`${plan.title} (${plan.tests.length} tests)`);
|
|
65
|
+
for (let i = 0; i < plan.tests.length; i++) {
|
|
66
|
+
const test = plan.tests[i];
|
|
67
|
+
tag('info').log(`${i + 1}. ${test.scenario}`);
|
|
68
|
+
}
|
|
69
|
+
tag('info').log('');
|
|
70
|
+
tag('info').log('Run test from this plan as:');
|
|
71
|
+
tag('info').log(`${getCliName()} test 1 --from-plan ${file.name}`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
private resolvePlanFile(target: string, files: PlanFile[]): PlanFile {
|
|
75
|
+
const index = Number.parseInt(target, 10);
|
|
76
|
+
if (!Number.isNaN(index) && String(index) === target) {
|
|
77
|
+
const file = files[index - 1];
|
|
78
|
+
if (!file) throw new Error(`Plan #${target} not found. Available: 1-${files.length}`);
|
|
79
|
+
return file;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const resolved = this.explorBot.resolvePlanPath(target);
|
|
83
|
+
if (!existsSync(resolved)) {
|
|
84
|
+
throw new Error(`Plan file not found: ${resolved}`);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
name: path.basename(resolved),
|
|
89
|
+
path: resolved,
|
|
90
|
+
modifiedAt: statSync(resolved).mtimeMs,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
interface PlanFile {
|
|
96
|
+
name: string;
|
|
97
|
+
path: string;
|
|
98
|
+
modifiedAt: number;
|
|
99
|
+
}
|
|
@@ -6,13 +6,20 @@ import { BaseCommand, type Suggestion } from './base-command.js';
|
|
|
6
6
|
export class TestCommand extends BaseCommand {
|
|
7
7
|
name = 'test';
|
|
8
8
|
description = 'Launch tester agent to execute test scenarios';
|
|
9
|
+
options = [{ flags: '--from-plan <file>', description: 'Load plan file before selecting tests' }];
|
|
9
10
|
suggestions: Suggestion[] = [
|
|
10
11
|
{ command: 'test', hint: 'run next test' },
|
|
11
12
|
{ command: 'plan', hint: 'create new plan' },
|
|
12
13
|
];
|
|
13
14
|
|
|
14
15
|
async execute(args: string): Promise<void> {
|
|
16
|
+
const { opts, args: remaining } = this.parseArgs(args);
|
|
17
|
+
if (opts.fromPlan) {
|
|
18
|
+
this.explorBot.loadPlan(String(opts.fromPlan));
|
|
19
|
+
}
|
|
20
|
+
|
|
15
21
|
const plan = this.explorBot.getCurrentPlan();
|
|
22
|
+
const selector = remaining.join(' ');
|
|
16
23
|
Stats.mode = 'test';
|
|
17
24
|
Stats.focus = plan?.title;
|
|
18
25
|
const toExecute: Test[] = [];
|
|
@@ -22,22 +29,22 @@ export class TestCommand extends BaseCommand {
|
|
|
22
29
|
return plan;
|
|
23
30
|
};
|
|
24
31
|
|
|
25
|
-
if (!
|
|
32
|
+
if (!selector) {
|
|
26
33
|
const pending = requirePlan().getPendingTests();
|
|
27
34
|
if (pending.length === 0) {
|
|
28
35
|
throw new Error('All tests are already complete. Please run /plan to create new test scenarios.');
|
|
29
36
|
}
|
|
30
37
|
toExecute.push(pending[0]);
|
|
31
|
-
} else if (
|
|
38
|
+
} else if (selector === '*' || selector === 'all') {
|
|
32
39
|
toExecute.push(...requirePlan().getPendingTests());
|
|
33
|
-
} else if (
|
|
40
|
+
} else if (selector.match(/^[\d,\-\s]+$/)) {
|
|
34
41
|
const visible = requirePlan().tests.filter((t) => t.enabled);
|
|
35
|
-
const indices = parseTestIndices(
|
|
42
|
+
const indices = parseTestIndices(selector, visible.length);
|
|
36
43
|
for (const idx of indices) {
|
|
37
44
|
toExecute.push(visible[idx]);
|
|
38
45
|
}
|
|
39
46
|
} else {
|
|
40
|
-
const matching = plan?.getPendingTests().filter((test) => test.scenario.toLowerCase().includes(
|
|
47
|
+
const matching = plan?.getPendingTests().filter((test) => test.scenario.toLowerCase().includes(selector.toLowerCase())) || [];
|
|
41
48
|
if (matching.length > 0) {
|
|
42
49
|
toExecute.push(...matching);
|
|
43
50
|
} else {
|
|
@@ -45,12 +52,12 @@ export class TestCommand extends BaseCommand {
|
|
|
45
52
|
if (!state) {
|
|
46
53
|
throw new Error('No page loaded. Please navigate to a page first.');
|
|
47
54
|
}
|
|
48
|
-
const newTest = new Test(
|
|
55
|
+
const newTest = new Test(selector, 'unknown', [], state.url);
|
|
49
56
|
if (plan) {
|
|
50
57
|
plan.addTest(newTest);
|
|
51
|
-
tag('info').log(`Created new test: "${
|
|
58
|
+
tag('info').log(`Created new test: "${selector}" and added to current plan.`);
|
|
52
59
|
} else {
|
|
53
|
-
tag('info').log(`Created ad-hoc test: "${
|
|
60
|
+
tag('info').log(`Created ad-hoc test: "${selector}"`);
|
|
54
61
|
}
|
|
55
62
|
toExecute.push(newTest);
|
|
56
63
|
}
|
package/src/config.ts
CHANGED
|
@@ -84,6 +84,8 @@ interface PilotAgentConfig extends AgentConfig {
|
|
|
84
84
|
interface NavigatorAgentConfig extends AgentConfig {
|
|
85
85
|
addHtmlOnTry?: number;
|
|
86
86
|
maxAttempts?: number;
|
|
87
|
+
verifyAttempts?: number;
|
|
88
|
+
verifyTimeout?: number;
|
|
87
89
|
}
|
|
88
90
|
|
|
89
91
|
type HealFn = (ctx: { I: any }) => Promise<void> | void;
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
+
import { createRequire } from 'node:module';
|
|
1
2
|
import { readFile } from 'node:fs/promises';
|
|
2
|
-
// @ts-ignore — package ships a .js re-export without typings for this sub-path
|
|
3
|
-
import * as playwrightUtils from 'playwright-core/lib/utils';
|
|
4
3
|
import { createDebug } from './utils/logger.ts';
|
|
5
4
|
|
|
6
5
|
const debugLog = createDebug('explorbot:playwright-recorder');
|
|
@@ -12,10 +11,25 @@ const RECORDABLE: Record<string, Set<string>> = {
|
|
|
12
11
|
|
|
13
12
|
const PLAYWRIGHT_INCOMPATIBLE = "Playwright output is not compatible with this Playwright version (playwright-core/lib/utils does not expose asLocator). Use output.framework: 'codeceptjs' instead, or pin Playwright to a version shipping lib/utils/isomorphic/locatorGenerators.js.";
|
|
14
13
|
|
|
14
|
+
let cachedAsLocator: ((lang: string, selector: string) => string) | null = null;
|
|
15
|
+
let asLocatorLoadAttempted = false;
|
|
16
|
+
const nodeRequire = typeof require === 'function' ? require : createRequire(import.meta.url);
|
|
17
|
+
|
|
15
18
|
function getAsLocator(): (lang: string, selector: string) => string {
|
|
16
|
-
|
|
17
|
-
if (
|
|
18
|
-
|
|
19
|
+
if (cachedAsLocator) return cachedAsLocator;
|
|
20
|
+
if (asLocatorLoadAttempted) throw new Error(PLAYWRIGHT_INCOMPATIBLE);
|
|
21
|
+
|
|
22
|
+
asLocatorLoadAttempted = true;
|
|
23
|
+
try {
|
|
24
|
+
const mod = nodeRequire('playwright-core/lib/utils');
|
|
25
|
+
if (typeof (mod as any)?.asLocator === 'function') {
|
|
26
|
+
cachedAsLocator = (mod as any).asLocator;
|
|
27
|
+
return cachedAsLocator!;
|
|
28
|
+
}
|
|
29
|
+
} catch {
|
|
30
|
+
// Module not exported or not found
|
|
31
|
+
}
|
|
32
|
+
throw new Error(PLAYWRIGHT_INCOMPATIBLE);
|
|
19
33
|
}
|
|
20
34
|
|
|
21
35
|
export interface TraceCall {
|
package/src/reporter.ts
CHANGED
|
@@ -29,7 +29,7 @@ export class Reporter {
|
|
|
29
29
|
this.reporterEnabled = Reporter.resolveEnabled(config);
|
|
30
30
|
this.stateManager = stateManager;
|
|
31
31
|
|
|
32
|
-
if (this.reporterEnabled &&
|
|
32
|
+
if (this.reporterEnabled && config?.html) {
|
|
33
33
|
this.configureHtmlPipe();
|
|
34
34
|
}
|
|
35
35
|
|
|
@@ -63,6 +63,7 @@ export class Reporter {
|
|
|
63
63
|
static resolveEnabled(config?: ReporterConfig): boolean {
|
|
64
64
|
if (config?.enabled === true) return true;
|
|
65
65
|
if (config?.enabled === false) return false;
|
|
66
|
+
if (config?.html || config?.markdown) return true;
|
|
66
67
|
return Boolean(process.env.TESTOMATIO);
|
|
67
68
|
}
|
|
68
69
|
|
|
@@ -88,12 +89,8 @@ export class Reporter {
|
|
|
88
89
|
|
|
89
90
|
private configureRunGroup(runGroup: string | null | undefined): void {
|
|
90
91
|
if (process.env.TESTOMATIO_RUNGROUP_TITLE) return;
|
|
91
|
-
if (runGroup
|
|
92
|
-
|
|
93
|
-
process.env.TESTOMATIO_RUNGROUP_TITLE = runGroup;
|
|
94
|
-
return;
|
|
95
|
-
}
|
|
96
|
-
process.env.TESTOMATIO_RUNGROUP_TITLE = `Explorbot ${new Date().toISOString().slice(0, 10)}`;
|
|
92
|
+
if (!runGroup) return;
|
|
93
|
+
process.env.TESTOMATIO_RUNGROUP_TITLE = runGroup;
|
|
97
94
|
}
|
|
98
95
|
|
|
99
96
|
async startRun(): Promise<void> {
|