explorbot 0.2.3 → 0.2.5

package/docs/workflow/reporting.md

@@ -0,0 +1,133 @@
+# Test Reporting
+
+Explorbot generates test reports with [@testomatio/reporter](https://github.com/testomatio/reporter). Reports include test steps, screenshots, and result messages for every run. The [API tester](../api-testing/basics.md) reports through the same reporter, so the HTML and Testomat.io settings below apply to API runs too.
+
+The [Testomat.io cloud reporter](#testomatio-cloud-report) is the recommended way to review runs, especially on CI: pair it with the [Analyst](#session-analysis) and [screencasts](#screencasts) and each run opens with a written overview you can drill from down to a video of any single test. The [HTML](#html-report-local) and [markdown](#markdown-report-local) reports are the local alternatives. For the full CI setup, see [Continuous Integration](./ci.md#reporting-on-ci).
+
+## Session analysis
+
+After `/explore` and `/freesail` runs, the [Analyst agent](../web-testing/agents.md#analyst-agent) writes a summary that clusters findings by root cause.
+
+The same markdown is printed to the console, written to disk, and set as the run description on Testomat.io when the cloud reporter is enabled.
+
+**On disk:** `output/reports/<mode>-<sessionName>.md` — for example `explore-WiseFox42.md` or `freesail-CleverOwl91.md`. Each run gets a unique name, so nothing is overwritten.
+
+**On Testomat.io:** the markdown becomes the run description, so the analysis sits next to the test list in the cloud dashboard with no extra setup.
+
+See [Analyst Agent](../web-testing/agents.md#analyst-agent) for the report format and configuration options.
+
+## Screencasts
+
+Explorbot can record a video of every test instead of leaving you a trail of screenshots. Enable it on the Historian agent:
+
+```js
+export default {
+  ai: {
+    agents: {
+      historian: {
+        screencast: true,  // or { size: { width: 1280, height: 720 }, quality: 95 }
+      },
+    },
+  },
+};
+```
+
+Each test gets one video in `output/screencasts/`, with the executed actions overlaid in the corner and the current scenario step shown as a chapter title — the video narrates itself. The file is attached to the test as an artifact, so with [artifact storage configured](#artifacts-in-cloud-reports) it appears on the test in Testomat.io, ready to watch next to the failure message.
+
+## HTML report (local)
+
+With `html: true` under `reporter` in `explorbot.config.js`, Explorbot creates an HTML report after each run in `output/reports/<mode>-<sessionName>.html`, for example `explore-WiseFox42.html`. Each session gets its own file, so nothing is overwritten. Open it in a browser to review results. Configs generated by `explorbot init` enable it, so new projects get HTML reports out of the box.
+
+![HTML report](https://github.com/testomatio/explorbot/blob/main/docs/assets/html-report.png)
+
+For cloud reporting with history and team features, see [Testomat.io cloud report](#testomatio-cloud-report).
+
+## Markdown report (local)
+
+Explorbot can write a markdown report next to the HTML one. It's plain text, so you can paste it into a PR description, a chat thread, or a CI summary.
+
+Opt in via `explorbot.config.js`:
+
+```js
+export default {
+  reporter: {
+    enabled: true,
+    html: true,
+    markdown: true,
+  },
+};
+```
+
+Output: `output/reports/<mode>-<sessionName>-tests.md`, for example `explore-WiseFox42-tests.md`. Like the HTML report, the filename is session-scoped, so successive runs don't overwrite each other. The `-tests` suffix keeps it distinct from the [Analyst report](#session-analysis), which writes `<mode>-<sessionName>.md` in the same folder.
+
+The markdown report isn't generated unless `markdown: true` is set. Configs generated by `explorbot init` enable it as well.
+
+## Testomat.io cloud report
+
+[Testomat.io](https://testomat.io) provides a cloud dashboard with test history, analytics, and team collaboration. It's free with unlimited test runs.
+
+![Testomat.io cloud report](https://github.com/testomatio/explorbot/blob/main/docs/assets/cloud-report.png)
+
+### Setup
+
+1. Register at [app.testomat.io](https://app.testomat.io).
+2. Create an empty project.
+3. Copy the project API key.
+
+### Run with the key
+
+```bash
+TESTOMATIO=tstmt_your_key_here npx explorbot explore /
+```
+
+Set the key in your shell profile or CI environment so it's always active.
+
+### Options
+
+| Variable | Description |
+|----------|-------------|
+| `TESTOMATIO` | Project API key (required) |
+| `TESTOMATIO_TITLE` | Custom name for the test run |
+| `TESTOMATIO_ENV` | Environment label (e.g. `staging`, `production`) |
+| `TESTOMATIO_SHARED_RUN` | Merge parallel executions into one run |
+| `TESTOMATIO_RUNGROUP_TITLE` | Group successive runs under one heading (overrides `reporter.runGroup`) |
+
+See the [@testomatio/reporter docs](https://github.com/testomatio/reporter/blob/2.x/docs/pipes/testomatio.md) for the full list.
+
+### Run group
+
+Set `reporter.runGroup` to file successive runs under one heading on Testomat.io. There is no built-in default, but configs generated by `explorbot init` set it to today's date (`YYYY-MM-DD`), so all sessions from one day appear together in the dashboard.
+
+Change or remove it via `explorbot.config.js`:
+
+```js
+export default {
+  reporter: {
+    enabled: true,
+    runGroup: 'Smoke Suite',  // any string; remove to disable grouping
+  },
+};
+```
+
+If set, `TESTOMATIO_RUNGROUP_TITLE` from the environment takes precedence over the config.
+
+## Artifacts in cloud reports
+
+Explorbot attaches screenshots to test steps and, when [screencasts](#screencasts) are enabled, a video to every test. To see them in Testomat.io, configure an S3-compatible storage provider under Settings > Artifacts in Testomat.io. Without it, screenshots and screencasts won't appear in cloud reports.
+
+Set these environment variables:
+
+```bash
+S3_ACCESS_KEY_ID=your_access_key
+S3_SECRET_ACCESS_KEY=your_secret_key
+S3_BUCKET=your-bucket-name
+S3_REGION=us-east-1
+```
+
+For non-AWS providers (DigitalOcean Spaces, Cloudflare R2, Google Cloud Storage, Minio), also set:
+
+```bash
+S3_ENDPOINT=https://your-provider-endpoint.com
+```
+
+See the [@testomatio/reporter artifacts docs](https://github.com/testomatio/reporter/blob/2.x/docs/artifacts.md) for provider-specific examples.

package/docs/workflow/test-plans.md

@@ -0,0 +1,90 @@
+# Test Plans
+
+A test plan is a markdown file with a suite of scenarios for the Tester to run. The [Planner](../web-testing/planner.md) generates plans, but the format is plain markdown. You can write plans by hand, edit generated ones, or check them into version control.
+
+Explorbot saves plans to `output/plans/` by default. The same parser loads them whether they were generated or written by hand. [API testing](../api-testing/basics.md) saves and loads its plans in this same format.
+
+The format is a dialect of the [Testomat.io classical markdown format](https://docs.testomat.io/project/import-export/export-tests/classical-tests-markdown-format/). It adds a `### Prerequisite` block that tells Explorbot which page to open before each test.
+
+## Format
+
+```markdown
+<!-- suite -->
+# Plan Title
+
+### Prerequisite
+
+* URL: /relative-path
+
+<!-- test
+priority: critical
+-->
+# Scenario written as a user-facing sentence
+
+## Requirements
+/relative-path
+
+## Steps
+* First step in plain language
+* Second step
+
+## Expected
+* First expected outcome
+* Second expected outcome
+```
+
+One file can hold several suites. Each begins with its own `<!-- suite -->` marker and parses as an independent plan.
+
+## Elements
+
+### `<!-- suite -->`
+
+Marks the start of a plan. The `#` heading on the next line becomes the plan's title. This follows the Testomat.io convention of HTML-comment metadata blocks.
+
+### `### Prerequisite`
+
+Holds the suite-level URL as a single bullet:
+
+```
+* URL: /relative-path
+```
+
+The URL is required. Without it, the suite's tests do not run. Make it relative to the configured base URL (start with `/`), so the same plan runs against staging, production, or a local dev server without edits.
+
+Tests without their own `## Requirements` URL use this URL as their start page. Explorbot navigates to it before each scenario.
+
+### `<!-- test priority: … -->`
+
+Opens a test block. Valid priorities: `critical`, `important`, `high`, `normal`, `low`. Omit it and the priority defaults to `normal`. See [Test Priorities](../web-testing/planner.md#test-priorities) for what each level means.
+
+### `#` Scenario heading
+
+A single `#` heading inside a test block is the scenario description. Write it as a business outcome, not a click path.
+
+### `## Requirements`
+
+A per-test start URL, written as a plain line (no bullet) right under the heading. It overrides the suite Prerequisite URL for this test. The Planner writes it for every test; when authoring plans by hand, give each test its own Requirements URL.
+
+### `## Steps`
+
+A bulleted list (`* `) of planned actions in plain language. The Tester treats these as guidance, not a strict script, and may adapt them to what it sees on the page. To span a step across lines, indent continuation lines with 2 spaces.
+
+The Testomat.io classical format inlines `*Expected*:` inside each step. Explorbot splits actions and outcomes into separate `## Steps` and `## Expected` sections.
+
+### `## Expected`
+
+A bulleted list (`* `) of expected outcomes. Each outcome should describe a verifiable change: a data change, a state change, or a UI change with a side effect. See the Planner's [outcome-strength guidance](../web-testing/planner.md#built-in-styles) for what counts.
+
+The Tester passes a test only when it has verified every expected outcome.
+
+## Reusing saved plans
+
+Saved plans are reusable. Run `npx explorbot explore --configure="new:25%"` to load the matching plan, re-run a subset of its tests, and let the planner generate a few new ones to fill the budget. Filter old picks by priority or style, and order them by priority, file index, or random shuffle. See the [`--configure` reference in commands.md](../reference/commands.md#explore) for the full key list and examples.
+
+## See Also
+
+- [Automated Tests](../web-testing/automated-tests.md) — the runnable Playwright or CodeceptJS files Explorbot writes after executing a plan
+- [Planner](../web-testing/planner.md) — how plans are generated
+- [Planning Styles](./planning-styles.md) — how planning styles shape scenarios
+- [Commands](../reference/commands.md) — `/plan`, `/explore`, `npx explorbot plan`
+- [Rerun](../web-testing/rerun.md) — re-executing generated tests

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",
@@ -19,6 +19,7 @@
   },
   "files": [
     "dist/",
+    "docs/",
     "src/**/*.ts",
     "src/**/*.tsx",
     "bin/**/*.ts",
@@ -119,7 +120,7 @@
     "ora-classic": "^5.4.2",
     "parse5": "^8.0.0",
     "pixelmatch": "^7.2.0",
-    "playwright": "^1.60",
+    "playwright": "^1.62",
     "pngjs": "^7.0.0",
     "react": "^19.1.1",
     "sambanova-ai-provider": "^1.2.2",

package/rules/navigator/verification-actions.md

@@ -105,9 +105,29 @@ Checks that page source does NOT contain expected text.
   I.dontSeeInSource('error-class');
 </example>
 
+### Asserting the state of a control
+
+State means disabled, checked, readonly, required, selected, expanded. A presence assertion says
+nothing about it, so assert it with an attribute selector:
+
+<example>
+  I.seeElement('button[aria-label="Submit"][disabled]');
+  I.dontSeeElement('button[aria-label="Submit"][disabled]');
+  I.seeElement('input[name="accept"][checked]');
+  I.seeElement('button[aria-label="Details"][aria-expanded="true"]');
+</example>
+
+Use I.seeElement for the state you expect and I.dontSeeElement for the state you expect to be
+absent — that pair expresses both directions.
+
+I.seeAttributesOnElements(<locator>, { disabled: true }) also exists, but only takes a state that
+must be PRESENT and does not resolve reliably against a role/text locator. Prefer the selector form above.
+
 <verification_rules>
 Be strict in assertions to avoid false positives.
 Prefer I.seeElement() with ARIA locators - most reliable.
+For a claim about a control's state, assert it with an attribute selector — presence of the element is not evidence of its state.
+If no assertion above can express the claim, say so instead of proposing an assertion that checks something weaker.
 I.see() and I.dontSee() MUST include context parameter.
 For input field values, ALWAYS use I.seeInField() — never check value via CSS attribute selectors or I.seeInSource.
 Prefer text locators (label, name, placeholder) for form fields: I.seeInField('Search', 'value') over I.seeInField('input[name="search"]', 'value').

package/src/action-result.ts

@@ -30,6 +30,7 @@ interface ActionResultData extends WebPageState {
   iframeSnapshots?: Array<{ src: string; html: string; id?: string }>;
   ariaSnapshot?: string | null;
   ariaSnapshotFile?: string;
+  focusedElement?: FocusedElement | null;
   iframeURL?: string;
   links?: Link[];
 }
@@ -74,6 +75,7 @@ export class ActionResult implements ActionResultData {
   private snapshotCache = new TTLCache<string>();
   readonly logFile: string | undefined = undefined;
   readonly ariaSnapshotFile: string | undefined = undefined;
+  readonly focusedElement: FocusedElement | null = null;
   private _ariaSnapshot: string | null | undefined = undefined;
   private _lastExtractedHtml: string | undefined = undefined;
   notes: string[] = [];
@@ -107,6 +109,9 @@ export class ActionResult implements ActionResultData {
     if (data.ariaSnapshotFile !== undefined) {
       this.ariaSnapshotFile = data.ariaSnapshotFile;
     }
+    if (data.focusedElement !== undefined) {
+      this.focusedElement = data.focusedElement;
+    }
 
     // Store HTML in a private property if provided
     if (data.html !== undefined) {
@@ -643,3 +648,9 @@ export class Diff {
     this._ariaChangeCount = ariaDiff.count;
   }
 }
+
+export interface FocusedElement {
+  role: string;
+  name: string;
+  value?: string;
+}

package/src/action.ts

@@ -3,7 +3,7 @@ import { join } from 'node:path';
 import { context, trace } from '@opentelemetry/api';
 import { container, recorder } from 'codeceptjs';
 import * as codeceptjs from 'codeceptjs';
-import { ActionResult } from './action-result.js';
+import { ActionResult, type FocusedElement } from './action-result.js';
 import { clearActivity, setActivity } from './activity.ts';
 import { ConfigParser, outputPath } from './config.js';
 import type { ExplorbotConfig } from './config.js';
@@ -19,6 +19,8 @@ import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCo
 
 const debugLog = createDebug('explorbot:action');
 const CAPTURE_NAVIGATION_TRANSITION_ATTEMPTS = 3;
+const DEFAULT_ACTION_TIMEOUT = 3000;
+const DEFAULT_PAGE_TIMEOUT = 3000;
 
 class Action {
   private actor: CodeceptJS.I;
@@ -32,6 +34,7 @@ class Action {
   public playwrightHelper: any;
   public playwrightGroupId: string | null = null;
   public assertionSteps: Array<{ name: string; args: any[] }> = [];
+  public lastValue: unknown;
   private recorder?: PlaywrightRecorder;
   private recovery: RecoveryRunner;
   private mainDocumentStatus: number | undefined = undefined;
@@ -54,6 +57,7 @@ class Action {
     try {
       await (this.actor as any).saveScreenshot(filename);
       if (currentState) currentState.screenshotFile = filename;
+      tag('data').log('screenshot', { path: outputPath('states', filename) });
       return filename;
     } catch (err) {
       debugLog('Screenshot failed:', err);
@@ -102,7 +106,10 @@ class Action {
         const screenshotPath = join(statesDir, filename);
         screenshotFile = await page
           ?.screenshot({ path: screenshotPath, fullPage: true })
-          .then(() => filename)
+          .then(() => {
+            tag('data').log('screenshot', { path: screenshotPath });
+            return filename;
+          })
           .catch((err: Error) => {
             debugLog('Screenshot failed, continuing without it:', err);
             return undefined;
@@ -133,10 +140,12 @@ class Action {
 
       let ariaSnapshot: string | null = null;
       let ariaSnapshotFile: string | undefined = undefined;
+      let focusedElement: FocusedElement | null = null;
 
       try {
         const page = this.playwrightHelper.page;
         ariaSnapshot = await page.locator('body').ariaSnapshot();
+        focusedElement = await page.evaluate(readFocusedElement);
       } catch (err) {
         debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
       }
@@ -160,6 +169,7 @@ class Action {
         iframeSnapshots,
         ariaSnapshot,
         ariaSnapshotFile,
+        focusedElement,
         iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
       });
       this.stateManager.updateState(result, codeBlock);
@@ -297,16 +307,21 @@ class Action {
         throw new Error('No valid I.* or page.* commands found in code block');
       }
 
+      this.playwrightHelper?.page?.setDefaultTimeout(this.config.action?.timeout ?? DEFAULT_ACTION_TIMEOUT);
+
       if (isPlaywright) {
         const page = this.playwrightHelper.page;
         await playwrightSandbox(page, sanitizedCode);
         await sleep(this.config.action?.delay || 500);
       } else {
-        codeceptJSSandbox(this.actor, sanitizedCode);
+        const returned = codeceptJSSandbox(this.actor, sanitizedCode);
         await recorder.add(() => sleep(this.config.action?.delay || 500));
         await recorder.promise();
+        this.lastValue = await returned;
       }
 
+      this.restorePageTimeout();
+
       if (executedSteps.length > 0) {
         codeString = executedSteps.join('\n');
       }
@@ -325,6 +340,7 @@ class Action {
       this.assertionSteps = [];
       throw err;
     } finally {
+      this.restorePageTimeout();
       detachMainDocumentResponse();
       if (groupId) await this.recorder!.endAction();
       detachStepLogger(stepListener);
@@ -374,6 +390,10 @@ class Action {
     return this.actionResult;
   }
 
+  private restorePageTimeout(): void {
+    this.playwrightHelper?.page?.setDefaultTimeout(this.config.playwright.timeout ?? DEFAULT_PAGE_TIMEOUT);
+  }
+
   private async waitForPageReadiness(page: any): Promise<void> {
     await waitForPageReadiness(page, {
       timeout: this.config.playwright.waitForTimeout,
@@ -442,3 +462,23 @@ const detachStepLogger = (listener: StepListener) => {
   codeceptjs.event.dispatcher.off(codeceptjs.event.step.passed, listener);
   codeceptjs.event.dispatcher.off(codeceptjs.event.step.failed, listener);
 };
+
+const readFocusedElement = () => {
+  const el = document.activeElement as any;
+  if (!el || el === document.body) return null;
+
+  const tag = el.tagName.toLowerCase();
+  const textish = new Set(['text', 'search', 'email', 'password', 'url', 'tel', 'number']);
+  let role = el.getAttribute('role') || tag;
+  if (tag === 'textarea' || el.isContentEditable) role = 'textbox';
+  if (tag === 'input' && textish.has(el.type)) role = 'textbox';
+  if (tag === 'select') role = 'combobox';
+  if (tag === 'a') role = 'link';
+
+  const label = el.getAttribute('aria-label') || el.getAttribute('placeholder') || el.labels?.[0]?.textContent || el.textContent || '';
+  const focused: { role: string; name: string; value?: string } = { role, name: label.trim().slice(0, 80) };
+
+  const value = el.value ?? el.textContent;
+  if (typeof value === 'string' && value) focused.value = value.slice(0, 200);
+  return focused;
+};

package/src/ai/captain/web-mode.ts

@@ -18,7 +18,7 @@ export function WithWebMode<T extends Constructor>(Base: T) {
       });
       const { see, context, visualClick, learnExperience } = agentTools;
 
-      return {
+      const tools: Record<string, any> = {
         navigate: tool({
           description: 'Navigate to a URL or page description using AI-powered navigation.',
           inputSchema: z.object({
@@ -123,11 +123,14 @@ export function WithWebMode<T extends Constructor>(Base: T) {
         }),
 
         ...codeceptTools,
-        see,
         context,
-        visualClick,
         learnExperience,
       };
+
+      if (see) tools.see = see;
+      if (visualClick) tools.visualClick = visualClick;
+
+      return tools;
     }
 
     webModePrompt(): string {

package/src/ai/captain.ts

@@ -21,6 +21,7 @@ import type { Navigator } from './navigator.ts';
 import type { Provider } from './provider.ts';
 import { Researcher } from './researcher.ts';
 import { TaskAgent } from './task-agent.ts';
+import { withdrawVisionTools } from './tools.ts';
 
 const MAX_STEPS = 15;
 
@@ -437,6 +438,8 @@ export class Captain extends CaptainBase implements Agent {
           return;
         }
 
+        withdrawVisionTools(tools);
+
         const currentState = stateManager.getCurrentState();
         if (!currentState && this.getMode() !== 'idle') {
           stop();