explorbot 0.3.5 → 0.4.0

package/dist/src/utils/overlay.js

@@ -1,43 +1,213 @@
 import { detectFocusArea } from './aria.js';
-import { HTML_EXTRACTION_LIMITS, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, extractHeadings } from './html.js';
-export const OVERLAY_SELECTORS = {
-    semanticOverlays: ['[role="dialog"]', '[role="listbox"]', '[role="menu"]', '[role="tooltip"]:not([style*="display: none"]):not([style*="visibility: hidden"])', '[class*="modal"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="popup"]', '[class*="drawer"]', '[class*="lightbox"]'],
-    modalOverlays: ['[role="dialog"]', '[role="alertdialog"]', '[aria-modal="true"]', '[class*="modal"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="popup"]', '[class*="drawer"]', '[class*="lightbox"]'],
-    overlaySemanticSelector: '[role="dialog"], [role="alertdialog"], [aria-modal="true"], [role="listbox"], [role="menu"], [role="tooltip"]',
-};
+import { pathToXPath } from './html-diff.js';
+import { extractHeadings } from './html.js';
+import { createDebug } from './logger.js';
+const debugLog = createDebug('explorbot:overlay');
 export class Overlay {
     type;
     name;
+    root;
+    html;
+    xpath;
+    parent;
     constructor(data = {}) {
         this.type = data.type ?? null;
         this.name = data.name ?? null;
+        this.root = data.root ?? null;
+        this.html = data.html ?? null;
+        this.xpath = data.xpath ?? null;
+        this.parent = data.parent ?? null;
     }
     get detected() {
+        return this.type !== null && this.type !== 'region';
+    }
+    get present() {
         return this.type !== null;
     }
-    static fromHtml(html) {
-        const headings = extractHeadings(html);
-        const name = [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean).join(' ');
-        return new Overlay({ type: 'modal', name: name || null });
+    describe() {
+        if (!this.present)
+            return '';
+        let text = `${this.type} "${this.name || 'unnamed'}" opened`;
+        if (this.root)
+            text += `, scope: ${this.root}`;
+        return text;
+    }
+    withGeometry(geometry) {
+        return new Overlay({
+            type: this.type,
+            name: this.name || geometry.name,
+            root: geometry.root,
+            html: geometry.html,
+            xpath: geometry.xpath,
+        });
+    }
+    withParent(parent) {
+        return new Overlay({
+            type: this.type,
+            name: this.name,
+            root: this.root,
+            html: this.html,
+            xpath: this.xpath,
+            parent: { type: parent.type, name: parent.name, root: parent.root, xpath: parent.xpath },
+        });
     }
     static fromAria(snapshot) {
         return new Overlay(detectFocusArea(snapshot));
     }
     static resolve(data) {
-        if (data.overlayHtml)
-            return Overlay.fromHtml(data.overlayHtml);
         if (data.overlay)
             return new Overlay(data.overlay);
         return Overlay.fromAria(data.ariaSnapshot ?? null);
     }
-    static captureConfig() {
-        return {
-            interactiveContentSelector: HTML_SELECTORS.interactiveContent,
-            limits: HTML_EXTRACTION_LIMITS,
-            overlaySelectors: OVERLAY_SELECTORS.modalOverlays,
-            overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector,
-            visibilityLimits: HTML_VISIBILITY_LIMITS,
-            geometryFallback: false,
-        };
+}
+export class OverlayPage {
+    page;
+    constructor(page) {
+        this.page = page;
+    }
+    async detectRegion(diff) {
+        if (!diff.sameUrl && diff.similarity < SOFT_NAVIGATION_SIMILARITY)
+            return null;
+        for (const subRoot of this.appearedSubRoots(diff)) {
+            const probe = await this.probe(subRoot.elementXPath);
+            if (probe?.found && probe.onScreen && !probe.centerBelongs) {
+                debugLog('Appeared region is hidden or covered, trying the next candidate');
+                continue;
+            }
+            const overlay = this.toOverlay(subRoot, probe, diff.previousHtml);
+            if (!overlay.name && !overlay.root) {
+                debugLog('Appeared region has no name and no semantic root, ignoring it');
+                continue;
+            }
+            debugLog(`Region detected: ${overlay.describe()}`);
+            return overlay;
+        }
+        return null;
+    }
+    async isStillOpen(overlay) {
+        if (!overlay.xpath)
+            return true;
+        const probe = await this.probe(overlay.xpath);
+        if (!probe)
+            return true;
+        if (!probe.found)
+            return false;
+        if (probe.onScreen && !probe.centerBelongs && !probe.coveredByFloating)
+            return false;
+        return true;
+    }
+    appearedSubRoots(diff) {
+        const candidates = [];
+        for (const part of diff.parts) {
+            const appeared = part.added.find((line) => line.startsWith('ELEMENT:'));
+            if (!appeared)
+                continue;
+            if (part.subtree.length < SUBROOT_MIN_HTML)
+                continue;
+            if (diff.pageSize > 0 && part.rawSize > REGION_MAX_RATIO * diff.pageSize) {
+                debugLog(`Appeared subtree spans ${Math.round((part.rawSize / diff.pageSize) * 100)}% of the page — a new state, not a region`);
+                continue;
+            }
+            candidates.push({
+                container: part.container,
+                elementXPath: pathToXPath(appeared.slice('ELEMENT:'.length)),
+                subtree: part.subtree,
+                size: part.subtree.length,
+                rawSize: part.rawSize,
+                appearedSelector: part.appearedSelector,
+                fresh: !!this.freshHeading(part.subtree, diff.previousHtml),
+            });
+        }
+        if (candidates.length === 0)
+            return [];
+        const totalRawSize = candidates.reduce((sum, c) => sum + c.rawSize, 0);
+        const largest = Math.max(...candidates.map((c) => c.rawSize));
+        if (largest < REGION_DOMINANCE * totalRawSize) {
+            debugLog('Changes are scattered across the page, no dominant region');
+            return [];
+        }
+        candidates.sort((a, b) => Number(b.fresh) - Number(a.fresh) || b.size - a.size);
+        return candidates;
     }
+    async probe(xpath) {
+        if (!this.page)
+            return null;
+        return this.page
+            .evaluate(({ probeSource, config }) => {
+            const probe = new Function(`return ${probeSource}`)();
+            return probe(config);
+        }, { probeSource: inspectRegion.toString(), config: { xpath } })
+            .catch((err) => {
+            debugLog('Region probe failed:', err.message);
+            return null;
+        });
+    }
+    toOverlay(subRoot, probe, previousHtml) {
+        let type = 'region';
+        if (probe?.centerBelongs && probe.floating)
+            type = 'modal';
+        let root = null;
+        if (subRoot.container !== 'body' && !subRoot.container.startsWith('//'))
+            root = subRoot.container;
+        if (!root && subRoot.appearedSelector)
+            root = subRoot.appearedSelector;
+        return new Overlay({ type, name: this.nameFrom(subRoot.subtree, previousHtml), root, html: subRoot.subtree, xpath: subRoot.elementXPath });
+    }
+    nameFrom(html, previousHtml) {
+        const fresh = this.freshHeading(html, previousHtml);
+        if (fresh)
+            return fresh;
+        const candidates = this.headingsOf(html);
+        if (candidates.length === 0)
+            return null;
+        return candidates.join(' ');
+    }
+    freshHeading(html, previousHtml) {
+        const fresh = this.headingsOf(html).filter((heading) => !previousHtml.includes(heading));
+        return fresh[0] ?? null;
+    }
+    headingsOf(html) {
+        const headings = extractHeadings(html);
+        return [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean);
+    }
+}
+const SUBROOT_MIN_HTML = 5_000;
+const SOFT_NAVIGATION_SIMILARITY = 50;
+const REGION_MAX_RATIO = 0.6;
+const REGION_DOMINANCE = 0.7;
+// Serialized via toString() into page.evaluate — must stay a plain function with no outer-scope references.
+function inspectRegion(config) {
+    const probe = { found: false, onScreen: false, floating: false, centerBelongs: false, coveredByFloating: false };
+    const isFloating = (start) => {
+        let node = start;
+        while (node && node !== document.body) {
+            const style = window.getComputedStyle(node);
+            if (style.position === 'fixed' || style.position === 'absolute' || (Number.parseInt(style.zIndex || '0', 10) || 0) > 0)
+                return true;
+            node = node.parentElement;
+        }
+        return false;
+    };
+    const result = document.evaluate(config.xpath, document, null, 9, null);
+    const node = result.singleNodeValue;
+    if (!node || node.nodeType !== 1)
+        return probe;
+    const element = node;
+    const rect = element.getBoundingClientRect();
+    if (rect.width === 0 && rect.height === 0)
+        return probe;
+    probe.found = true;
+    probe.floating = isFloating(element);
+    const left = Math.max(rect.left, 0);
+    const top = Math.max(rect.top, 0);
+    const right = Math.min(rect.right, window.innerWidth);
+    const bottom = Math.min(rect.bottom, window.innerHeight);
+    if (right <= left || bottom <= top)
+        return probe;
+    probe.onScreen = true;
+    const hit = document.elementFromPoint((left + right) / 2, (top + bottom) / 2);
+    probe.centerBelongs = !!hit && (hit === element || element.contains(hit));
+    if (hit && !probe.centerBelongs)
+        probe.coveredByFloating = isFloating(hit);
+    return probe;
 }

package/dist/src/utils/request-map.d.ts

@@ -0,0 +1,7 @@
+import type { RequestResult } from '../api/request-result.js';
+export declare class RequestMap {
+    requests: Map<string, RequestResult>;
+    constructor(requests?: RequestResult[]);
+    add(request: RequestResult): void;
+    get(id: string | number): RequestResult | undefined;
+}

package/dist/src/utils/request-map.js

@@ -0,0 +1,16 @@
+export class RequestMap {
+    requests = new Map();
+    constructor(requests = []) {
+        for (const request of requests)
+            this.add(request);
+    }
+    add(request) {
+        const { id } = request.extractIdAndTitle();
+        if (id === undefined)
+            return;
+        this.requests.set(String(id), request);
+    }
+    get(id) {
+        return this.requests.get(String(id));
+    }
+}

package/dist/src/utils/url-matcher.js

@@ -9,6 +9,8 @@ export function isDynamicSegment(segment) {
     catch {
         /* config not loaded yet */
     }
+    if (/^v\d+$/i.test(segment))
+        return false;
     // numeric: /users/123
     if (/^\d+$/.test(segment))
         return true;
@@ -18,8 +20,8 @@ export function isDynamicSegment(segment) {
     // ULID: /items/01ARZ3NDEKTSV4RRFFQ69G5FAV
     if (/^[0-9A-HJKMNP-TV-Z]{26}$/.test(segment))
         return true;
-    // hex ID (4+ chars): /suite/70dae98a
-    if (/^[a-f0-9]{4,}$/i.test(segment))
+    // hex ID (4+ chars): /suite/70dae98a — short letters-only hex reads as a word
+    if (/^[a-f0-9]{4,}$/i.test(segment) && (segment.length >= 8 || /\d/.test(segment)))
         return true;
     // hex-prefixed slug (8+ hex before dash): /suite/95ef0c94-mobile
     if (/^[a-f0-9]{8,}-/i.test(segment))

package/docs/reference/commands.md

@@ -106,6 +106,7 @@ EXPLORBOT_AI_PROVIDER=openrouter \
 | `EXPLORBOT_KNOWLEDGE_FILE` | Path to a knowledge markdown file |
 | `EXPLORBOT_API_SPEC` | OpenAPI spec path for the API boat |
 | `EXPLORBOT_NO_BANNER` | Suppress the startup banner, for machine-readable output |
+| `EXPLORBOT_MAX_DURATION` | Wall-clock budget in minutes for an explore run; same as --max-duration |
 <!-- END env -->
 
 `npx explorbot config` prints the values a run actually uses — models per role, the config file behind them, the output, knowledge and experience directories, and every `EXPLORBOT_*` variable currently set. The boats answer for their own configuration the same way: `npx explorbot api config`, `npx explorbot docs config`, `npx explorbot prima config`. Add `--json` on any of them to get the same values as an object a script can read.
@@ -234,6 +235,7 @@ The CLI form navigates to `<path>` first. The TUI form always runs on the curren
 | Option | Description |
 |---|---|
 | `--max-tests <n>` | Hard cap on tests executed in this run. Sub-page expansion stops once the cap is hit. |
+| `--max-duration <minutes>` | Wall-clock budget for the whole run. New tests, planning, and sub-page expansion stop before the limit; an in-flight test is cut at the hard cutoff so the report, Testomatio run, and teardown always complete. Set it a few minutes below your CI job timeout. |
 | `--focus <feature>` | Narrow planning to a single feature area (e.g. `--focus checkout`). The focus also becomes part of the saved plan filename. |
 | `--configure <spec>` | Reuse a saved plan, mix old + new tests, filter by style/priority, control sub-page behavior. See below. |
 | `--dry-run` | Mark every picked test as `skipped` instead of executing. New-test planning still runs (so you can preview what would be picked) but no AI tester actions and no plan-file writes. |
@@ -351,6 +353,7 @@ npx explorbot freesail /dashboard --scope /app --max-tests 20
 | `--shallow` | Breadth-first: pick the globally least-visited page |
 | `--scope <prefix>` | Restrict navigation to URLs starting with this prefix |
 | `--max-tests <n>` | Maximum number of tests to run |
+| `--max-duration <minutes>` | Wall-clock budget in minutes for the whole run; freesail stops starting new pages before the limit |
 
 ### research
 
@@ -431,6 +434,8 @@ npx explorbot test 3 --from-plan output/plans/login.md   # index first, plan via
 | `--grep <pattern>` | Run only tests whose scenario matches the pattern |
 | `--from-plan <file>` | Load this plan file when the first argument is a test index |
 
+The plan names the site it runs against: the URL of its `### Prerequisite` section, or the `## Requirements` URL of its first test. With a [global installation](configuration.md#running-from-anywhere-the-global-installation) that is enough to run a plan from any directory without a project config — `npx explorbot test ~/plans/checkout.md` registers the site and stores its output under `~/.explorbot/sites/<host>/`. Naming a saved plan is enough too: `npx explorbot test checkout` looks for `checkout.md` in the current directory, then in the plans directory of every registered site.
+
 ### drill
 
 Drill all components on a page to learn interactions.
@@ -685,7 +690,9 @@ html: /home/you/.explorbot/sites/app.example.com/output/prima/2026-08-04T10-04-2
 
 `used:` is code that already executed: for `go` the CodeceptJS step it ran, for `pw` the Playwright expression you passed, which a CodeceptJS test needs wrapped in `I.usePlaywrightTo(...)`. Log lines can precede the envelope, so start parsing at the first `###` line.
 
-`### Changes` renders on every action envelope, saying `no change` when the tree is identical — so a successful command proves what it did instead of leaving you to check. `check` and `do` report per step rather than in aggregate: `### Steps` names each step with the code it ran and what proved it, and `page after each step:` points at the captures. `check` adds `### Expected outcomes`; `ask`, `research`, and `verify` add `### Answer`, `### Research`, or `### Assertions`; `pw` adds `### Value` when its expression returns one. `network:` appears under `### Artifacts` only when requests were captured, and everything else recorded for a command is behind `prima status <hash>`.
+`### Changes` renders on every action envelope, saying `no change` when the tree is identical — so a successful command proves what it did instead of leaving you to check. `check` and `do` report per step rather than in aggregate: `### Steps` names each step with the code it ran and what proved it, and `page after each step:` names the per-step captures. `check` adds `### Expected outcomes`; `ask`, `research`, and `verify` add `### Answer`, `### Research`, or `### Assertions`; `pw` adds `### Value` when its expression returns one.
+
+`### Artifacts` names the files every command leaves on disk: the aria tree and the html always, `screenshot:` and `network:` when a screenshot was taken or requests were captured. The aria tree carries values and checked states, so one `grep` over it settles a question the envelope did not answer, without asking the page again. `prima status <hash>` reprints those paths for an earlier command and lists whatever else was kept under the hash; it only reads recorded files, so it needs no browser and outlives the session.
 
 **A failed action is a failure.** Nothing is retried along a different route and no other element is substituted, so `ok: true` means the action you asked for is the one that landed. A failure adds `### Failure` with the error and the compact ARIA of the page, so you can retarget from the envelope itself instead of opening the artifact files.
 

package/docs/reference/websocket.md

@@ -19,6 +19,7 @@ Every message is JSON with a `type` and a `ts`, plus whatever that type carries.
 | `screenshot` | the screenshot file just written |
 | `research` | the researcher's map of a page: markdown and its file |
 | `report` | the analyst's end-of-session report: markdown and its file |
+| `docs` | the doc collector's spec index: markdown and its file |
 | `activity` | what the run is doing this second |
 | `log` | a log line and its level |
 | `ask` | a question for a human, carrying an `askId` |