explorbot 0.3.4 → 0.4.0

package/dist/boat/prima/src/envelope.js

@@ -1,5 +1,7 @@
-import { mkdirSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
+export const STATUS_FILE = 'status.json';
+const ARTIFACT_FILES = { aria: 'aria.yml', html: 'page.html', screenshot: 'page.png', network: 'network.jsonl' };
 const EXPECTATION_LABELS = {
     passed: 'PASSED       ',
     failed: 'FAILED       ',
@@ -13,21 +15,35 @@ export function renderEnvelope(data) {
 export function writeArtifacts(dir, snapshot) {
     mkdirSync(dir, { recursive: true });
     const paths = {
-        aria: path.resolve(dir, 'aria.yml'),
-        html: path.resolve(dir, 'page.html'),
+        aria: path.resolve(dir, ARTIFACT_FILES.aria),
+        html: path.resolve(dir, ARTIFACT_FILES.html),
     };
     writeFileSync(paths.aria, snapshot.aria ?? '', 'utf-8');
     writeFileSync(paths.html, snapshot.html ?? '', 'utf-8');
     if (snapshot.screenshot) {
-        paths.screenshot = path.resolve(dir, 'page.png');
+        paths.screenshot = path.resolve(dir, ARTIFACT_FILES.screenshot);
         writeFileSync(paths.screenshot, snapshot.screenshot);
     }
     if (!snapshot.requests.length)
         return paths;
-    paths.network = path.resolve(dir, 'network.jsonl');
+    paths.network = path.resolve(dir, ARTIFACT_FILES.network);
     writeFileSync(paths.network, snapshot.requests.map((request) => `${JSON.stringify(request)}\n`).join(''), 'utf-8');
     return paths;
 }
+export function readArtifacts(dir) {
+    const paths = { aria: path.resolve(dir, ARTIFACT_FILES.aria), html: path.resolve(dir, ARTIFACT_FILES.html) };
+    if (existsSync(path.resolve(dir, ARTIFACT_FILES.screenshot)))
+        paths.screenshot = path.resolve(dir, ARTIFACT_FILES.screenshot);
+    if (existsSync(path.resolve(dir, ARTIFACT_FILES.network)))
+        paths.network = path.resolve(dir, ARTIFACT_FILES.network);
+    const recorded = [...Object.values(ARTIFACT_FILES), STATUS_FILE];
+    const files = readdirSync(dir)
+        .filter((entry) => !recorded.includes(entry))
+        .sort();
+    if (files.length)
+        paths.files = files;
+    return paths;
+}
 function renderResult(data) {
     const lines = [`ok: ${data.ok}`, `command: ${data.command}`];
     if (data.used?.length)
@@ -70,7 +86,7 @@ function renderSteps(data) {
             lines.push(`      ${line}`);
     });
     if (data.stepFiles)
-        lines.push('', `page after each step: ${data.stepFiles}`);
+        lines.push('', `page after each step: ${data.stepFiles}/<n>-<step>.{aria.yaml,html,diff.yaml}`);
     return section('Steps', lines.join('\n'));
 }
 function renderExpectations(data) {
@@ -154,6 +170,8 @@ export function renderArtifacts(data) {
         lines.push(`screenshot: ${data.artifacts.screenshot}`);
     if (data.artifacts.network)
         lines.push(`network:    ${data.artifacts.network}`);
+    if (data.artifacts.files?.length)
+        lines.push(`also here:  ${data.artifacts.files.join(', ')}`);
     return section('Artifacts', lines.join('\n'));
 }
 function align(label, marker, width) {

package/dist/boat/prima/src/prima.js

@@ -23,7 +23,7 @@ import { browserErrorMessage } from "../../../src/utils/browser-errors.js";
 import { pluralize } from "../../../src/utils/logger.js";
 import { mdq } from "../../../src/utils/markdown-query.js";
 import { safeFilename } from "../../../src/utils/strings.js";
-import { writeArtifacts } from "./envelope.js";
+import { STATUS_FILE, readArtifacts, writeArtifacts } from "./envelope.js";
 import { isFunctionExpression, takePwValue, toCodeceptWrapper } from "./pw-parser.js";
 import { readDescriptors, selectDescriptor } from "./pw-registry.js";
 import { latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from "./session-log.js";
@@ -341,8 +341,6 @@ export class Prima {
         const problems = [...unreached.map((expectation) => `not reached: ${expectation.text}`), ...contradicted.map((expectation) => `the picture and the run disagree about: ${expectation.text}`)];
         if (problems.length)
             envelope.failure = { error: problems.join('\n') };
-        if (contradicted.length)
-            envelope.artifacts = this.artifacts;
         if (!test.hasFinished || test.isSkipped) {
             envelope.ok = false;
             envelope.failure = { error: `the run did not complete, so it established nothing about the app: ${notes.at(-1)?.message || 'no steps were recorded'}` };
@@ -924,14 +922,17 @@ export class Prima {
         return response?.text || '';
     }
     async successEnvelope(command, used, result, previousState) {
+        const changes = await this.pageChanges(result, previousState, used[0]);
+        const status = await this.saveStatus(result);
         return {
             ok: true,
             command,
             used,
             page: this.pageBlock(result, previousState),
-            changes: await this.pageChanges(result, previousState, used[0]),
+            changes,
             instance: await this.instanceInfo(),
-            status: await this.saveStatus(result),
+            status,
+            artifacts: this.artifacts,
         };
     }
     async failureEnvelope(command, error, previousState) {
@@ -939,23 +940,27 @@ export class Prima {
         const failure = { error: browserErrorMessage(error) };
         if (result.ariaSnapshot)
             failure.compactAria = compactAriaSnapshot(result.ariaSnapshot, true);
+        const status = await this.saveStatus(result);
         return {
             ok: false,
             command,
             page: this.pageBlock(result, previousState),
             failure,
             instance: await this.instanceInfo(),
-            status: await this.saveStatus(result),
+            status,
+            artifacts: this.artifacts,
         };
     }
     async reportEnvelope(command, result, previousState, outcome) {
+        const status = await this.saveStatus(result);
         return {
             ok: true,
             command,
             page: this.pageBlock(result, previousState),
             ...outcome,
             instance: await this.instanceInfo(),
-            status: await this.saveStatus(result),
+            status,
+            artifacts: this.artifacts,
         };
     }
     async capturedResult(previousState, opts = {}) {
@@ -1000,23 +1005,30 @@ export class Prima {
         return lines.join('\n');
     }
     async status(hash) {
+        if (!this.artifactsDir) {
+            const sites = listSites();
+            const site = sites.find((candidate) => existsSync(path.join(candidate.dir, 'output', 'prima', hash))) || sites[0];
+            if (site && !this.configBaseUrl())
+                this.sessionUrl = site.url;
+            await this.loadConfig();
+        }
         const dir = this.statusDir(hash);
-        const statusFile = path.join(dir, 'status.json');
+        const statusFile = path.join(dir, STATUS_FILE);
         if (!existsSync(statusFile))
-            return this.toolFailureEnvelope(`status ${hash}`, `No command was recorded under ${hash}. Every envelope prints its own hash on the Instance line.`);
+            return this.toolFailureEnvelope(`status ${hash}`, `No command was recorded under ${dir}. Every envelope prints its own hash on the Instance line.`);
         const saved = JSON.parse(readFileSync(statusFile, 'utf-8'));
         return {
             ok: true,
             command: `status ${hash}`,
             page: saved.page,
             instance: await this.instanceInfo(),
-            artifacts: { aria: path.join(dir, 'aria.yml'), html: path.join(dir, 'page.html') },
+            artifacts: readArtifacts(dir),
         };
     }
     async saveStatus(result) {
         const hash = this.statusHash();
         await this.writeSnapshot(result);
-        writeFileSync(path.join(this.statusDir(hash), 'status.json'), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8');
+        writeFileSync(path.join(this.statusDir(hash), STATUS_FILE), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8');
         return hash;
     }
     async writeStepFiles(index, label, diff) {

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.3.4",
+  "version": "0.4.0",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",
@@ -77,7 +77,7 @@
   "dependencies": {
     "@ai-sdk/anthropic": "^4.0",
     "@ai-sdk/google": "^4.0.18",
-    "@ai-sdk/groq": "^4.0",
+    "@ai-sdk/groq": "^4.0.34",
     "@ai-sdk/mistral": "^4.0.13",
     "@ai-sdk/openai": "^4.0",
     "@ai-sdk/otel": "^1.0.2",

package/dist/src/action-result.d.ts

@@ -29,7 +29,6 @@ interface ActionResultData extends WebPageState {
     focusedElement?: FocusedElement | null;
     iframeURL?: string;
     links?: Link[];
-    overlayHtml?: string;
 }
 export interface PageDiff {
     urlChanged: boolean;
@@ -42,6 +41,7 @@ export interface PageDiff {
     consoleErrors?: string[];
     htmlParts?: HtmlDiffPart[];
     iframes?: string;
+    areaOfInterest?: string;
 }
 export interface ToolResultMetadata {
     url: string;
@@ -84,6 +84,10 @@ export declare class ActionResult implements ActionResultData {
     links: Link[];
     verifications?: Record<string, boolean>;
     overlay: Overlay;
+    _diffCache: {
+        previousId: number | undefined;
+        diff: Diff;
+    } | null;
     constructor(data: ActionResultData);
     get hash(): string;
     get html(): string;
@@ -117,8 +121,10 @@ export declare class ActionResult implements ActionResultData {
     get relativeUrl(): string | null;
     get isInsideIframe(): boolean;
     getStateHash(): string;
+    get baseHash(): string;
     diff(previousState: ActionResult | null): Promise<Diff>;
     toToolResult(previousState: ActionResult | null, locator: string): Promise<ToolResultMetadata>;
+    computeStateHash(includeRegion: boolean): string;
     consoleErrors(): string[];
 }
 export declare class Diff {
@@ -140,6 +146,8 @@ export declare class Diff {
     get ariaChangeCount(): number;
     get htmlDiff(): HtmlDiffResult | null;
     get messages(): string[];
+    get similarity(): number;
+    get pageSize(): number;
     calculate(): Promise<void>;
 }
 export interface NetworkCall {

package/dist/src/action-result.js

@@ -39,6 +39,7 @@ export class ActionResult {
     links = [];
     verifications;
     overlay = new Overlay();
+    _diffCache = null;
     constructor(data) {
         this.id = data.id;
         this.timestamp = data.timestamp ?? new Date();
@@ -108,6 +109,7 @@ export class ActionResult {
     set html(value) {
         this._html = value;
         this.snapshotCache.clear();
+        this._diffCache = null;
     }
     get screenshot() {
         if (this._screenshot) {
@@ -193,6 +195,14 @@ export class ActionResult {
     isRelevantExperienceRecord(record, options) {
         if (!record.url || !this.url)
             return false;
+        if (record.region && this.overlay.name !== record.region)
+            return false;
+        if (record.root) {
+            if (!this.overlay.present)
+                return false;
+            if (this.overlay.root && this.overlay.root !== record.root)
+                return false;
+        }
         if (this.isMatchedBy(record))
             return true;
         if (!options?.includeDescendantExperience)
@@ -386,24 +396,17 @@ export class ActionResult {
         return !!this.iframeURL;
     }
     getStateHash() {
-        const parts = [];
-        parts.push(this.relativeUrl || this.url || '/');
-        this.extractHeadings(this.html);
-        if (this.h1)
-            parts.push(`h1_${this.h1}`);
-        if (this.h2)
-            parts.push(`h2_${this.h2}`);
-        let stateString = slugify(parts.map((part) => part.substring(0, 100)).join('_'));
-        if (stateString.length > 200) {
-            stateString = stateString.substring(0, 200);
-            if (stateString.endsWith('_')) {
-                stateString = stateString.slice(0, -1);
-            }
-        }
-        return stateString;
+        return this.computeStateHash(true);
+    }
+    get baseHash() {
+        return this.computeStateHash(false);
     }
     async diff(previousState) {
-        return Diff.create(this, previousState);
+        if (this._diffCache && this._diffCache.previousId === previousState?.id)
+            return this._diffCache.diff;
+        const diff = await Diff.create(this, previousState);
+        this._diffCache = { previousId: previousState?.id, diff };
+        return diff;
     }
     async toToolResult(previousState, locator) {
         const result = {
@@ -443,7 +446,18 @@ export class ActionResult {
             pageDiff.ariaChanges = diff.ariaChanged;
             pageDiff.ariaChangeCount = diff.ariaChangeCount;
         }
-        if (diff.htmlParts.length > 0) {
+        if (this.overlay.present && (!previousState.overlay.present || previousState.overlay.name !== this.overlay.name)) {
+            pageDiff.areaOfInterest = this.overlay.describe();
+        }
+        if (pageDiff.areaOfInterest && this.overlay.html && this.overlay.root) {
+            const htmlConfig = ConfigParser.getInstance().getConfig().html;
+            let subtree = await minifyHtml(htmlCombinedSnapshot(this.overlay.html, htmlConfig?.combined));
+            if (subtree.length > HTML_PART_SUBTREE_BUDGET) {
+                subtree = `${subtree.slice(0, HTML_PART_SUBTREE_BUDGET)}...<!-- truncated -->`;
+            }
+            pageDiff.htmlParts = [{ container: this.overlay.root, subtree, rawSize: subtree.length, added: [], removed: [] }];
+        }
+        else if (diff.isSameUrl() && diff.htmlParts.length > 0) {
             const collapsed = collapseHtmlParts(await diff.cleanedHtmlParts());
             if (collapsed.length > 0) {
                 pageDiff.htmlParts = collapsed;
@@ -456,6 +470,25 @@ export class ActionResult {
         }
         return result;
     }
+    computeStateHash(includeRegion) {
+        const parts = [];
+        parts.push(this.relativeUrl || this.url || '/');
+        this.extractHeadings(this.html);
+        if (this.h1)
+            parts.push(`h1_${this.h1}`);
+        if (this.h2)
+            parts.push(`h2_${this.h2}`);
+        if (includeRegion && this.overlay.present && this.overlay.name)
+            parts.push(`region_${this.overlay.name}`);
+        let stateString = slugify(parts.map((part) => part.substring(0, 100)).join('_'));
+        if (stateString.length > 200) {
+            stateString = stateString.substring(0, 200);
+            if (stateString.endsWith('_')) {
+                stateString = stateString.slice(0, -1);
+            }
+        }
+        return stateString;
+    }
     consoleErrors() {
         const errors = [];
         for (const log of this.browserLogs) {
@@ -560,14 +593,20 @@ export class Diff {
     get messages() {
         return this._messages;
     }
+    get similarity() {
+        return this._htmlDiffResult?.similarity ?? 0;
+    }
+    get pageSize() {
+        return this._htmlDiffResult?.pageSize ?? 0;
+    }
     async calculate() {
         if (!this.previous)
             return;
+        this._htmlDiffResult = await htmlDiff(this.previous.html, this.current.html, ConfigParser.getInstance().getConfig().html);
         if (!this._isSameUrl) {
             this._messages = liveRegionMessages(this.previous.html, this.current.html);
             return;
         }
-        this._htmlDiffResult = await htmlDiff(this.previous.html, this.current.html, ConfigParser.getInstance().getConfig().html);
         this._messages = this._htmlDiffResult.messages;
         const ariaDiff = diffAriaSnapshots(this.previous.ariaSnapshot, this.current.ariaSnapshot);
         this._ariaDiffResult = ariaDiff.text;

package/dist/src/action.d.ts

@@ -33,7 +33,7 @@ declare class Action {
         includeScreenshot?: boolean;
         codeBlock?: string;
     }): Promise<ActionResult>;
-    captureOverlayHtml(): Promise<string>;
+    detectRegionOfInterest(result: ActionResult): Promise<void>;
     captureMainDocumentStatus(): Promise<number | undefined>;
     captureResponses(): () => void;
     recordNetworkCall(request: any, status: number): void;

package/dist/src/action.js

@@ -8,9 +8,9 @@ import { clearActivity, setActivity } from "./activity.js";
 import { ConfigParser, outputPath } from './config.js';
 import { Observability } from "./observability.js";
 import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from "./utils/browser-errors.js";
-import { captureHtmlForSnapshot, getVisibleOverlayHtmlExtractorSource, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
+import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
 import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
-import { Overlay } from './utils/overlay.js';
+import { Overlay, OverlayPage } from './utils/overlay.js';
 import { sleep, waitForPageReadiness } from "./utils/page-readiness.js";
 import { safeFilename } from "./utils/strings.js";
 import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from "./utils/web-sandbox.js";
@@ -133,13 +133,10 @@ class Action {
             let ariaSnapshot = null;
             let ariaSnapshotFile = undefined;
             let focusedElement = null;
-            let overlayHtml = '';
             try {
                 const page = this.playwrightHelper.page;
                 ariaSnapshot = await page.locator('body').ariaSnapshot();
                 focusedElement = await page.evaluate(readFocusedElement);
-                if (!frame)
-                    overlayHtml = await this.captureOverlayHtml();
             }
             catch (err) {
                 debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
@@ -166,9 +163,10 @@ class Action {
                 ariaSnapshot,
                 ariaSnapshotFile,
                 focusedElement,
-                overlayHtml: overlayHtml || undefined,
                 iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
             });
+            if (!frame)
+                await this.detectRegionOfInterest(result).catch((err) => debugLog('Region detection failed:', err.message));
             this.stateManager.updateState(result, codeBlock);
             return result;
         }
@@ -181,11 +179,64 @@ class Action {
             return new ActionResult({ url, error: msg });
         }
     }
-    async captureOverlayHtml() {
-        return this.playwrightHelper.page.evaluate(({ extractorSource, config }) => {
-            const extract = new Function(`return ${extractorSource}`)();
-            return extract(config);
-        }, { extractorSource: getVisibleOverlayHtmlExtractorSource(), config: Overlay.captureConfig() });
+    async detectRegionOfInterest(result) {
+        const previousState = this.stateManager.getCurrentState();
+        if (!previousState)
+            return;
+        const previous = ActionResult.fromState(previousState);
+        const previousOverlay = previous.overlay;
+        const sameUrl = !!previous.url && result.isSameUrl({ url: previous.url });
+        const overlayPage = new OverlayPage(this.playwrightHelper.page);
+        if (result.overlay.detected && previousOverlay.detected && previousOverlay.root && previousOverlay.type === result.overlay.type && previousOverlay.name === result.overlay.name) {
+            result.overlay = previousOverlay;
+            return;
+        }
+        if (!previous.html)
+            return;
+        if (previous.html === result.html) {
+            if (sameUrl && previousOverlay.present && previousOverlay.xpath && !result.overlay.detected)
+                result.overlay = previousOverlay;
+            return;
+        }
+        let carried = null;
+        if (sameUrl && previousOverlay.present && previousOverlay.xpath) {
+            if (await overlayPage.isStillOpen(previousOverlay)) {
+                carried = previousOverlay;
+            }
+            else {
+                debugLog(`Region closed: ${previousOverlay.name || previousOverlay.type}`);
+                if (!result.overlay.detected) {
+                    const parent = previousOverlay.parent;
+                    if (parent?.xpath) {
+                        const restored = new Overlay(parent);
+                        if (await overlayPage.isStillOpen(restored))
+                            result.overlay = restored;
+                    }
+                    return;
+                }
+            }
+        }
+        const diff = await result.diff(previous);
+        const detected = await overlayPage.detectRegion({
+            parts: diff.htmlParts,
+            pageSize: diff.pageSize,
+            similarity: diff.similarity,
+            sameUrl,
+            previousHtml: previous.html,
+        });
+        if (result.overlay.detected) {
+            if (detected)
+                result.overlay = result.overlay.withGeometry(detected);
+            return;
+        }
+        if (detected) {
+            result.overlay = detected;
+            if (carried)
+                result.overlay = detected.withParent(carried);
+            return;
+        }
+        if (carried)
+            result.overlay = carried;
     }
     async captureMainDocumentStatus() {
         if (this.mainDocumentStatus)
@@ -349,6 +400,14 @@ class Action {
                 await recorder.reset();
                 await recorder.start();
             }
+            if (executedSteps.length > 0) {
+                codeString = executedSteps.map((step) => step.command).join('\n');
+            }
+            if (!isFatalBrowserError(err)) {
+                const captured = await this.captureOnce({ codeBlock: codeString }).catch(() => null);
+                if (captured && !captured.error)
+                    this.actionResult = captured;
+            }
             this.assertionSteps = [];
             throw err;
         }
@@ -448,14 +507,30 @@ async function captureTitle(page, actor) {
 }
 const ASSERTION_STEP_NAMES = new Set(['see', 'dontSee', 'seeElement', 'dontSeeElement', 'seeInField', 'dontSeeInField', 'seeInCurrentUrl', 'dontSeeInCurrentUrl']);
 export const attachStepLogger = (target, assertionsTarget) => {
+    const recorded = new WeakMap();
+    let batchFailed = false;
     const listener = (step, error) => {
         if (!step?.toCode)
             return;
         if (step.name?.startsWith('grab'))
             return;
+        const existing = recorded.get(step);
+        if (existing) {
+            if (!error && !existing.success) {
+                existing.success = true;
+                existing.error = undefined;
+                batchFailed = target.some((entry) => !entry.success);
+            }
+            return;
+        }
+        if (batchFailed)
+            return;
         const executed = { command: step.toCode(), success: !error };
-        if (error)
+        if (error) {
             executed.error = errorToString(error);
+            batchFailed = true;
+        }
+        recorded.set(step, executed);
         target.push(executed);
         if (assertionsTarget && ASSERTION_STEP_NAMES.has(step.name)) {
             assertionsTarget.push({ name: step.name, args: step.args || [] });

package/dist/src/ai/driller.d.ts

@@ -135,7 +135,6 @@ export declare class Driller extends TaskAgent implements Agent {
     createVerifiedActionTools(baseTools: Record<string, any>, component: ComponentInfo): Record<string, any>;
     hasVerifiedAction(componentId: string): boolean;
     detectNestedOverlayContext(component: ComponentInfo, result: any): Promise<string | null>;
-    getVisibleOverlayHtml(): Promise<string>;
     getComponentScopeHtml(component: ComponentInfo, originalState: ActionResult): Promise<string>;
     saveToKnowledge(knowledgePath: string, state: ActionResult, results: InteractionResult[]): Promise<void>;
     generateKnowledgeContent(state: ActionResult, interactions: InteractionResult[]): string;

package/dist/src/ai/driller.js

@@ -6,10 +6,9 @@ import { setActivity } from "../activity.js";
 import { Observability } from "../observability.js";
 import { Plan, Test, TestResult } from "../test-plan.js";
 import { collectInteractiveNodes } from "../utils/aria.js";
-import { EXPLORBOT_ATTRS, HTML_COMPOSITE_AREA_HINTS, HTML_COMPOSITE_TARGET_ROLES, HTML_EXTRACTION_LIMITS, HTML_FORM_CONTROL_ROLES, HTML_FORM_CONTROL_TAGS, HTML_INTERACTIVE_ROLES, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, getComponentScopeHtmlExtractorSource, getVisibleOverlayHtmlExtractorSource, inferHtmlRole, } from "../utils/html.js";
+import { EXPLORBOT_ATTRS, HTML_COMPOSITE_AREA_HINTS, HTML_COMPOSITE_TARGET_ROLES, HTML_EXTRACTION_LIMITS, HTML_FORM_CONTROL_ROLES, HTML_FORM_CONTROL_TAGS, HTML_INTERACTIVE_ROLES, HTML_SELECTORS, getComponentScopeHtmlExtractorSource, inferHtmlRole } from "../utils/html.js";
 import { createDebug, tag } from "../utils/logger.js";
 import { loop, pause } from "../utils/loop.js";
-import { OVERLAY_SELECTORS } from "../utils/overlay.js";
 import { annotatePageElements } from "../utils/web-annotate.js";
 import { eidxInContainer } from "../utils/web-eidx.js";
 import { WebElement } from "../utils/web-element.js";
@@ -547,8 +546,12 @@ export class Driller extends TaskAgent {
     async detectNestedOverlayContext(component, result) {
         if (!result?.pageDiff?.ariaChanges || result.pageDiff.urlChanged)
             return null;
-        const overlayHtml = await this.getVisibleOverlayHtml();
-        if (!overlayHtml)
+        const parts = result.pageDiff.htmlParts ?? [];
+        let appeared = parts.filter((part) => part.added?.length > 0);
+        if (result.pageDiff.areaOfInterest)
+            appeared = parts;
+        const appearedHtml = appeared.map((part) => part.subtree).join('\n');
+        if (!appearedHtml)
             return null;
         const state = this.stateManager.getCurrentState();
         if (!state)
@@ -561,7 +564,7 @@ export class Driller extends TaskAgent {
       Keep the recorded code reusable and include the parent-opening action when the nested element requires the overlay to be open.
 
       <overlay_html>
-      ${overlayHtml}
+      ${appearedHtml}
       </overlay_html>
 
       <current_page_aria>
@@ -570,21 +573,6 @@ export class Driller extends TaskAgent {
       </nested_overlay>
     `;
     }
-    async getVisibleOverlayHtml() {
-        return this.explorer.withPage((page) => page.evaluate(({ extractorSource, config }) => {
-            const extract = new Function(`return ${extractorSource}`)();
-            return extract(config);
-        }, {
-            extractorSource: getVisibleOverlayHtmlExtractorSource(),
-            config: {
-                interactiveContentSelector: HTML_SELECTORS.interactiveContent,
-                limits: HTML_EXTRACTION_LIMITS,
-                overlaySelectors: OVERLAY_SELECTORS.semanticOverlays,
-                overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector,
-                visibilityLimits: HTML_VISIBILITY_LIMITS,
-            },
-        }));
-    }
     async getComponentScopeHtml(component, originalState) {
         const scopedHtml = await this.explorer.withPage((page) => page.evaluate(({ eidx, extractorSource, config }) => {
             const extract = new Function(`return ${extractorSource}`)();

package/dist/src/ai/fisherman-tools.d.ts

@@ -74,6 +74,7 @@ export declare function createFishermanTools(apiClient: ApiClient, requestStore:
             statusText?: undefined;
             category?: undefined;
             errorPreview?: undefined;
+            extracted?: undefined;
         } | {
             success: boolean;
             status: number;
@@ -81,9 +82,11 @@ export declare function createFishermanTools(apiClient: ApiClient, requestStore:
             category: ResponseCategory;
             errorPreview: string;
             error?: undefined;
+            extracted?: undefined;
         } | {
             success: boolean;
             status: number;
+            extracted: Record<string, any>;
             error?: undefined;
             statusText?: undefined;
             category?: undefined;
@@ -102,6 +105,10 @@ export declare function createFishermanTools(apiClient: ApiClient, requestStore:
             }[];
         }, {
             finished: boolean;
+            error: string;
+        } | {
+            finished: boolean;
+            error?: undefined;
         }, import("@ai-sdk/provider-utils").Context>>;
         stop: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
             reason: any;
@@ -111,6 +118,7 @@ export declare function createFishermanTools(apiClient: ApiClient, requestStore:
     };
     getResult: () => FishermanResult;
     isFinished: () => boolean;
+    finishFromText: (text?: string) => void;
 };
 export interface FishermanResult {
     success: boolean;
@@ -119,6 +127,7 @@ export interface FishermanResult {
         type: string;
         id?: string | number;
         title?: string;
+        request?: string;
     }>;
     failed: Array<{
         type: string;