explorbot 0.3.1 → 0.3.2

package/bin/explorbot-cli.ts

@@ -9,6 +9,7 @@ import { render } from 'ink';
 import React from 'react';
 import { App } from '../src/components/App.js';
 import { StatusPane } from '../src/components/StatusPane.js';
+import { flushTelemetry } from '../src/ai/provider.js';
 import { ConfigParser, EXPLORBOT_ENV_VARS, PROVIDERS } from '../src/config.js';
 import { ExplorBot, type ExplorBotOptions } from '../src/explorbot.js';
 import { remote } from '../src/remote.js';
@@ -103,9 +104,7 @@ async function startTUI(explorBot: ExplorBot): Promise<void> {
 async function showStatsAndExit(code: number): Promise<never> {
   if (remote.isAttached()) {
     await remote.close(code);
-    process.exit(code);
-  }
-  if (Stats.hasActivity()) {
+  } else if (Stats.hasActivity()) {
     await new Promise<void>((resolve) => {
       const { unmount } = render(
         React.createElement(StatusPane, {
@@ -121,6 +120,7 @@ async function showStatsAndExit(code: number): Promise<never> {
       );
     });
   }
+  await flushTelemetry();
   process.exit(code);
 }
 

package/dist/bin/explorbot-cli.js

@@ -9,6 +9,7 @@ import { render } from 'ink';
 import React from 'react';
 import { App } from '../src/components/App.js';
 import { StatusPane } from '../src/components/StatusPane.js';
+import { flushTelemetry } from '../src/ai/provider.js';
 import { ConfigParser, EXPLORBOT_ENV_VARS, PROVIDERS } from '../src/config.js';
 import { ExplorBot } from '../src/explorbot.js';
 import { remote } from '../src/remote.js';
@@ -78,9 +79,8 @@ async function startTUI(explorBot) {
 async function showStatsAndExit(code) {
     if (remote.isAttached()) {
         await remote.close(code);
-        process.exit(code);
     }
-    if (Stats.hasActivity()) {
+    else if (Stats.hasActivity()) {
         await new Promise((resolve) => {
             const { unmount } = render(React.createElement(StatusPane, {
                 onComplete: () => {
@@ -93,6 +93,7 @@ async function showStatsAndExit(code) {
             });
         });
     }
+    await flushTelemetry();
     process.exit(code);
 }
 addCommonOptions(program.command('start [path]').description('Start web exploration')).action(async (startPath, options) => {

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/dist/src/action-result.d.ts

@@ -2,6 +2,7 @@ import { type HtmlConfig } from './config.js';
 import type { Link, WebPageState } from './state-manager.js';
 import { TTLCache } from './utils/cache.js';
 import { type HtmlDiffPart, type HtmlDiffResult } from './utils/html-diff.js';
+import { Overlay } from './utils/overlay.js';
 interface ActionResultData extends WebPageState {
     html?: string;
     fullUrl?: string | undefined;
@@ -28,6 +29,7 @@ interface ActionResultData extends WebPageState {
     focusedElement?: FocusedElement | null;
     iframeURL?: string;
     links?: Link[];
+    overlayHtml?: string;
 }
 export interface PageDiff {
     urlChanged: boolean;
@@ -81,6 +83,7 @@ export declare class ActionResult implements ActionResultData {
     notes: string[];
     links: Link[];
     verifications?: Record<string, boolean>;
+    overlay: Overlay;
     constructor(data: ActionResultData);
     get hash(): string;
     get html(): string;

package/dist/src/action-result.js

@@ -5,6 +5,7 @@ import { TTLCache } from "./utils/cache.js";
 import { htmlDiff, liveRegionMessages } from "./utils/html-diff.js";
 import { extractHeadings, extractLinks, extractTargetedHtml, htmlCombinedSnapshot, htmlMinimalUISnapshot, htmlTextSnapshot, minifyHtml } from "./utils/html.js";
 import { createDebug } from "./utils/logger.js";
+import { Overlay } from "./utils/overlay.js";
 import { slugify } from "./utils/strings.js";
 import { extractStatePath, matchesUrl } from "./utils/url-matcher.js";
 const debugLog = createDebug('explorbot:state');
@@ -37,6 +38,7 @@ export class ActionResult {
     notes = [];
     links = [];
     verifications;
+    overlay = new Overlay();
     constructor(data) {
         this.id = data.id;
         this.timestamp = data.timestamp ?? new Date();
@@ -77,6 +79,7 @@ export class ActionResult {
         if (data.ariaSnapshot !== undefined) {
             this._ariaSnapshot = data.ariaSnapshot;
         }
+        this.overlay = Overlay.resolve(data);
         if (!this.fullUrl && this.url) {
             this.fullUrl = this.url;
         }

package/dist/src/action.d.ts

@@ -32,6 +32,7 @@ declare class Action {
         includeScreenshot?: boolean;
         codeBlock?: string;
     }): Promise<ActionResult>;
+    captureOverlayHtml(): Promise<string>;
     captureMainDocumentStatus(): Promise<number | undefined>;
     captureResponses(): () => void;
     recordNetworkCall(request: any, status: number): void;

package/dist/src/action.js

@@ -8,8 +8,9 @@ import { clearActivity, setActivity } from "./activity.js";
 import { ConfigParser, outputPath } from './config.js';
 import { Observability } from "./observability.js";
 import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from "./utils/browser-errors.js";
-import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
+import { captureHtmlForSnapshot, getVisibleOverlayHtmlExtractorSource, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
 import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
+import { Overlay } from './utils/overlay.js';
 import { sleep, waitForPageReadiness } from "./utils/page-readiness.js";
 import { safeFilename } from "./utils/strings.js";
 import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from "./utils/web-sandbox.js";
@@ -131,10 +132,13 @@ class Action {
             let ariaSnapshot = null;
             let ariaSnapshotFile = undefined;
             let focusedElement = null;
+            let overlayHtml = '';
             try {
                 const page = this.playwrightHelper.page;
                 ariaSnapshot = await page.locator('body').ariaSnapshot();
                 focusedElement = await page.evaluate(readFocusedElement);
+                if (!frame)
+                    overlayHtml = await this.captureOverlayHtml();
             }
             catch (err) {
                 debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
@@ -161,6 +165,7 @@ class Action {
                 ariaSnapshot,
                 ariaSnapshotFile,
                 focusedElement,
+                overlayHtml: overlayHtml || undefined,
                 iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
             });
             this.stateManager.updateState(result, codeBlock);
@@ -175,6 +180,12 @@ class Action {
             return new ActionResult({ url, error: msg });
         }
     }
+    async captureOverlayHtml() {
+        return this.playwrightHelper.page.evaluate(({ extractorSource, config }) => {
+            const extract = new Function(`return ${extractorSource}`)();
+            return extract(config);
+        }, { extractorSource: getVisibleOverlayHtmlExtractorSource(), config: Overlay.captureConfig() });
+    }
     async captureMainDocumentStatus() {
         if (this.mainDocumentStatus)
             return this.mainDocumentStatus;

package/dist/src/ai/driller.js

@@ -9,6 +9,7 @@ import { collectInteractiveNodes } from "../utils/aria.js";
 import { EXPLORBOT_ATTRS, HTML_COMPOSITE_AREA_HINTS, HTML_COMPOSITE_TARGET_ROLES, HTML_EXTRACTION_LIMITS, HTML_FORM_CONTROL_ROLES, HTML_FORM_CONTROL_TAGS, HTML_INTERACTIVE_ROLES, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, getComponentScopeHtmlExtractorSource, getVisibleOverlayHtmlExtractorSource, inferHtmlRole, } from "../utils/html.js";
 import { createDebug, tag } from "../utils/logger.js";
 import { loop, pause } from "../utils/loop.js";
+import { OVERLAY_SELECTORS } from "../utils/overlay.js";
 import { annotatePageElements } from "../utils/web-annotate.js";
 import { eidxInContainer } from "../utils/web-eidx.js";
 import { WebElement } from "../utils/web-element.js";
@@ -578,7 +579,8 @@ export class Driller extends TaskAgent {
             config: {
                 interactiveContentSelector: HTML_SELECTORS.interactiveContent,
                 limits: HTML_EXTRACTION_LIMITS,
-                overlaySelectors: HTML_SELECTORS.semanticOverlays,
+                overlaySelectors: OVERLAY_SELECTORS.semanticOverlays,
+                overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector,
                 visibilityLimits: HTML_VISIBILITY_LIMITS,
             },
         }));

package/dist/src/ai/pilot.js

@@ -5,7 +5,7 @@ import { ActionResult } from "../action-result.js";
 import { ConfigParser } from "../config.js";
 import { Stats } from "../stats.js";
 import { TestResult } from "../test-plan.js";
-import { collectInteractiveNodes, detectFocusArea } from "../utils/aria.js";
+import { collectInteractiveNodes } from "../utils/aria.js";
 import { ErrorPageError } from "../utils/error-page.js";
 import { createDebug, tag } from "../utils/logger.js";
 const debugLog = createDebug('explorbot:pilot');
@@ -738,7 +738,7 @@ export class Pilot {
         lines.push(`h2: ${state.h2 || ''}`);
         lines.push(`h3: ${state.h3 || ''}`);
         lines.push(`h4: ${state.h4 || ''}`);
-        const focusArea = detectFocusArea(state.ariaSnapshot);
+        const focusArea = state.overlay;
         if (focusArea.detected) {
             lines.push(`modal: ${focusArea.name || focusArea.type}`);
         }

package/dist/src/ai/provider.d.ts

@@ -7,6 +7,7 @@ declare class AiError extends Error {
 }
 export declare class ContextLengthError extends Error {
 }
+export declare function flushTelemetry(): Promise<void>;
 export declare class Provider {
     config: AIConfig;
     telemetryEnabled: boolean;
@@ -17,6 +18,7 @@ export declare class Provider {
     modelCallWaiters: (() => void)[];
     constructor(config: AIConfig);
     validateConnection(): Promise<void>;
+    stop(): Promise<void>;
     getModelForAgent(agentName?: string): any;
     getAgenticModel(agentName?: string): any;
     getVisionModel(): any;

package/dist/src/ai/provider.js

@@ -31,6 +31,15 @@ function createHarmonyChannelFallbackTool() {
     });
 }
 let telemetryRegistered = false;
+let beforeExitFlushHooked = false;
+let activeOtelSdk = null;
+export async function flushTelemetry() {
+    const sdk = activeOtelSdk;
+    activeOtelSdk = null;
+    if (!sdk)
+        return;
+    await sdk.shutdown().catch((error) => debugLog(`Telemetry flush failed: ${error instanceof Error ? error.message : error}`));
+}
 const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
 function extractCachedTokens(usage) {
     if (!usage)
@@ -105,6 +114,9 @@ export class Provider {
             throw new AiError(`AI connection failed: ${error.message}`);
         }
     }
+    async stop() {
+        await flushTelemetry();
+    }
     getModelForAgent(agentName) {
         if (!agentName) {
             return this.config.model;
@@ -251,7 +263,12 @@ export class Provider {
             spanProcessors: [processor],
             instrumentations: [],
         });
+        activeOtelSdk = this.otelSdk;
         void this.otelSdk.start();
+        if (!beforeExitFlushHooked) {
+            process.on('beforeExit', () => void flushTelemetry());
+            beforeExitFlushHooked = true;
+        }
         if (!telemetryRegistered) {
             registerTelemetry(new OpenTelemetry());
             telemetryRegistered = true;
@@ -293,7 +310,7 @@ export class Provider {
     }
     async invokeConversation(conversation, tools, options = {}) {
         const response = tools ? await this.generateWithTools(conversation.messages, conversation.model, tools, options) : await this.chat(conversation.messages, conversation.model, options);
-        const responseMessages = response.response?.messages || [];
+        const responseMessages = response.responseMessages || [];
         if (responseMessages.length > 0) {
             conversation.messages.push(...responseMessages);
             tag('debug').log('Added', responseMessages.length, 'messages from response');

package/dist/src/ai/researcher/deep-analysis.js

@@ -1,7 +1,7 @@
 import dedent from 'dedent';
 import { ActionResult } from '../../action-result.js';
 import { executionController } from "../../execution-controller.js";
-import { detectFocusArea, diffAriaSnapshots } from "../../utils/aria.js";
+import { diffAriaSnapshots } from "../../utils/aria.js";
 import { extractCodeBlocks } from "../../utils/code-extractor.js";
 import { tag } from '../../utils/logger.js';
 import { mdq } from "../../utils/markdown-query.js";
@@ -60,7 +60,7 @@ export function WithDeepAnalysis(Base) {
             this._appendExtendedResearch(result, expandedSections, navigationLinks);
         }
         async researchOverlay(current, previous, pageStateHash) {
-            const focusArea = detectFocusArea(current.ariaSnapshot);
+            const focusArea = current.overlay;
             if (!focusArea.detected || !focusArea.name)
                 return null;
             if (focusArea.type !== 'dialog' && focusArea.type !== 'modal')

package/dist/src/ai/tester.js

@@ -9,7 +9,6 @@ import { Observability } from "../observability.js";
 import { normalizeUrl } from "../state-manager.js";
 import { Stats } from "../stats.js";
 import { TestResult } from "../test-plan.js";
-import { detectFocusArea } from "../utils/aria.js";
 import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
 import { createDebug, tag } from "../utils/logger.js";
 import { loop } from "../utils/loop.js";
@@ -466,7 +465,7 @@ export class Tester extends TaskAgent {
         this.previousUrl = currentUrl;
         this.previousStateHash = currentStateHash;
         let context = '';
-        const focusArea = detectFocusArea(currentState.ariaSnapshot);
+        const focusArea = currentState.overlay;
         const focusedElement = currentState.focusedElement;
         if (focusedElement) {
             const isTextInput = ['textbox', 'combobox', 'searchbox'].includes(focusedElement.role);

package/dist/src/commands/exit-command.js

@@ -9,7 +9,7 @@ export class ExitCommand extends BaseCommand {
     aliases = ['quit'];
     async execute(_args) {
         await this.explorBot.printSessionAnalysis();
-        await this.explorBot.getExplorer().stop();
+        await this.explorBot.stop();
         if (Stats.hasActivity()) {
             await new Promise((resolve) => {
                 const { unmount } = render(React.createElement(StatusPane, {

package/dist/src/explorbot.js

@@ -114,6 +114,7 @@ export class ExplorBot {
     async stop() {
         this.agents.quartermaster?.stop();
         await this.explorer?.stop();
+        await this.provider?.stop();
     }
     async visitInitialState() {
         await this.visit(this.options.from || '/');

package/dist/src/state-manager.d.ts

@@ -1,6 +1,7 @@
 import { ActionResult, type FocusedElement } from './action-result.js';
 import type { ExperienceTracker } from './experience-tracker.js';
 import type { Knowledge, KnowledgeTracker } from './knowledge-tracker.js';
+import { Overlay } from './utils/overlay.js';
 export interface Link {
     title: string;
     url: string;
@@ -40,6 +41,7 @@ export interface WebPageState {
     focusedElement?: FocusedElement | null;
     links?: Link[];
     verifications?: Record<string, boolean>;
+    overlay?: Overlay;
 }
 export interface StateTransition {
     /** Previous state (null if this is the first state) */

package/dist/src/state-manager.js

@@ -1,6 +1,6 @@
 import { ActionResult } from './action-result.js';
-import { detectFocusArea } from './utils/aria.js';
 import { createDebug, tag } from './utils/logger.js';
+import { Overlay } from './utils/overlay.js';
 import { slugify } from './utils/strings.js';
 import { extractStatePath } from './utils/url-matcher.js';
 const debugLog = createDebug('explorbot:state');
@@ -114,8 +114,8 @@ export class StateManager {
         return newState;
     }
     hasDialogAppeared(previousState, newState) {
-        const prevFocus = detectFocusArea(previousState?.ariaSnapshot ?? null);
-        const newFocus = detectFocusArea(newState.ariaSnapshot ?? null);
+        const prevFocus = previousState?.overlay ?? Overlay.fromAria(previousState?.ariaSnapshot ?? null);
+        const newFocus = newState.overlay ?? Overlay.fromAria(newState.ariaSnapshot ?? null);
         return !prevFocus.detected && newFocus.detected;
     }
     /**

package/dist/src/utils/html.d.ts

@@ -14,7 +14,6 @@ export declare const HTML_SELECTORS: {
     readonly interactiveControl: "button, a[href], input, select, textarea, [role=\"button\"], [role=\"link\"], [role=\"checkbox\"], [role=\"radio\"], [role=\"switch\"], [role=\"tab\"], [role=\"menuitem\"]";
     readonly labelLike: "h1, h2, h3, h4, h5, h6, legend, caption, label, [role=\"heading\"], [class*=\"title\"], [class*=\"label\"], [class*=\"header\"], [class*=\"name\"]";
     readonly semanticContextContainer: "section, article, form, fieldset, li, tr, td, th, [role=\"group\"], [role=\"tabpanel\"], [role=\"region\"], [class*=\"card\"], [class*=\"panel\"], [class*=\"item\"], [class*=\"usage\"], [class*=\"group\"]";
-    readonly semanticOverlays: readonly ["[role=\"dialog\"]", "[role=\"listbox\"]", "[role=\"menu\"]", "[role=\"tooltip\"]:not([style*=\"display: none\"]):not([style*=\"visibility: hidden\"])"];
 };
 export declare const HTML_VISIBILITY_LIMITS: {
     readonly maxViewportOverlayRatio: 0.95;
@@ -67,7 +66,9 @@ export type VisibleOverlayExtractionConfig = {
     interactiveContentSelector: string;
     limits: typeof HTML_EXTRACTION_LIMITS;
     overlaySelectors: readonly string[];
+    overlaySemanticSelector: string;
     visibilityLimits: typeof HTML_VISIBILITY_LIMITS;
+    geometryFallback?: boolean;
 };
 export type ComponentScopeExtractionConfig = {
     eidxAttr: string;

package/dist/src/utils/html.js

@@ -79,7 +79,6 @@ export const HTML_SELECTORS = {
     interactiveControl: 'button, a[href], input, select, textarea, [role="button"], [role="link"], [role="checkbox"], [role="radio"], [role="switch"], [role="tab"], [role="menuitem"]',
     labelLike: 'h1, h2, h3, h4, h5, h6, legend, caption, label, [role="heading"], [class*="title"], [class*="label"], [class*="header"], [class*="name"]',
     semanticContextContainer: 'section, article, form, fieldset, li, tr, td, th, [role="group"], [role="tabpanel"], [role="region"], [class*="card"], [class*="panel"], [class*="item"], [class*="usage"], [class*="group"]',
-    semanticOverlays: ['[role="dialog"]', '[role="listbox"]', '[role="menu"]', '[role="tooltip"]:not([style*="display: none"]):not([style*="visibility: hidden"])'],
 };
 export const HTML_VISIBILITY_LIMITS = {
     maxViewportOverlayRatio: 0.95,
@@ -447,8 +446,12 @@ export function extractVisibleOverlayHtml(config) {
         const { interactiveCount, text } = getUsefulContent(element);
         return interactiveCount > 0 || text.length > 0;
     }
-    const overlays = [];
+    function isFloatingOverlay(element) {
+        const style = window.getComputedStyle(element);
+        return style.position === 'fixed' || style.position === 'absolute' || Number.parseInt(style.zIndex || '0', 10) > 0;
+    }
     const seen = new Set();
+    const collected = [];
     for (const selector of config.overlaySelectors) {
         for (const element of Array.from(document.querySelectorAll(selector))) {
             if (seen.has(element))
@@ -456,13 +459,16 @@ export function extractVisibleOverlayHtml(config) {
             seen.add(element);
             if (!isVisible(element))
                 continue;
+            if (!element.matches(config.overlaySemanticSelector) && !isFloatingOverlay(element))
+                continue;
             const { interactiveCount, text } = getUsefulContent(element);
             if (interactiveCount === 0 && text.length === 0)
                 continue;
-            overlays.push(element.outerHTML.slice(0, config.limits.overlayHtmlLength));
+            collected.push(element);
         }
     }
-    if (overlays.length === 0) {
+    const overlays = collected.filter((element) => !collected.some((other) => other !== element && element.contains(other))).map((element) => element.outerHTML.slice(0, config.limits.overlayHtmlLength));
+    if (overlays.length === 0 && config.geometryFallback !== false) {
         const floatingCandidates = Array.from(document.body.querySelectorAll('*'))
             .filter((element) => !seen.has(element) && isVisible(element) && isLikelyFloatingOverlay(element))
             .sort((left, right) => {

package/dist/src/utils/overlay.d.ts

@@ -0,0 +1,24 @@
+import { type VisibleOverlayExtractionConfig } from './html.js';
+export declare const OVERLAY_SELECTORS: {
+    readonly semanticOverlays: readonly ["[role=\"dialog\"]", "[role=\"listbox\"]", "[role=\"menu\"]", "[role=\"tooltip\"]:not([style*=\"display: none\"]):not([style*=\"visibility: hidden\"])", "[class*=\"modal\"]", "[class*=\"dialog\"]", "[class*=\"overlay\"]", "[class*=\"popup\"]", "[class*=\"drawer\"]", "[class*=\"lightbox\"]"];
+    readonly modalOverlays: readonly ["[role=\"dialog\"]", "[role=\"alertdialog\"]", "[aria-modal=\"true\"]", "[class*=\"modal\"]", "[class*=\"dialog\"]", "[class*=\"overlay\"]", "[class*=\"popup\"]", "[class*=\"drawer\"]", "[class*=\"lightbox\"]"];
+    readonly overlaySemanticSelector: "[role=\"dialog\"], [role=\"alertdialog\"], [aria-modal=\"true\"], [role=\"listbox\"], [role=\"menu\"], [role=\"tooltip\"]";
+};
+export type OverlayData = {
+    type?: 'dialog' | 'modal' | null;
+    name?: string | null;
+};
+export declare class Overlay {
+    readonly type: 'dialog' | 'modal' | null;
+    readonly name: string | null;
+    constructor(data?: OverlayData);
+    get detected(): boolean;
+    static fromHtml(html: string): Overlay;
+    static fromAria(snapshot: string | null): Overlay;
+    static resolve(data: {
+        overlayHtml?: string;
+        overlay?: OverlayData | null;
+        ariaSnapshot?: string | null;
+    }): Overlay;
+    static captureConfig(): VisibleOverlayExtractionConfig;
+}

package/dist/src/utils/overlay.js

@@ -0,0 +1,43 @@
+import { detectFocusArea } from './aria.js';
+import { HTML_EXTRACTION_LIMITS, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, extractHeadings } from './html.js';
+export const OVERLAY_SELECTORS = {
+    semanticOverlays: ['[role="dialog"]', '[role="listbox"]', '[role="menu"]', '[role="tooltip"]:not([style*="display: none"]):not([style*="visibility: hidden"])', '[class*="modal"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="popup"]', '[class*="drawer"]', '[class*="lightbox"]'],
+    modalOverlays: ['[role="dialog"]', '[role="alertdialog"]', '[aria-modal="true"]', '[class*="modal"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="popup"]', '[class*="drawer"]', '[class*="lightbox"]'],
+    overlaySemanticSelector: '[role="dialog"], [role="alertdialog"], [aria-modal="true"], [role="listbox"], [role="menu"], [role="tooltip"]',
+};
+export class Overlay {
+    type;
+    name;
+    constructor(data = {}) {
+        this.type = data.type ?? null;
+        this.name = data.name ?? null;
+    }
+    get detected() {
+        return this.type !== null;
+    }
+    static fromHtml(html) {
+        const headings = extractHeadings(html);
+        const name = [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean).join(' ');
+        return new Overlay({ type: 'modal', name: name || null });
+    }
+    static fromAria(snapshot) {
+        return new Overlay(detectFocusArea(snapshot));
+    }
+    static resolve(data) {
+        if (data.overlayHtml)
+            return Overlay.fromHtml(data.overlayHtml);
+        if (data.overlay)
+            return new Overlay(data.overlay);
+        return Overlay.fromAria(data.ariaSnapshot ?? null);
+    }
+    static captureConfig() {
+        return {
+            interactiveContentSelector: HTML_SELECTORS.interactiveContent,
+            limits: HTML_EXTRACTION_LIMITS,
+            overlaySelectors: OVERLAY_SELECTORS.modalOverlays,
+            overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector,
+            visibilityLimits: HTML_VISIBILITY_LIMITS,
+            geometryFallback: false,
+        };
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/src/action-result.ts

@@ -7,6 +7,7 @@ import { TTLCache } from './utils/cache.ts';
 import { type HtmlDiffPart, type HtmlDiffResult, htmlDiff, liveRegionMessages } from './utils/html-diff.ts';
 import { extractHeadings, extractLinks, extractTargetedHtml, htmlCombinedSnapshot, htmlMinimalUISnapshot, htmlTextSnapshot, minifyHtml } from './utils/html.ts';
 import { createDebug } from './utils/logger.ts';
+import { Overlay } from './utils/overlay.ts';
 import { slugify } from './utils/strings.ts';
 import { extractStatePath, matchesUrl } from './utils/url-matcher.ts';
 
@@ -34,6 +35,7 @@ interface ActionResultData extends WebPageState {
   focusedElement?: FocusedElement | null;
   iframeURL?: string;
   links?: Link[];
+  overlayHtml?: string;
 }
 
 export interface PageDiff {
@@ -86,6 +88,7 @@ export class ActionResult implements ActionResultData {
   notes: string[] = [];
   public links: Link[] = [];
   public verifications?: Record<string, boolean>;
+  public overlay: Overlay = new Overlay();
 
   constructor(data: ActionResultData) {
     this.id = data.id;
@@ -131,6 +134,8 @@ export class ActionResult implements ActionResultData {
       this._ariaSnapshot = data.ariaSnapshot;
     }
 
+    this.overlay = Overlay.resolve(data);
+
     if (!this.fullUrl && this.url) {
       this.fullUrl = this.url;
     }

package/src/action.ts

@@ -11,8 +11,9 @@ import { Observability } from './observability.ts';
 import type { PlaywrightRecorder } from './playwright-recorder.ts';
 import type { StateManager } from './state-manager.js';
 import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts';
-import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
+import { captureHtmlForSnapshot, getVisibleOverlayHtmlExtractorSource, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
 import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
+import { Overlay } from './utils/overlay.js';
 import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
 import { safeFilename } from './utils/strings.ts';
 import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
@@ -144,11 +145,13 @@ class Action {
       let ariaSnapshot: string | null = null;
       let ariaSnapshotFile: string | undefined = undefined;
       let focusedElement: FocusedElement | null = null;
+      let overlayHtml = '';
 
       try {
         const page = this.playwrightHelper.page;
         ariaSnapshot = await page.locator('body').ariaSnapshot();
         focusedElement = await page.evaluate(readFocusedElement);
+        if (!frame) overlayHtml = await this.captureOverlayHtml();
       } catch (err) {
         debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
       }
@@ -177,6 +180,7 @@ class Action {
         ariaSnapshot,
         ariaSnapshotFile,
         focusedElement,
+        overlayHtml: overlayHtml || undefined,
         iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
       });
       this.stateManager.updateState(result, codeBlock);
@@ -190,6 +194,16 @@ class Action {
     }
   }
 
+  private async captureOverlayHtml(): Promise<string> {
+    return this.playwrightHelper.page.evaluate(
+      ({ extractorSource, config }: { extractorSource: string; config: any }) => {
+        const extract = new Function(`return ${extractorSource}`)() as (config: any) => string;
+        return extract(config);
+      },
+      { extractorSource: getVisibleOverlayHtmlExtractorSource(), config: Overlay.captureConfig() }
+    );
+  }
+
   private async captureMainDocumentStatus(): Promise<number | undefined> {
     if (this.mainDocumentStatus) return this.mainDocumentStatus;
 

package/src/ai/driller.ts

@@ -22,6 +22,7 @@ import {
 } from '../utils/html.ts';
 import { createDebug, tag } from '../utils/logger.ts';
 import { loop, pause } from '../utils/loop.ts';
+import { OVERLAY_SELECTORS } from '../utils/overlay.ts';
 import { annotatePageElements } from '../utils/web-annotate.ts';
 import { eidxInContainer } from '../utils/web-eidx.ts';
 import { WebElement } from '../utils/web-element.ts';
@@ -682,7 +683,8 @@ export class Driller extends TaskAgent implements Agent {
           config: {
             interactiveContentSelector: HTML_SELECTORS.interactiveContent,
             limits: HTML_EXTRACTION_LIMITS,
-            overlaySelectors: HTML_SELECTORS.semanticOverlays,
+            overlaySelectors: OVERLAY_SELECTORS.semanticOverlays,
+            overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector,
             visibilityLimits: HTML_VISIBILITY_LIMITS,
           },
         }

package/src/ai/pilot.ts

@@ -9,7 +9,7 @@ import type { PlaywrightRecorder } from '../playwright-recorder.ts';
 import type { StateManager } from '../state-manager.ts';
 import { Stats } from '../stats.ts';
 import { type Test, TestResult } from '../test-plan.ts';
-import { collectInteractiveNodes, detectFocusArea } from '../utils/aria.ts';
+import { collectInteractiveNodes } from '../utils/aria.ts';
 import { ErrorPageError } from '../utils/error-page.ts';
 import { createDebug, tag } from '../utils/logger.ts';
 
@@ -824,7 +824,7 @@ export class Pilot implements Agent {
     lines.push(`h3: ${state.h3 || ''}`);
     lines.push(`h4: ${state.h4 || ''}`);
 
-    const focusArea = detectFocusArea(state.ariaSnapshot);
+    const focusArea = state.overlay;
     if (focusArea.detected) {
       lines.push(`modal: ${focusArea.name || focusArea.type}`);
     } else {

package/src/ai/provider.ts

@@ -37,6 +37,15 @@ function createHarmonyChannelFallbackTool() {
 }
 
 let telemetryRegistered = false;
+let beforeExitFlushHooked = false;
+let activeOtelSdk: NodeSDK | null = null;
+
+export async function flushTelemetry(): Promise<void> {
+  const sdk = activeOtelSdk;
+  activeOtelSdk = null;
+  if (!sdk) return;
+  await sdk.shutdown().catch((error) => debugLog(`Telemetry flush failed: ${error instanceof Error ? error.message : error}`));
+}
 
 const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
 
@@ -117,6 +126,10 @@ export class Provider {
     }
   }
 
+  async stop(): Promise<void> {
+    await flushTelemetry();
+  }
+
   getModelForAgent(agentName?: string): any {
     if (!agentName) {
       return this.config.model;
@@ -270,7 +283,12 @@ export class Provider {
       spanProcessors: [processor],
       instrumentations: [],
     });
+    activeOtelSdk = this.otelSdk;
     void this.otelSdk.start();
+    if (!beforeExitFlushHooked) {
+      process.on('beforeExit', () => void flushTelemetry());
+      beforeExitFlushHooked = true;
+    }
     if (!telemetryRegistered) {
       registerTelemetry(new OpenTelemetry());
       telemetryRegistered = true;
@@ -324,7 +342,7 @@ export class Provider {
   async invokeConversation(conversation: Conversation, tools?: any, options: any = {}): Promise<{ conversation: Conversation; response: any; toolExecutions?: any[] } | null> {
     const response = tools ? await this.generateWithTools(conversation.messages, conversation.model, tools, options) : await this.chat(conversation.messages, conversation.model, options);
 
-    const responseMessages = response.response?.messages || [];
+    const responseMessages = response.responseMessages || [];
     if (responseMessages.length > 0) {
       conversation.messages.push(...responseMessages);
       tag('debug').log('Added', responseMessages.length, 'messages from response');

package/src/ai/researcher/deep-analysis.ts

@@ -5,7 +5,7 @@ import { executionController } from '../../execution-controller.ts';
 import type Explorer from '../../explorer.ts';
 import type { StateManager } from '../../state-manager.js';
 import { WebPageState } from '../../state-manager.js';
-import { detectFocusArea, diffAriaSnapshots } from '../../utils/aria.ts';
+import { diffAriaSnapshots } from '../../utils/aria.ts';
 import { extractCodeBlocks } from '../../utils/code-extractor.ts';
 import { tag } from '../../utils/logger.js';
 import { mdq } from '../../utils/markdown-query.ts';
@@ -86,7 +86,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
     }
 
     async researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise<string | null> {
-      const focusArea = detectFocusArea(current.ariaSnapshot);
+      const focusArea = current.overlay;
       if (!focusArea.detected || !focusArea.name) return null;
       if (focusArea.type !== 'dialog' && focusArea.type !== 'modal') return null;
 

package/src/ai/tester.ts

@@ -11,7 +11,6 @@ import { Observability } from '../observability.ts';
 import { type StateTransition, normalizeUrl } from '../state-manager.ts';
 import { Stats } from '../stats.ts';
 import { type Test, TestResult, type TestResultType } from '../test-plan.ts';
-import { detectFocusArea } from '../utils/aria.ts';
 import { ErrorPageError, isErrorPage } from '../utils/error-page.ts';
 import { createDebug, tag } from '../utils/logger.ts';
 import { loop } from '../utils/loop.ts';
@@ -531,7 +530,7 @@ export class Tester extends TaskAgent implements Agent {
 
     let context = '';
 
-    const focusArea = detectFocusArea(currentState.ariaSnapshot);
+    const focusArea = currentState.overlay;
 
     const focusedElement = currentState.focusedElement;
     if (focusedElement) {

package/src/commands/exit-command.ts

@@ -11,7 +11,7 @@ export class ExitCommand extends BaseCommand {
 
   async execute(_args: string): Promise<void> {
     await this.explorBot.printSessionAnalysis();
-    await this.explorBot.getExplorer().stop();
+    await this.explorBot.stop();
 
     if (Stats.hasActivity()) {
       await new Promise<void>((resolve) => {

package/src/explorbot.ts

@@ -142,6 +142,7 @@ export class ExplorBot {
   async stop(): Promise<void> {
     this.agents.quartermaster?.stop();
     await this.explorer?.stop();
+    await this.provider?.stop();
   }
 
   async visitInitialState(): Promise<void> {

package/src/state-manager.ts

@@ -1,8 +1,8 @@
 import { ActionResult, type FocusedElement } from './action-result.js';
 import type { ExperienceTracker } from './experience-tracker.js';
 import type { Knowledge, KnowledgeTracker } from './knowledge-tracker.js';
-import { detectFocusArea } from './utils/aria.js';
 import { createDebug, tag } from './utils/logger.js';
+import { Overlay } from './utils/overlay.js';
 import { slugify } from './utils/strings.js';
 import { extractStatePath } from './utils/url-matcher.js';
 
@@ -49,6 +49,7 @@ export interface WebPageState {
   focusedElement?: FocusedElement | null;
   links?: Link[];
   verifications?: Record<string, boolean>;
+  overlay?: Overlay;
 }
 
 export interface StateTransition {
@@ -206,8 +207,8 @@ export class StateManager {
   }
 
   private hasDialogAppeared(previousState: WebPageState | null, newState: WebPageState): boolean {
-    const prevFocus = detectFocusArea(previousState?.ariaSnapshot ?? null);
-    const newFocus = detectFocusArea(newState.ariaSnapshot ?? null);
+    const prevFocus = previousState?.overlay ?? Overlay.fromAria(previousState?.ariaSnapshot ?? null);
+    const newFocus = newState.overlay ?? Overlay.fromAria(newState.ariaSnapshot ?? null);
     return !prevFocus.detected && newFocus.detected;
   }
 

package/src/utils/html.ts

@@ -98,7 +98,6 @@ export const HTML_SELECTORS = {
   interactiveControl: 'button, a[href], input, select, textarea, [role="button"], [role="link"], [role="checkbox"], [role="radio"], [role="switch"], [role="tab"], [role="menuitem"]',
   labelLike: 'h1, h2, h3, h4, h5, h6, legend, caption, label, [role="heading"], [class*="title"], [class*="label"], [class*="header"], [class*="name"]',
   semanticContextContainer: 'section, article, form, fieldset, li, tr, td, th, [role="group"], [role="tabpanel"], [role="region"], [class*="card"], [class*="panel"], [class*="item"], [class*="usage"], [class*="group"]',
-  semanticOverlays: ['[role="dialog"]', '[role="listbox"]', '[role="menu"]', '[role="tooltip"]:not([style*="display: none"]):not([style*="visibility: hidden"])'],
 } as const;
 
 export const HTML_VISIBILITY_LIMITS = {
@@ -163,7 +162,9 @@ export type VisibleOverlayExtractionConfig = {
   interactiveContentSelector: string;
   limits: typeof HTML_EXTRACTION_LIMITS;
   overlaySelectors: readonly string[];
+  overlaySemanticSelector: string;
   visibilityLimits: typeof HTML_VISIBILITY_LIMITS;
+  geometryFallback?: boolean;
 };
 export type ComponentScopeExtractionConfig = {
   eidxAttr: string;
@@ -479,20 +480,28 @@ export function extractVisibleOverlayHtml(config: VisibleOverlayExtractionConfig
     return interactiveCount > 0 || text.length > 0;
   }
 
-  const overlays: string[] = [];
+  function isFloatingOverlay(element: Element): boolean {
+    const style = window.getComputedStyle(element as HTMLElement);
+    return style.position === 'fixed' || style.position === 'absolute' || Number.parseInt(style.zIndex || '0', 10) > 0;
+  }
+
   const seen = new Set<Element>();
+  const collected: Element[] = [];
   for (const selector of config.overlaySelectors) {
     for (const element of Array.from(document.querySelectorAll(selector))) {
       if (seen.has(element)) continue;
       seen.add(element);
       if (!isVisible(element)) continue;
+      if (!element.matches(config.overlaySemanticSelector) && !isFloatingOverlay(element)) continue;
       const { interactiveCount, text } = getUsefulContent(element);
       if (interactiveCount === 0 && text.length === 0) continue;
-      overlays.push((element as HTMLElement).outerHTML.slice(0, config.limits.overlayHtmlLength));
+      collected.push(element);
     }
   }
 
-  if (overlays.length === 0) {
+  const overlays = collected.filter((element) => !collected.some((other) => other !== element && element.contains(other))).map((element) => (element as HTMLElement).outerHTML.slice(0, config.limits.overlayHtmlLength));
+
+  if (overlays.length === 0 && config.geometryFallback !== false) {
     const floatingCandidates = Array.from(document.body.querySelectorAll('*'))
       .filter((element) => !seen.has(element) && isVisible(element) && isLikelyFloatingOverlay(element))
       .sort((left, right) => {

package/src/utils/overlay.ts

@@ -0,0 +1,51 @@
+import { detectFocusArea } from './aria.js';
+import { HTML_EXTRACTION_LIMITS, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, type VisibleOverlayExtractionConfig, extractHeadings } from './html.js';
+
+export const OVERLAY_SELECTORS = {
+  semanticOverlays: ['[role="dialog"]', '[role="listbox"]', '[role="menu"]', '[role="tooltip"]:not([style*="display: none"]):not([style*="visibility: hidden"])', '[class*="modal"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="popup"]', '[class*="drawer"]', '[class*="lightbox"]'],
+  modalOverlays: ['[role="dialog"]', '[role="alertdialog"]', '[aria-modal="true"]', '[class*="modal"]', '[class*="dialog"]', '[class*="overlay"]', '[class*="popup"]', '[class*="drawer"]', '[class*="lightbox"]'],
+  overlaySemanticSelector: '[role="dialog"], [role="alertdialog"], [aria-modal="true"], [role="listbox"], [role="menu"], [role="tooltip"]',
+} as const;
+
+export type OverlayData = { type?: 'dialog' | 'modal' | null; name?: string | null };
+
+export class Overlay {
+  readonly type: 'dialog' | 'modal' | null;
+  readonly name: string | null;
+
+  constructor(data: OverlayData = {}) {
+    this.type = data.type ?? null;
+    this.name = data.name ?? null;
+  }
+
+  get detected(): boolean {
+    return this.type !== null;
+  }
+
+  static fromHtml(html: string): Overlay {
+    const headings = extractHeadings(html);
+    const name = [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean).join(' ');
+    return new Overlay({ type: 'modal', name: name || null });
+  }
+
+  static fromAria(snapshot: string | null): Overlay {
+    return new Overlay(detectFocusArea(snapshot));
+  }
+
+  static resolve(data: { overlayHtml?: string; overlay?: OverlayData | null; ariaSnapshot?: string | null }): Overlay {
+    if (data.overlayHtml) return Overlay.fromHtml(data.overlayHtml);
+    if (data.overlay) return new Overlay(data.overlay);
+    return Overlay.fromAria(data.ariaSnapshot ?? null);
+  }
+
+  static captureConfig(): VisibleOverlayExtractionConfig {
+    return {
+      interactiveContentSelector: HTML_SELECTORS.interactiveContent,
+      limits: HTML_EXTRACTION_LIMITS,
+      overlaySelectors: OVERLAY_SELECTORS.modalOverlays,
+      overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector,
+      visibilityLimits: HTML_VISIBILITY_LIMITS,
+      geometryFallback: false,
+    };
+  }
+}