explorbot 0.1.15 → 0.1.16

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.1.15",
+  "version": "0.1.16",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/dist/src/ai/researcher/focus.js

@@ -1,20 +1,61 @@
-import { detectFocusArea } from "../../utils/aria.js";
 import { mdq } from "../../utils/markdown-query.js";
 export const FOCUSED_MARKER = '> **Focused**';
 const FOCUS_SKIP_SECTIONS = new Set(['navigation', 'menu']);
 export function hasFocusedSection(text) {
     return text.includes(FOCUSED_MARKER);
 }
-export function detectFocusFromAria(ariaSnapshot, sections) {
-    const focusArea = detectFocusArea(ariaSnapshot);
-    if (!focusArea.detected)
-        return null;
-    if (focusArea.type === 'dialog' || focusArea.type === 'modal') {
-        const dialogSection = sections.find((s) => s.containerCss && (s.containerCss.includes('[role="dialog"]') || s.containerCss.includes('[role="alertdialog"]') || s.containerCss.includes('[aria-modal')));
-        if (dialogSection)
-            return dialogSection.name;
+export async function detectFocusedSection(page, sections) {
+    const candidates = [];
+    for (const section of sections) {
+        if (!section.containerCss)
+            continue;
+        const key = section.name.toLowerCase().replace(/^section:\s*/, '');
+        if (FOCUS_SKIP_SECTIONS.has(key))
+            continue;
+        try {
+            const locator = page.locator(section.containerCss).first();
+            if (!(await locator.isVisible()))
+                continue;
+            const probe = await locator.evaluate((el) => {
+                const dialogSelector = '[role="dialog"], [role="alertdialog"], [aria-modal="true"]';
+                const isDialog = el.matches(dialogSelector) || !!el.querySelector(dialogSelector);
+                let cur = el;
+                let maxZ = 0;
+                while (cur && cur !== document.body) {
+                    const cs = window.getComputedStyle(cur);
+                    if (cs.position !== 'static') {
+                        const z = Number.parseInt(cs.zIndex, 10);
+                        if (!Number.isNaN(z) && z > maxZ)
+                            maxZ = z;
+                    }
+                    cur = cur.parentElement;
+                }
+                const shadow = window.getComputedStyle(el).boxShadow;
+                const hasShadow = !!shadow && shadow !== 'none';
+                return { isDialog, zIndex: maxZ, hasShadow };
+            });
+            candidates.push({ name: section.name, ...probe });
+        }
+        catch { }
     }
-    return null;
+    if (candidates.length === 0)
+        return null;
+    const dialogs = candidates.filter((c) => c.isDialog);
+    const pool = dialogs.length > 0 ? dialogs : candidates;
+    const winner = pool.reduce((best, c) => {
+        if (!best)
+            return c;
+        if (c.zIndex !== best.zIndex)
+            return c.zIndex > best.zIndex ? c : best;
+        if (c.hasShadow !== best.hasShadow)
+            return c.hasShadow ? c : best;
+        return best;
+    }, null);
+    if (!winner)
+        return null;
+    if (dialogs.length === 0 && winner.zIndex === 0 && !winner.hasShadow)
+        return null;
+    return winner.name;
 }
 export function markSectionAsFocused(result, sectionName) {
     if (hasFocusedSection(result.text))

package/dist/src/ai/researcher/sections.js

@@ -3,6 +3,8 @@ import { executionController } from "../../execution-controller.js";
 import { tag } from '../../utils/logger.js';
 import { RulesLoader } from "../../utils/rules-loader.js";
 import { locatorRule as generalLocatorRuleText } from '../rules.js';
+import { markSectionAsFocused } from "./focus.js";
+import { ResearchResult } from "./research-result.js";
 export function WithSections(Base) {
     return class extends Base {
         async researchBySections() {
@@ -40,10 +42,12 @@ export function WithSections(Base) {
             if (parts.length === 0) {
                 throw new Error('Per-section research produced no sections — AI responses all empty or NOT_PRESENT');
             }
-            let merged = parts.join('\n\n');
-            if (focusCss)
-                merged += '\n\n> Focused: Focus';
-            return merged;
+            const merged = parts.join('\n\n');
+            if (!focusCss)
+                return merged;
+            const focused = new ResearchResult(merged, this.actionResult?.url || '');
+            markSectionAsFocused(focused, 'Focus');
+            return focused.text;
         }
         async _detectFocusCss() {
             const focusSections = this.explorer.getConfig().ai?.agents?.researcher?.focusSections;

package/dist/src/ai/researcher.js

@@ -16,7 +16,7 @@ import { ContextLengthError } from './provider.js';
 import { findSimilarResearch, getCachedResearch, saveResearch } from "./researcher/cache.js";
 import { WithCoordinates } from "./researcher/coordinates.js";
 import { WithDeepAnalysis } from "./researcher/deep-analysis.js";
-import { detectFocusFromAria, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from "./researcher/focus.js";
+import { detectFocusedSection, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from "./researcher/focus.js";
 import { WithLocators } from "./researcher/locators.js";
 import { extractValidContainers, formatResearchSummary, parseResearchSections } from "./researcher/parser.js";
 import { ResearchResult } from "./researcher/research-result.js";
@@ -186,18 +186,13 @@ export class Researcher extends ResearcherBase {
             if (!interrupted() && fix && result.locators.some((l) => l.valid === false)) {
                 await this.fixBrokenSections(result, activeConversation);
             }
-            // Focused section: parse AI declaration, then ARIA fallback
-            const focusMatch = result.text.match(/^>\s*Focused:\s*(.+)/m);
-            if (focusMatch) {
-                result.text = result.text.replace(focusMatch[0], '');
-                markSectionAsFocused(result, focusMatch[1].trim());
-            }
-            if (!hasFocusedSection(result.text)) {
+            // Focused section: unified Playwright probe (HTML+CSS+visibility).
+            // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
+            if (!interrupted() && this.hasScreenshotToAnalyze) {
                 const sections = parseResearchSections(result.text);
-                const ariaSnapshot = this.actionResult?.getCompactARIA() || '';
-                const focusedName = detectFocusFromAria(ariaSnapshot, sections);
-                if (focusedName)
-                    markSectionAsFocused(result, focusedName);
+                const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
+                if (focused)
+                    markSectionAsFocused(result, focused);
             }
             // Stage 4: Visual analysis
             if (!interrupted() && this.hasScreenshotToAnalyze) {
@@ -232,8 +227,8 @@ export class Researcher extends ResearcherBase {
                 await this.backfillCoordinates(result);
                 await this.backfillBrokenLocators(result);
             }
-            // Focused section: final fallback
-            if (!hasFocusedSection(result.text)) {
+            // Focused section: final fallback (vision-only — without a screenshot we don't infer focus)
+            if (this.hasScreenshotToAnalyze && !hasFocusedSection(result.text)) {
                 const sections = parseResearchSections(result.text);
                 const fallback = pickDefaultFocusedSection(sections);
                 if (fallback)
@@ -388,16 +383,6 @@ export class Researcher extends ResearcherBase {
 
       | Element | ARIA | CSS | eidx |
       </section_format>
-
-      <focused_section>
-      At the end of your output, declare the primary focus area on a single line:
-
-      > Focused: <exact section name>
-
-      - If a dialog/modal/drawer/overlay exists, it is focused.
-      - Otherwise pick the section where the main business action happens (list for catalog, detail for item page, content for article).
-      - Navigation and menu/toolbar are never focused.
-      </focused_section>
     `;
     }
     async buildResearchPrompt() {

package/dist/src/ai/tester.js

@@ -329,10 +329,15 @@ export class Tester extends TaskAgent {
                     : undefined,
                 catch: async ({ error, stop }) => {
                     tag('error').log(`Test execution error: ${error}`);
+                    const message = error instanceof Error ? error.message : String(error);
                     if (!task.hasFinished) {
-                        task.addNote(`Execution error: ${error instanceof Error ? error.message : String(error)}`);
+                        task.addNote(`Execution error: ${message}`);
                     }
-                    stop();
+                    if (error instanceof Error && error.name === 'AbortError') {
+                        stop();
+                        return;
+                    }
+                    conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`);
                 },
             });
             if (task.hasFinished)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.1.15",
+  "version": "0.1.16",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/src/ai/researcher/focus.ts

@@ -1,4 +1,4 @@
-import { detectFocusArea } from '../../utils/aria.ts';
+import type { Page } from 'playwright';
 import { mdq } from '../../utils/markdown-query.ts';
 import type { ResearchSection } from './parser.ts';
 import type { ResearchResult } from './research-result.ts';
@@ -10,16 +10,65 @@ export function hasFocusedSection(text: string): boolean {
   return text.includes(FOCUSED_MARKER);
 }
 
-export function detectFocusFromAria(ariaSnapshot: string | null, sections: ResearchSection[]): string | null {
-  const focusArea = detectFocusArea(ariaSnapshot);
-  if (!focusArea.detected) return null;
+interface FocusProbe {
+  name: string;
+  isDialog: boolean;
+  zIndex: number;
+  hasShadow: boolean;
+}
+
+export async function detectFocusedSection(page: Page, sections: ResearchSection[]): Promise<string | null> {
+  const candidates: FocusProbe[] = [];
+
+  for (const section of sections) {
+    if (!section.containerCss) continue;
+    const key = section.name.toLowerCase().replace(/^section:\s*/, '');
+    if (FOCUS_SKIP_SECTIONS.has(key)) continue;
+
+    try {
+      const locator = page.locator(section.containerCss).first();
+      if (!(await locator.isVisible())) continue;
+
+      const probe = await locator.evaluate((el) => {
+        const dialogSelector = '[role="dialog"], [role="alertdialog"], [aria-modal="true"]';
+        const isDialog = el.matches(dialogSelector) || !!el.querySelector(dialogSelector);
+
+        let cur: Element | null = el;
+        let maxZ = 0;
+        while (cur && cur !== document.body) {
+          const cs = window.getComputedStyle(cur);
+          if (cs.position !== 'static') {
+            const z = Number.parseInt(cs.zIndex, 10);
+            if (!Number.isNaN(z) && z > maxZ) maxZ = z;
+          }
+          cur = cur.parentElement;
+        }
 
-  if (focusArea.type === 'dialog' || focusArea.type === 'modal') {
-    const dialogSection = sections.find((s) => s.containerCss && (s.containerCss.includes('[role="dialog"]') || s.containerCss.includes('[role="alertdialog"]') || s.containerCss.includes('[aria-modal')));
-    if (dialogSection) return dialogSection.name;
+        const shadow = window.getComputedStyle(el).boxShadow;
+        const hasShadow = !!shadow && shadow !== 'none';
+
+        return { isDialog, zIndex: maxZ, hasShadow };
+      });
+
+      candidates.push({ name: section.name, ...probe });
+    } catch {}
   }
 
-  return null;
+  if (candidates.length === 0) return null;
+
+  const dialogs = candidates.filter((c) => c.isDialog);
+  const pool = dialogs.length > 0 ? dialogs : candidates;
+
+  const winner = pool.reduce<FocusProbe | null>((best, c) => {
+    if (!best) return c;
+    if (c.zIndex !== best.zIndex) return c.zIndex > best.zIndex ? c : best;
+    if (c.hasShadow !== best.hasShadow) return c.hasShadow ? c : best;
+    return best;
+  }, null);
+
+  if (!winner) return null;
+  if (dialogs.length === 0 && winner.zIndex === 0 && !winner.hasShadow) return null;
+  return winner.name;
 }
 
 export function markSectionAsFocused(result: ResearchResult, sectionName: string): void {

package/src/ai/researcher/sections.ts

@@ -7,7 +7,9 @@ import { tag } from '../../utils/logger.js';
 import { RulesLoader } from '../../utils/rules-loader.ts';
 import type { Provider } from '../provider.js';
 import { locatorRule as generalLocatorRuleText } from '../rules.js';
+import { markSectionAsFocused } from './focus.ts';
 import type { Constructor } from './mixin.ts';
+import { ResearchResult } from './research-result.ts';
 
 export interface SectionMethods {
   researchBySections(): Promise<string>;
@@ -54,9 +56,11 @@ export function WithSections<T extends Constructor>(Base: T) {
         throw new Error('Per-section research produced no sections — AI responses all empty or NOT_PRESENT');
       }
 
-      let merged = parts.join('\n\n');
-      if (focusCss) merged += '\n\n> Focused: Focus';
-      return merged;
+      const merged = parts.join('\n\n');
+      if (!focusCss) return merged;
+      const focused = new ResearchResult(merged, this.actionResult?.url || '');
+      markSectionAsFocused(focused, 'Focus');
+      return focused.text;
     }
 
     private async _detectFocusCss(): Promise<string | null> {

package/src/ai/researcher.ts

@@ -24,7 +24,7 @@ import { ContextLengthError, type Provider } from './provider.js';
 import { findSimilarResearch, getCachedResearch, saveResearch } from './researcher/cache.ts';
 import { type CoordinateMethods, WithCoordinates } from './researcher/coordinates.ts';
 import { type DeepAnalysisMethods, WithDeepAnalysis } from './researcher/deep-analysis.ts';
-import { detectFocusFromAria, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from './researcher/focus.ts';
+import { detectFocusedSection, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from './researcher/focus.ts';
 import { type LocatorMethods, WithLocators } from './researcher/locators.ts';
 import { extractValidContainers, formatResearchSummary, parseResearchSections } from './researcher/parser.ts';
 import { ResearchResult } from './researcher/research-result.ts';
@@ -234,17 +234,12 @@ export class Researcher extends ResearcherBase implements Agent {
         await this.fixBrokenSections(result, activeConversation);
       }
 
-      // Focused section: parse AI declaration, then ARIA fallback
-      const focusMatch = result.text.match(/^>\s*Focused:\s*(.+)/m);
-      if (focusMatch) {
-        result.text = result.text.replace(focusMatch[0], '');
-        markSectionAsFocused(result, focusMatch[1].trim());
-      }
-      if (!hasFocusedSection(result.text)) {
+      // Focused section: unified Playwright probe (HTML+CSS+visibility).
+      // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
+      if (!interrupted() && this.hasScreenshotToAnalyze) {
         const sections = parseResearchSections(result.text);
-        const ariaSnapshot = this.actionResult?.getCompactARIA() || '';
-        const focusedName = detectFocusFromAria(ariaSnapshot, sections);
-        if (focusedName) markSectionAsFocused(result, focusedName);
+        const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
+        if (focused) markSectionAsFocused(result, focused);
       }
 
       // Stage 4: Visual analysis
@@ -281,8 +276,8 @@ export class Researcher extends ResearcherBase implements Agent {
         await this.backfillBrokenLocators(result);
       }
 
-      // Focused section: final fallback
-      if (!hasFocusedSection(result.text)) {
+      // Focused section: final fallback (vision-only — without a screenshot we don't infer focus)
+      if (this.hasScreenshotToAnalyze && !hasFocusedSection(result.text)) {
         const sections = parseResearchSections(result.text);
         const fallback = pickDefaultFocusedSection(sections);
         if (fallback) markSectionAsFocused(result, fallback);
@@ -451,16 +446,6 @@ export class Researcher extends ResearcherBase implements Agent {
 
       | Element | ARIA | CSS | eidx |
       </section_format>
-
-      <focused_section>
-      At the end of your output, declare the primary focus area on a single line:
-
-      > Focused: <exact section name>
-
-      - If a dialog/modal/drawer/overlay exists, it is focused.
-      - Otherwise pick the section where the main business action happens (list for catalog, detail for item page, content for article).
-      - Navigation and menu/toolbar are never focused.
-      </focused_section>
     `;
   }
 

package/src/ai/tester.ts

@@ -387,10 +387,15 @@ export class Tester extends TaskAgent implements Agent {
             : undefined,
           catch: async ({ error, stop }) => {
             tag('error').log(`Test execution error: ${error}`);
+            const message = error instanceof Error ? error.message : String(error);
             if (!task.hasFinished) {
-              task.addNote(`Execution error: ${error instanceof Error ? error.message : String(error)}`);
+              task.addNote(`Execution error: ${message}`);
             }
-            stop();
+            if (error instanceof Error && error.name === 'AbortError') {
+              stop();
+              return;
+            }
+            conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`);
           },
         }
       );