explorbot 0.1.15 → 0.1.17

package/src/ai/researcher/focus.ts

@@ -1,4 +1,4 @@
-import { detectFocusArea } from '../../utils/aria.ts';
+import type { Page } from 'playwright';
 import { mdq } from '../../utils/markdown-query.ts';
 import type { ResearchSection } from './parser.ts';
 import type { ResearchResult } from './research-result.ts';
@@ -10,16 +10,65 @@ export function hasFocusedSection(text: string): boolean {
   return text.includes(FOCUSED_MARKER);
 }
 
-export function detectFocusFromAria(ariaSnapshot: string | null, sections: ResearchSection[]): string | null {
-  const focusArea = detectFocusArea(ariaSnapshot);
-  if (!focusArea.detected) return null;
+interface FocusProbe {
+  name: string;
+  isDialog: boolean;
+  zIndex: number;
+  hasShadow: boolean;
+}
+
+export async function detectFocusedSection(page: Page, sections: ResearchSection[]): Promise<string | null> {
+  const candidates: FocusProbe[] = [];
+
+  for (const section of sections) {
+    if (!section.containerCss) continue;
+    const key = section.name.toLowerCase().replace(/^section:\s*/, '');
+    if (FOCUS_SKIP_SECTIONS.has(key)) continue;
+
+    try {
+      const locator = page.locator(section.containerCss).first();
+      if (!(await locator.isVisible())) continue;
+
+      const probe = await locator.evaluate((el) => {
+        const dialogSelector = '[role="dialog"], [role="alertdialog"], [aria-modal="true"]';
+        const isDialog = el.matches(dialogSelector) || !!el.querySelector(dialogSelector);
+
+        let cur: Element | null = el;
+        let maxZ = 0;
+        while (cur && cur !== document.body) {
+          const cs = window.getComputedStyle(cur);
+          if (cs.position !== 'static') {
+            const z = Number.parseInt(cs.zIndex, 10);
+            if (!Number.isNaN(z) && z > maxZ) maxZ = z;
+          }
+          cur = cur.parentElement;
+        }
 
-  if (focusArea.type === 'dialog' || focusArea.type === 'modal') {
-    const dialogSection = sections.find((s) => s.containerCss && (s.containerCss.includes('[role="dialog"]') || s.containerCss.includes('[role="alertdialog"]') || s.containerCss.includes('[aria-modal')));
-    if (dialogSection) return dialogSection.name;
+        const shadow = window.getComputedStyle(el).boxShadow;
+        const hasShadow = !!shadow && shadow !== 'none';
+
+        return { isDialog, zIndex: maxZ, hasShadow };
+      });
+
+      candidates.push({ name: section.name, ...probe });
+    } catch {}
   }
 
-  return null;
+  if (candidates.length === 0) return null;
+
+  const dialogs = candidates.filter((c) => c.isDialog);
+  const pool = dialogs.length > 0 ? dialogs : candidates;
+
+  const winner = pool.reduce<FocusProbe | null>((best, c) => {
+    if (!best) return c;
+    if (c.zIndex !== best.zIndex) return c.zIndex > best.zIndex ? c : best;
+    if (c.hasShadow !== best.hasShadow) return c.hasShadow ? c : best;
+    return best;
+  }, null);
+
+  if (!winner) return null;
+  if (dialogs.length === 0 && winner.zIndex === 0 && !winner.hasShadow) return null;
+  return winner.name;
 }
 
 export function markSectionAsFocused(result: ResearchResult, sectionName: string): void {

package/src/ai/researcher/sections.ts

@@ -7,7 +7,9 @@ import { tag } from '../../utils/logger.js';
 import { RulesLoader } from '../../utils/rules-loader.ts';
 import type { Provider } from '../provider.js';
 import { locatorRule as generalLocatorRuleText } from '../rules.js';
+import { markSectionAsFocused } from './focus.ts';
 import type { Constructor } from './mixin.ts';
+import { ResearchResult } from './research-result.ts';
 
 export interface SectionMethods {
   researchBySections(): Promise<string>;
@@ -54,9 +56,11 @@ export function WithSections<T extends Constructor>(Base: T) {
         throw new Error('Per-section research produced no sections — AI responses all empty or NOT_PRESENT');
       }
 
-      let merged = parts.join('\n\n');
-      if (focusCss) merged += '\n\n> Focused: Focus';
-      return merged;
+      const merged = parts.join('\n\n');
+      if (!focusCss) return merged;
+      const focused = new ResearchResult(merged, this.actionResult?.url || '');
+      markSectionAsFocused(focused, 'Focus');
+      return focused.text;
     }
 
     private async _detectFocusCss(): Promise<string | null> {

package/src/ai/researcher.ts

@@ -24,7 +24,7 @@ import { ContextLengthError, type Provider } from './provider.js';
 import { findSimilarResearch, getCachedResearch, saveResearch } from './researcher/cache.ts';
 import { type CoordinateMethods, WithCoordinates } from './researcher/coordinates.ts';
 import { type DeepAnalysisMethods, WithDeepAnalysis } from './researcher/deep-analysis.ts';
-import { detectFocusFromAria, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from './researcher/focus.ts';
+import { detectFocusedSection, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from './researcher/focus.ts';
 import { type LocatorMethods, WithLocators } from './researcher/locators.ts';
 import { extractValidContainers, formatResearchSummary, parseResearchSections } from './researcher/parser.ts';
 import { ResearchResult } from './researcher/research-result.ts';
@@ -234,17 +234,12 @@ export class Researcher extends ResearcherBase implements Agent {
         await this.fixBrokenSections(result, activeConversation);
       }
 
-      // Focused section: parse AI declaration, then ARIA fallback
-      const focusMatch = result.text.match(/^>\s*Focused:\s*(.+)/m);
-      if (focusMatch) {
-        result.text = result.text.replace(focusMatch[0], '');
-        markSectionAsFocused(result, focusMatch[1].trim());
-      }
-      if (!hasFocusedSection(result.text)) {
+      // Focused section: unified Playwright probe (HTML+CSS+visibility).
+      // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
+      if (!interrupted() && this.hasScreenshotToAnalyze) {
         const sections = parseResearchSections(result.text);
-        const ariaSnapshot = this.actionResult?.getCompactARIA() || '';
-        const focusedName = detectFocusFromAria(ariaSnapshot, sections);
-        if (focusedName) markSectionAsFocused(result, focusedName);
+        const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
+        if (focused) markSectionAsFocused(result, focused);
       }
 
       // Stage 4: Visual analysis
@@ -281,8 +276,8 @@ export class Researcher extends ResearcherBase implements Agent {
         await this.backfillBrokenLocators(result);
       }
 
-      // Focused section: final fallback
-      if (!hasFocusedSection(result.text)) {
+      // Focused section: final fallback (vision-only — without a screenshot we don't infer focus)
+      if (this.hasScreenshotToAnalyze && !hasFocusedSection(result.text)) {
         const sections = parseResearchSections(result.text);
         const fallback = pickDefaultFocusedSection(sections);
         if (fallback) markSectionAsFocused(result, fallback);
@@ -451,16 +446,6 @@ export class Researcher extends ResearcherBase implements Agent {
 
       | Element | ARIA | CSS | eidx |
       </section_format>
-
-      <focused_section>
-      At the end of your output, declare the primary focus area on a single line:
-
-      > Focused: <exact section name>
-
-      - If a dialog/modal/drawer/overlay exists, it is focused.
-      - Otherwise pick the section where the main business action happens (list for catalog, detail for item page, content for article).
-      - Navigation and menu/toolbar are never focused.
-      </focused_section>
     `;
   }
 

package/src/ai/tester.ts

@@ -387,10 +387,15 @@ export class Tester extends TaskAgent implements Agent {
             : undefined,
           catch: async ({ error, stop }) => {
             tag('error').log(`Test execution error: ${error}`);
+            const message = error instanceof Error ? error.message : String(error);
             if (!task.hasFinished) {
-              task.addNote(`Execution error: ${error instanceof Error ? error.message : String(error)}`);
+              task.addNote(`Execution error: ${message}`);
             }
-            stop();
+            if (error instanceof Error && error.name === 'AbortError') {
+              stop();
+              return;
+            }
+            conversation.addUserText(`Previous AI call failed: ${message}. Take a different approach on the next step.`);
           },
         }
       );
@@ -725,6 +730,7 @@ export class Tester extends TaskAgent implements Agent {
     - Use pressKey() for pressing special keys (Enter, Escape, Tab, Arrow keys) or key combinations with modifiers (Ctrl+A, Shift+Delete, etc.)
     - Use container CSS locators from <page_ui_map> to interact with elements inside sections
     - Systematically use record({ notes: ["..."] }) to write your findings, planned actions, observations, etc.
+    - When creating/editing/deleting a named entity, include its identifier verbatim in the note — Pilot uses it to confirm provenance.
     - Call record({ notes: ["..."], status: "success" }) when you see success/info message on a page or when expected outcome is achieved
     - Call record({ notes: ["..."], status: "fail" }) when an expected outcome cannot be achieved or has failed or you see error/alert/warning message on a page
     - NEVER call record(status: "success") if your last verify() or see() call FAILED. A failed check means the outcome is NOT confirmed — use record(status: "fail") instead, or retry with a different approach.