explorbot 0.2.0 → 0.2.2

package/src/ai/researcher/locators.ts

@@ -6,6 +6,7 @@ import { parseAriaLocator } from '../../utils/aria.ts';
 import { tag } from '../../utils/logger.js';
 import { mdq } from '../../utils/markdown-query.ts';
 import { WebElement } from '../../utils/web-element.ts';
+import { isDynamicId } from '../../utils/xpath.ts';
 import type { Conversation } from '../conversation.ts';
 import type { Provider } from '../provider.js';
 import { locatorRule as generalLocatorRuleText } from '../rules.js';
@@ -38,7 +39,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
     declare provider: Provider;
     declare actionResult: ActionResult | undefined;
 
-    async testLocators(locators: Locator[]): Promise<void> {
+    async testLocators(locators: Locator[], opts: { scope?: boolean } = {}): Promise<void> {
       let broken = 0;
       for (const loc of locators) {
         if (executionController.isInterrupted()) break;
@@ -51,20 +52,21 @@ export function WithLocators<T extends Constructor>(Base: T) {
           continue;
         }
         try {
-          const count = await this.explorer.playwrightLocatorCount((page) => {
+          const count = await this.explorer.withPage((page) => {
             const base = loc.container ? page.locator(loc.container) : page;
             if (loc.type === 'aria') {
               const parsed = parseAriaLocator(loc.locator);
-              if (!parsed) return page.locator('__invalid__');
-              return base.getByRole(parsed.role as any, { name: parsed.text });
+              if (!parsed) return page.locator('__invalid__').count();
+              return base.getByRole(parsed.role as any, { name: parsed.text }).count();
             }
             const converted = loc.locator.replace(/:contains\(/g, ':has-text(');
             if (converted !== loc.locator) {
               loc.locator = converted;
             }
-            return base.locator(loc.locator);
+            return base.locator(loc.locator).count();
           });
           loc.valid = count === 1;
+          if (opts.scope && count > 1) loc.valid = true;
           loc.pwLocator = buildPwLocatorString(loc);
           if (!loc.valid) {
             loc.error = count === 0 ? '0 elements' : `${count} elements`;
@@ -194,7 +196,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
       }
 
       if (needsXpath.length > 0) {
-        const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath));
+        const webElements = await this.explorer.withPage((page) => WebElement.fromEidxList(page, needsXpath));
         const changedSections = new Set<(typeof sections)[0]>();
         for (const w of webElements) {
           const entry = needsXpathEls.get(w.eidx!);
@@ -209,6 +211,60 @@ export function WithLocators<T extends Constructor>(Base: T) {
       await this.validateContainers(result);
     }
 
+    async resolveContainers(result: ResearchResult): Promise<Locator[]> {
+      const containerLocs = result.containerLocators;
+      await this.testLocators(containerLocs, { scope: true });
+      for (const loc of containerLocs.filter((l) => l.valid === false)) {
+        if (await this.recoverContainerFromChildren(result, loc.locator)) loc.valid = true;
+      }
+      return containerLocs.filter((l) => l.valid === false);
+    }
+
+    private async recoverContainerFromChildren(result: ResearchResult, css: string): Promise<boolean> {
+      const sections = parseResearchSections(result.text).filter((s) => s.containerCss === css);
+      let recovered = 0;
+
+      for (const section of sections) {
+        const eidxList = section.elements.map((el) => el.eidx).filter(Boolean) as string[];
+        if (eidxList.length < 2) continue;
+
+        const ancestor = await this.explorer.withPage((page) => WebElement.commonAncestor(page, eidxList));
+        if (!ancestor) continue;
+
+        const candidates: string[] = [];
+        const id = ancestor.attrs.id;
+        if (id && !isDynamicId(id)) candidates.push(`#${id}`);
+        for (const cls of ancestor.filteredClasses) {
+          if (!/^[\w-]+$/.test(cls)) continue;
+          candidates.push(`${ancestor.tag}.${cls}`);
+        }
+
+        let unique: string | null = null;
+        let multiple: string | null = null;
+        for (const candidate of candidates) {
+          let count = 0;
+          try {
+            count = await this.explorer.playwrightLocatorCount((page) => page.locator(candidate));
+          } catch {}
+          if (count === 1) {
+            unique = candidate;
+            break;
+          }
+          if (count > 1 && !multiple) multiple = candidate;
+        }
+
+        const newCss = unique || multiple;
+        if (!newCss) continue;
+
+        debugLog(`Recovered container: '${css}' → '${newCss}' in "${section.name}"`);
+        this.updateSectionContainer(result, section, newCss);
+        recovered++;
+      }
+
+      if (recovered > 0 && recovered < sections.length) debugLog(`Container '${css}' recovered in ${recovered}/${sections.length} sections, still broken for the rest`);
+      return recovered > 0 && recovered === sections.length;
+    }
+
     private async validateContainers(result: ResearchResult): Promise<void> {
       const sections = parseResearchSections(result.text);
 
@@ -217,7 +273,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
 
         let count = 0;
         try {
-          count = await this.explorer.playwrightLocatorCount((page) => page.locator(section.containerCss!));
+          count = await this.explorer.withPage((page) => page.locator(section.containerCss!).count());
         } catch {}
 
         if (count >= 1) continue;
@@ -226,7 +282,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
         if (simplified) {
           let simplifiedCount = 0;
           try {
-            simplifiedCount = await this.explorer.playwrightLocatorCount((page) => page.locator(simplified));
+            simplifiedCount = await this.explorer.withPage((page) => page.locator(simplified).count());
           } catch {}
 
           if (simplifiedCount >= 1) {
@@ -275,7 +331,8 @@ export interface Locator {
 }
 
 export interface LocatorMethods {
-  testLocators(locators: Locator[]): Promise<void>;
+  testLocators(locators: Locator[], opts?: { scope?: boolean }): Promise<void>;
   fixBrokenSections(result: ResearchResult, conversation: Conversation): Promise<void>;
   backfillBrokenLocators(result: ResearchResult): Promise<void>;
+  resolveContainers(result: ResearchResult): Promise<Locator[]>;
 }

package/src/ai/researcher/sections.ts

@@ -1,5 +1,6 @@
 import dedent from 'dedent';
 import type { ActionResult } from '../../action-result.js';
+import type { ExplorbotConfig } from '../../config.ts';
 import { executionController } from '../../execution-controller.ts';
 import type Explorer from '../../explorer.ts';
 import type { StateManager } from '../../state-manager.js';
@@ -20,6 +21,7 @@ export function WithSections<T extends Constructor>(Base: T) {
     declare explorer: Explorer;
     declare provider: Provider;
     declare stateManager: StateManager;
+    declare config: ExplorbotConfig;
     declare actionResult: ActionResult | undefined;
 
     async researchBySections(): Promise<string> {
@@ -64,11 +66,11 @@ export function WithSections<T extends Constructor>(Base: T) {
     }
 
     private async _detectFocusCss(): Promise<string | null> {
-      const focusSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.focusSections as string[] | undefined;
+      const focusSections = (this.config.ai?.agents?.researcher as any)?.focusSections as string[] | undefined;
       if (!focusSections?.length) return null;
 
       for (const css of focusSections) {
-        const count = await this.explorer.playwrightLocatorCount((page: any) => page.locator(css)).catch(() => 0);
+        const count = await this.explorer.withPage((page) => page.locator(css).count()).catch(() => 0);
         if (count > 0) return css;
       }
       return null;

package/src/ai/researcher.ts

@@ -1,23 +1,22 @@
 import dedent from 'dedent';
 import { ActionResult } from '../action-result.js';
 import { setActivity } from '../activity.ts';
-import { ConfigParser, outputPath } from '../config.ts';
+import { ConfigParser, type ExplorbotConfig, outputPath } from '../config.ts';
 import { executionController } from '../execution-controller.ts';
 import type { ExperienceTracker } from '../experience-tracker.ts';
 import type Explorer from '../explorer.ts';
-import type { KnowledgeTracker } from '../knowledge-tracker.ts';
 import { Observability } from '../observability.ts';
 import type { StateManager } from '../state-manager.js';
 import { WebPageState } from '../state-manager.js';
 import { Stats } from '../stats.ts';
 import { diffAriaSnapshots } from '../utils/aria.ts';
 import { ErrorPageError, detectPageCondition } from '../utils/error-page.ts';
-import { HooksRunner } from '../utils/hooks-runner.ts';
 import { isBodyEmpty } from '../utils/html.ts';
 import { createDebug, tag } from '../utils/logger.js';
 import { mdq } from '../utils/markdown-query.ts';
 import { RulesLoader } from '../utils/rules-loader.ts';
-import type { Agent } from './agent.js';
+import { annotatePageElements } from '../utils/web-annotate.ts';
+import type { Agent, AgentDeps } from './agent.js';
 import type { Navigator } from './navigator.ts';
 import { ContextLengthError, type Provider } from './provider.js';
 import { findSimilarResearch, getCachedResearch, saveResearch } from './researcher/cache.ts';
@@ -55,20 +54,16 @@ export class Researcher extends ResearcherBase implements Agent {
   declare explorer: Explorer;
   declare provider: Provider;
   declare stateManager: StateManager;
+  declare config: ExplorbotConfig;
   private experienceTracker!: ExperienceTracker;
   private hasScreenshotToAnalyze = false;
   declare actionResult: ActionResult | undefined;
-  private hooksRunner!: HooksRunner;
 
-  constructor(explorer: Explorer, provider: Provider) {
-    super();
-    this.explorer = explorer;
-    this.provider = provider;
-    this.stateManager = explorer.getStateManager();
-    this.experienceTracker = this.stateManager.getExperienceTracker();
-    this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
+  constructor(deps: AgentDeps) {
+    super(deps);
+    this.experienceTracker = deps.stateManager.getExperienceTracker();
 
-    const ai = explorer.getConfig().ai;
+    const ai = deps.config.ai;
     if (ai) {
       ai.agents ??= {};
       ai.agents.researcher ??= {};
@@ -80,18 +75,6 @@ export class Researcher extends ResearcherBase implements Agent {
     throw new Error('not implemented');
   }
 
-  protected getExperienceTracker(): ExperienceTracker {
-    return this.experienceTracker;
-  }
-
-  protected getKnowledgeTracker(): KnowledgeTracker {
-    return this.explorer.getKnowledgeTracker();
-  }
-
-  protected getProvider(): Provider {
-    return this.provider;
-  }
-
   static getCachedResearch(state: WebPageState): string {
     return getCachedResearch(state.hash || '');
   }
@@ -110,7 +93,7 @@ export class Researcher extends ResearcherBase implements Agent {
 
   async research(state: WebPageState, opts: { screenshot?: boolean; force?: boolean; deep?: boolean; data?: boolean; fix?: boolean; _retriesLeft?: number } = {}): Promise<string> {
     const { screenshot = false, force = false, deep = false, data = false, fix = true } = opts;
-    const maxRetries = (this.explorer.getConfig().ai?.agents?.researcher as any)?.retries ?? 2;
+    const maxRetries = (this.config.ai?.agents?.researcher as any)?.retries ?? 2;
     let retriesLeft = opts._retriesLeft ?? maxRetries;
     this.actionResult = ActionResult.fromState(state);
     const stateHash = state.hash || this.actionResult.getStateHash();
@@ -134,9 +117,9 @@ export class Researcher extends ResearcherBase implements Agent {
       await this.ensureNavigated(displayUrl, screenshot && this.provider.hasVision());
       await this.hooksRunner.runBeforeHook('researcher', state.url);
 
-      const annotatedElements = await this.explorer.annotateElements();
+      const { elements: annotatedElements } = await this.explorer.withPage(annotatePageElements);
       debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
-      this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
+      this.actionResult = await this.explorer.capture({ screenshot: screenshot && this.provider.hasVision() });
 
       const condition = detectPageCondition(this.actionResult!);
       if (condition === 'error') {
@@ -208,11 +191,10 @@ export class Researcher extends ResearcherBase implements Agent {
       }
 
       if (!interrupted()) {
-        const containerLocs = result.containerLocators;
-        await this.testLocators(containerLocs);
-        const brokenContainers = containerLocs.filter((l) => l.valid === false);
-        if (containerLocs.length > 0 && brokenContainers.length === containerLocs.length && retriesLeft > 0) {
-          tag('warning').log(`All ${containerLocs.length} containers broken, retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
+        const brokenContainers = await this.resolveContainers(result);
+        const containerCount = result.containers.length;
+        if (containerCount > 0 && brokenContainers.length === containerCount && retriesLeft > 0) {
+          tag('warning').log(`All ${containerCount} containers broken, retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
           await new Promise((r) => setTimeout(r, 2000));
           return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 } as any);
         }
@@ -226,14 +208,6 @@ export class Researcher extends ResearcherBase implements Agent {
 
         const toTest = result.locators.filter((l) => l.valid === null);
         await this.testLocators(toTest);
-
-        const brokenCount = result.locators.filter((l) => l.valid === false).length;
-        const brokenRatio = result.locators.length > 0 ? brokenCount / result.locators.length : 0;
-        if (brokenRatio > 0.8 && retriesLeft > 0) {
-          tag('warning').log(`${Math.round(brokenRatio * 100)}% locators broken, waiting 3s and retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
-          await new Promise((r) => setTimeout(r, 3000));
-          return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 } as any);
-        }
       }
 
       // Stage 3: Fix broken sections via AI conversation continuation
@@ -245,7 +219,7 @@ export class Researcher extends ResearcherBase implements Agent {
       // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
       if (!interrupted() && this.hasScreenshotToAnalyze) {
         const sections = parseResearchSections(result.text);
-        const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
+        const focused = await this.explorer.withPage((page) => detectFocusedSection(page, sections));
         if (focused) markSectionAsFocused(result, focused);
       }
 
@@ -254,11 +228,11 @@ export class Researcher extends ResearcherBase implements Agent {
         const validContainers = extractValidContainers(result.text);
         result.parseLocators();
         const freshContainerLocs = result.containerLocators;
-        await this.testLocators(freshContainerLocs);
+        await this.testLocators(freshContainerLocs, { scope: true });
         const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
         const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
         await this.visuallyAnnotateElements({ containers });
-        this.actionResult = await this.explorer.capturePageWithScreenshot();
+        this.actionResult = await this.explorer.capture({ screenshot: true });
         const visualResult = await this.analyzeScreenshotForVisualProps();
         if (visualResult.elements.size > 0) {
           await this.mergeVisualData(result, visualResult.elements);
@@ -334,8 +308,7 @@ export class Researcher extends ResearcherBase implements Agent {
   private async ensureNavigated(url: string, screenshot?: boolean): Promise<void> {
     if (!this.actionResult) {
       debugLog('No action result, navigating to URL');
-      await this.explorer.visit(url);
-      this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
+      this.actionResult = await this.explorer.visit(url, { screenshot });
       return;
     }
 
@@ -345,7 +318,7 @@ export class Researcher extends ResearcherBase implements Agent {
 
     if (!isEmpty && isOnCurrentState) {
       if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
-        this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
+        this.actionResult = await this.explorer.capture({ screenshot });
       }
       return;
     }
@@ -353,8 +326,7 @@ export class Researcher extends ResearcherBase implements Agent {
     if (isEmpty && isOnCurrentState) {
       debugLog('HTML body empty on current URL, waiting for content');
       tag('step').log('Page body is empty, waiting for content...');
-      await this.explorer.visit(url);
-      this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
+      this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false });
       await this.waitUntilSettled(screenshot ?? false);
       return;
     }
@@ -362,22 +334,21 @@ export class Researcher extends ResearcherBase implements Agent {
     debugLog('Not on current state, navigating to URL');
     tag('step').log('Navigating to URL...');
 
-    await this.explorer.visit(url);
-    this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
+    this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false });
   }
 
   private async waitUntilSettled(screenshot: boolean): Promise<boolean> {
-    const errorPageTimeout = (this.explorer.getConfig().ai?.agents?.researcher as any)?.errorPageTimeout ?? 10;
+    const errorPageTimeout = (this.config.ai?.agents?.researcher as any)?.errorPageTimeout ?? 10;
     if (errorPageTimeout <= 0) return false;
 
     const includeScreenshot = screenshot && this.provider.hasVision();
 
     try {
-      await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
+      await this.explorer.withPage((page) => page.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
     } catch {}
 
-    await this.explorer.annotateElements();
-    this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
+    await this.explorer.withPage(annotatePageElements);
+    this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
 
     let condition = detectPageCondition(this.actionResult!);
     if (condition === 'error') {
@@ -387,8 +358,8 @@ export class Researcher extends ResearcherBase implements Agent {
 
     for (let i = 0; i < 3; i++) {
       await new Promise((r) => setTimeout(r, 1000));
-      await this.explorer.annotateElements();
-      this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
+      await this.explorer.withPage(annotatePageElements);
+      this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
       condition = detectPageCondition(this.actionResult!);
       if (condition === 'error') {
         throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
@@ -400,7 +371,7 @@ export class Researcher extends ResearcherBase implements Agent {
   }
 
   private getConfiguredSections(): Record<string, string> {
-    const configSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.sections as string[] | undefined;
+    const configSections = (this.config.ai?.agents?.researcher as any)?.sections as string[] | undefined;
     if (!configSections?.length) return POSSIBLE_SECTIONS;
     const filtered: Record<string, string> = {};
     for (const key of configSections) {
@@ -467,7 +438,7 @@ export class Researcher extends ResearcherBase implements Agent {
     if (!this.actionResult) throw new Error('actionResult is not set');
 
     const html = await this.actionResult.combinedHtml();
-    const knowledge = this.explorer.getKnowledgeTracker().renderRelevantKnowledge(this.actionResult);
+    const knowledge = this.knowledgeTracker.renderRelevantKnowledge(this.actionResult);
 
     const ariaSnapshot = this.actionResult.getCompactARIA();
 
@@ -728,9 +699,9 @@ export class Researcher extends ResearcherBase implements Agent {
   async cancelInUi() {
     const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
 
-    await this.explorer.executeAction('I.clickXY(0, 0)');
+    await this.explorer.action().execute('I.clickXY(0, 0)');
     if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null).text) return;
 
-    await this.explorer.executeAction(`I.pressKey('Escape')`);
+    await this.explorer.action().execute(`I.pressKey('Escape')`);
   }
 }

package/src/ai/task-agent.ts

@@ -1,6 +1,11 @@
 import type { ActionResult } from '../action-result.js';
+import type { ExplorbotConfig } from '../config.ts';
 import type { ExperienceTracker } from '../experience-tracker.js';
+import type Explorer from '../explorer.ts';
 import type { KnowledgeTracker } from '../knowledge-tracker.js';
+import type { StateManager } from '../state-manager.ts';
+import { HooksRunner } from '../utils/hooks-runner.ts';
+import type { AgentDeps, ToolDeps } from './agent.ts';
 import { Historian } from './historian.js';
 import type { Navigator } from './navigator.js';
 import type { Provider } from './provider.js';
@@ -17,6 +22,12 @@ function createNullProxy<T extends object>(): T {
 }
 
 export abstract class TaskAgent {
+  explorer!: Explorer;
+  provider!: Provider;
+  config!: ExplorbotConfig;
+  stateManager!: StateManager;
+  knowledgeTracker!: KnowledgeTracker;
+  protected hooksRunner!: HooksRunner;
   protected consecutiveFailures = 0;
   protected consecutiveEmptyResults = 0;
   protected recentToolCalls: any[] = [];
@@ -25,10 +36,41 @@ export abstract class TaskAgent {
   private _historian: Historian | null = null;
   private _quartermaster: Quartermaster | null = null;
 
+  constructor(deps?: AgentDeps) {
+    if (!deps) return;
+    this.explorer = deps.explorer;
+    this.provider = deps.ai;
+    this.config = deps.config;
+    this.stateManager = deps.stateManager;
+    this.knowledgeTracker = deps.knowledgeTracker;
+    this.hooksRunner = new HooksRunner(deps.explorer, deps.config);
+  }
+
+  setHistorian(historian: Historian): void {
+    this._historian = historian;
+  }
+
+  setQuartermaster(quartermaster: Quartermaster): void {
+    this._quartermaster = quartermaster;
+  }
+
   protected abstract getNavigator(): Navigator;
-  protected abstract getExperienceTracker(): ExperienceTracker;
-  protected abstract getKnowledgeTracker(): KnowledgeTracker;
-  protected abstract getProvider(): Provider;
+
+  protected get toolDeps(): ToolDeps {
+    return { explorer: this.explorer, stateManager: this.stateManager, ai: this.provider };
+  }
+
+  protected getExperienceTracker(): ExperienceTracker {
+    return this.stateManager.getExperienceTracker();
+  }
+
+  protected getKnowledgeTracker(): KnowledgeTracker {
+    return this.knowledgeTracker;
+  }
+
+  protected getProvider(): Provider {
+    return this.provider;
+  }
 
   protected getKnowledge(actionResult: ActionResult): string {
     return this.getKnowledgeTracker().renderRelevantKnowledge(actionResult);
@@ -38,19 +80,11 @@ export abstract class TaskAgent {
     return this.getExperienceTracker().renderExperienceTocFor(actionResult);
   }
 
-  setHistorian(historian: Historian): void {
-    this._historian = historian;
-  }
-
   protected getHistorian(): Historian {
     if (this._historian) return this._historian;
     return createNullProxy<Historian>();
   }
 
-  setQuartermaster(quartermaster: Quartermaster): void {
-    this._quartermaster = quartermaster;
-  }
-
   protected getQuartermaster(): Quartermaster {
     if (this._quartermaster) return this._quartermaster;
     return createNullProxy<Quartermaster>();