explorbot 0.2.0 → 0.2.1

package/src/state-manager.ts

@@ -1,6 +1,6 @@
 import { ActionResult } from './action-result.js';
 import type { ExperienceTracker } from './experience-tracker.js';
-import type { KnowledgeTracker, Knowledge } from './knowledge-tracker.js';
+import type { Knowledge, KnowledgeTracker } from './knowledge-tracker.js';
 import { detectFocusArea } from './utils/aria.js';
 import { createDebug } from './utils/logger.js';
 import { slugify } from './utils/strings.js';
@@ -67,9 +67,15 @@ export interface StateTransition {
 
 export type StateChangeListener = (event: StateTransition) => void;
 
+export interface TabInfo {
+  url: string;
+  title: string;
+}
+
 export type { Knowledge };
 
 export class StateManager {
+  otherTabs: TabInfo[] = [];
   private currentState: WebPageState | null = null;
   private stateHistory: StateTransition[] = [];
   private allVisitedUrls: Set<string> = new Set();

package/src/utils/hooks-runner.ts

@@ -54,11 +54,9 @@ export class HooksRunner {
   private async executeHook(hook: Hook, url: string): Promise<void> {
     try {
       if (hook.type === 'playwright') {
-        const page = this.explorer.playwrightHelper.page;
-        await hook.hook({ page, url });
+        await this.explorer.withPage(async (page) => hook.hook({ page, url }));
       } else {
-        const I = this.explorer.actor;
-        await hook.hook({ I, url });
+        await hook.hook({ I: this.explorer.actor, url });
       }
     } catch (error) {
       debugLog(`Hook error: ${error}`);

package/src/utils/test-files.ts

@@ -76,7 +76,7 @@ export async function dryRunTestFile(filePath: string): Promise<void> {
     },
   };
 
-  (global as any).output_dir = path.join(projectRoot, 'output', 'states');
+  (global as any).output_dir = ConfigParser.getInstance().getStatesDir();
   (global as any).codecept_dir = projectRoot;
 
   codeceptjs.container.create(codeceptConfig, {});

package/src/utils/web-annotate.ts

@@ -0,0 +1,64 @@
+import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './html.ts';
+import { createDebug } from './logger.js';
+import { WebElement } from './web-element.ts';
+
+const debugLog = createDebug('explorbot:web-annotate');
+
+const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
+
+const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']);
+
+function parseAriaRefs(ariaSnapshot: string): Array<{ role: string; name: string; ref: string }> {
+  const entries: Array<{ role: string; name: string; ref: string }> = [];
+  for (const line of ariaSnapshot.split('\n')) {
+    const match = line.match(REF_LINE_PATTERN);
+    if (!match) continue;
+    if (!ANNOTATABLE_ROLES.has(match[2])) continue;
+    entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
+  }
+  return entries;
+}
+
+export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: string; elements: WebElement[] }> {
+  const ariaSnapshot: string = await page.locator('body').ariaSnapshot({ mode: 'ai' });
+  const refEntries = parseAriaRefs(ariaSnapshot);
+
+  const byRole = new Map<string, Array<{ name: string; ref: string }>>();
+  for (const { role, name, ref } of refEntries) {
+    let list = byRole.get(role);
+    if (!list) {
+      list = [];
+      byRole.set(role, list);
+    }
+    list.push({ name, ref });
+  }
+
+  const elements: WebElement[] = [];
+  for (const [role, entries] of byRole) {
+    try {
+      const rawList = await page.getByRole(role).evaluateAll(
+        (domElements: Element[], [data, extractFnStr, config]: [Array<{ name: string; ref: string }>, string, typeof ELEMENT_EXTRACTION_CONFIG]) => {
+          const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any;
+          const results: any[] = [];
+          let ariaIdx = 0;
+          for (const el of domElements) {
+            if (ariaIdx >= data.length) break;
+            el.setAttribute(config.attrs.eidx, data[ariaIdx].ref);
+            const elData = extract(el, config);
+            if (elData) results.push(elData);
+            ariaIdx++;
+          }
+          return results;
+        },
+        [entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG]
+      );
+      for (const raw of rawList) {
+        elements.push(WebElement.fromRawData(raw, role));
+      }
+    } catch {
+      debugLog(`Failed to annotate role=${role}`);
+    }
+  }
+
+  return { ariaSnapshot, elements };
+}

package/src/utils/web-eidx.ts

@@ -0,0 +1,21 @@
+import { ELEMENT_EXTRACTION_CONFIG } from './html.ts';
+
+export async function eidxInContainer(page: any, containerCss: string | null): Promise<string[]> {
+  const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`;
+  const elements = await page.locator(selector).all();
+  const result: string[] = [];
+  for (const el of elements) {
+    const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx).catch(() => null);
+    if (attr) result.push(attr);
+  }
+  return result;
+}
+
+export async function eidxByLocator(page: any, locator: string, container?: string | null): Promise<string | null> {
+  const base = container ? page.locator(container) : page;
+  const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator);
+  return el
+    .first()
+    .getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx)
+    .catch(() => null);
+}

package/src/utils/web-element.ts

@@ -147,6 +147,36 @@ export class WebElement {
     return rawList.map((d) => WebElement.fromRawData(d));
   }
 
+  static async commonAncestor(page: any, eidxList: string[]): Promise<WebElement | null> {
+    const validEidxList = eidxList.filter((eidx) => /^e\d+$/i.test(eidx));
+    if (validEidxList.length < 2) return null;
+
+    const raw: RawElementData | null = await page.evaluate(
+      ([list, extractFnStr, config]: [string[], string, ElementExtractionConfig]) => {
+        const extract = new Function(`return ${extractFnStr}`)() as (el: Element, cfg: ElementExtractionConfig) => any;
+        const els: Element[] = [];
+        for (const eidx of list) {
+          const el = document.querySelector(`[${config.attrs.eidx}="${eidx}"]`);
+          if (el && !els.includes(el)) els.push(el);
+        }
+        if (els.length < 2) return null;
+
+        const containsAll = (node: Element) => els.every((el) => node.contains(el));
+        let ancestor: Element | null = els[0].parentElement;
+        while (ancestor && !containsAll(ancestor)) ancestor = ancestor.parentElement;
+        if (!ancestor) return null;
+
+        const tag = ancestor.tagName.toLowerCase();
+        if (tag === 'body' || tag === 'html') return null;
+        return extract(ancestor, config);
+      },
+      [validEidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] as [string[], string, ElementExtractionConfig]
+    );
+
+    if (!raw) return null;
+    return WebElement.fromRawData(raw);
+  }
+
   static async findByXPath(html: string, xpath: string): Promise<{ totalFound: number; elements: WebElement[]; error?: string }> {
     const result = await evaluateXPath(html, xpath);
     if (result.error) return { totalFound: 0, elements: [], error: result.error };