explorbot 0.0.5 → 0.1.0

package/src/explorer.ts

@@ -19,6 +19,7 @@ import { Test } from './test-plan.ts';
 import { RequestStore } from './api/request-store.ts';
 import { XhrCapture } from './api/xhr-capture.ts';
 import { createDebug, log, tag } from './utils/logger.js';
+import { WebElement, extractElementData } from './utils/web-element.ts';
 
 declare global {
   namespace NodeJS {
@@ -308,35 +309,23 @@ class Explorer {
     return action;
   }
 
-  async annotateElements(): Promise<number> {
-    const page = this.playwrightHelper.page;
-    const roles = ['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem'];
-    let idx = 1;
-    for (const role of roles) {
-      const elements = await page.getByRole(role).all();
-      for (const el of elements) {
-        await el.evaluate((node: Element, i: number) => {
-          node.setAttribute('data-explorbot-eidx', String(i));
-        }, idx);
-        idx++;
-      }
-    }
-    return idx - 1;
+  async annotateElements(): Promise<{ ariaSnapshot: string; elements: WebElement[] }> {
+    return annotatePageElements(this.playwrightHelper.page);
   }
 
   async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise<number> {
     return visuallyAnnotateContainers(this.playwrightHelper.page, opts?.containers || []);
   }
 
-  async getEidxInContainer(containerCss: string | null): Promise<number[]> {
+  async getEidxInContainer(containerCss: string | null): Promise<string[]> {
     const page = this.playwrightHelper.page;
     try {
       const selector = containerCss ? `${containerCss} [data-explorbot-eidx]` : '[data-explorbot-eidx]';
       const elements = await page.locator(selector).all();
-      const result: number[] = [];
+      const result: string[] = [];
       for (const el of elements) {
         const attr = await el.getAttribute('data-explorbot-eidx');
-        if (attr) result.push(Number.parseInt(attr, 10));
+        if (attr) result.push(attr);
       }
       return result;
     } catch (error) {
@@ -348,13 +337,12 @@ class Explorer {
     }
   }
 
-  async getEidxByLocator(locator: string, container?: string | null): Promise<number | null> {
+  async getEidxByLocator(locator: string, container?: string | null): Promise<string | null> {
     try {
       const page = this.playwrightHelper.page;
       const base = container ? page.locator(container) : page;
       const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator);
-      const eidx = await el.first().getAttribute('data-explorbot-eidx');
-      return eidx ? Number.parseInt(eidx, 10) : null;
+      return await el.first().getAttribute('data-explorbot-eidx');
     } catch (error) {
       if (this.isFatalBrowserError(error)) {
         tag('warning').log(`getEidxByLocator: ${error instanceof Error ? error.message : error}`);
@@ -710,4 +698,63 @@ function toCodeceptjsTest(test: Test): any {
   return codeceptjsTest;
 }
 
+const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
+
+const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']);
+
+function parseAriaRefs(ariaSnapshot: string): Array<{ role: string; name: string; ref: string }> {
+  const entries: Array<{ role: string; name: string; ref: string }> = [];
+  for (const line of ariaSnapshot.split('\n')) {
+    const match = line.match(REF_LINE_PATTERN);
+    if (!match) continue;
+    if (!ANNOTATABLE_ROLES.has(match[2])) continue;
+    entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
+  }
+  return entries;
+}
+
+export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: string; elements: WebElement[] }> {
+  const ariaSnapshot: string = await page.locator('body').ariaSnapshot({ forAI: true });
+  const refEntries = parseAriaRefs(ariaSnapshot);
+
+  const byRole = new Map<string, Array<{ name: string; ref: string }>>();
+  for (const { role, name, ref } of refEntries) {
+    let list = byRole.get(role);
+    if (!list) {
+      list = [];
+      byRole.set(role, list);
+    }
+    list.push({ name, ref });
+  }
+
+  const elements: WebElement[] = [];
+  for (const [role, entries] of byRole) {
+    try {
+      const rawList = await page.getByRole(role).evaluateAll(
+        (domElements: Element[], [data, extractFnStr]: [Array<{ name: string; ref: string }>, string]) => {
+          const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any;
+          const results: any[] = [];
+          let ariaIdx = 0;
+          for (const el of domElements) {
+            if (ariaIdx >= data.length) break;
+            el.setAttribute('data-explorbot-eidx', data[ariaIdx].ref);
+            const elData = extract(el);
+            if (elData) results.push(elData);
+            ariaIdx++;
+          }
+          return results;
+        },
+        [entries, extractElementData.toString()]
+      );
+      for (const raw of rawList) {
+        elements.push(WebElement.fromRawData(raw, role));
+      }
+    } catch {
+      debugLog(`Failed to annotate role=${role}`);
+    }
+  }
+
+  return { ariaSnapshot, elements };
+}
+
 export default Explorer;

package/src/utils/web-element.ts

@@ -7,6 +7,7 @@ type RawElementData = NonNullable<ReturnType<typeof extractElementData>>;
 
 export class WebElement {
   tag: string;
+  role: string;
   xpath: string;
   clickXPath: string;
   attrs: Record<string, string>;
@@ -14,8 +15,9 @@ export class WebElement {
   outerHTML: string;
   x: number;
   y: number;
-  constructor(data: { tag: string; xpath: string; clickXPath: string; attrs: Record<string, string>; text: string; outerHTML?: string; x: number; y: number }) {
+  constructor(data: { tag: string; role?: string; xpath: string; clickXPath: string; attrs: Record<string, string>; text: string; outerHTML?: string; x: number; y: number }) {
     this.tag = data.tag;
+    this.role = data.role || data.attrs.role || '';
     this.xpath = data.xpath;
     this.clickXPath = data.clickXPath;
     this.attrs = data.attrs;
@@ -40,9 +42,8 @@ export class WebElement {
     return `(${this.x}, ${this.y})`;
   }
 
-  get eidx(): number | null {
-    const val = this.attrs['data-explorbot-eidx'] || this.attrs.eidx;
-    return val ? Number.parseInt(val, 10) : null;
+  get eidx(): string | null {
+    return this.attrs['data-explorbot-eidx'] || this.attrs.eidx || null;
   }
 
   get isNavigationLink(): boolean {
@@ -56,9 +57,10 @@ export class WebElement {
     return cls.split(/\s+/).filter((c) => c.length > 2 && !isDynamicId(c) && !isGenericClass(c));
   }
 
-  private static fromRawData(d: RawElementData): WebElement {
+  static fromRawData(d: RawElementData, role?: string): WebElement {
     return new WebElement({
       tag: d.tag,
+      role,
       xpath: '',
       clickXPath: buildClickableXPath({ tag: d.tag, allAttrs: d.allAttrs, text: d.text } as XPathMatch),
       attrs: d.allAttrs,
@@ -93,15 +95,15 @@ export class WebElement {
     }
   }
 
-  static async fromEidx(page: any, eidx: number): Promise<WebElement | null> {
+  static async fromEidx(page: any, eidx: string): Promise<WebElement | null> {
     return WebElement.fromPlaywrightLocator(page.locator(`[data-explorbot-eidx="${eidx}"]`));
   }
 
-  static async fromEidxList(page: any, eidxList: number[]): Promise<WebElement[]> {
+  static async fromEidxList(page: any, eidxList: string[]): Promise<WebElement[]> {
     if (eidxList.length === 0) return [];
 
     const rawList: RawElementData[] = await page.evaluate(
-      ([list, extractFnStr]: [number[], string]) => {
+      ([list, extractFnStr]: [string[], string]) => {
         const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any;
         const results: any[] = [];
         for (const eidx of list) {
@@ -112,7 +114,7 @@ export class WebElement {
         }
         return results;
       },
-      [eidxList, extractElementData.toString()] as [number[], string]
+      [eidxList, extractElementData.toString()] as [string[], string]
     );
 
     return rawList.map((d) => WebElement.fromRawData(d));
@@ -125,7 +127,7 @@ export class WebElement {
   }
 }
 
-function extractElementData(el: Element) {
+export function extractElementData(el: Element) {
   const rect = el.getBoundingClientRect();
   if (rect.width === 0 && rect.height === 0) return null;