explorbot 0.0.1 → 0.1.0

package/src/ai/researcher/locators.ts

@@ -0,0 +1,282 @@
+import dedent from 'dedent';
+import type { ActionResult } from '../../action-result.js';
+import type Explorer from '../../explorer.ts';
+import { executionController } from '../../execution-controller.ts';
+import { parseAriaLocator } from '../../utils/aria.ts';
+import { tag } from '../../utils/logger.js';
+import { mdq } from '../../utils/markdown-query.ts';
+import { WebElement } from '../../utils/web-element.ts';
+import { FOCUSED_MARKER } from './focus.ts';
+import type { Conversation } from '../conversation.ts';
+import type { Provider } from '../provider.js';
+import { locatorRule as generalLocatorRuleText } from '../rules.js';
+import { type Constructor, debugLog } from './mixin.ts';
+import { parseResearchSections } from './parser.ts';
+import type { ResearchResult } from './research-result.ts';
+
+function firstCssSegment(css: string): string | null {
+  const parts = css.split(/\s*>\s*|\s+/);
+  return parts.length > 1 ? parts[0] : null;
+}
+
+export const DYNAMIC_ID_PATTERN = /^#ember\d|^\/\/[^[]*\[@id="ember\d|#react-select-|#rc-|#ng-|#cdk-|#mat-|data-ebd-id/;
+export const isForbiddenLocator = (s: string) => DYNAMIC_ID_PATTERN.test(s) || s.includes('data-explorbot-eidx') || /\[eidx=/.test(s);
+
+function buildPwLocatorString(loc: Locator): string {
+  const base = loc.container ? `locate('${loc.container}')` : 'page';
+  if (loc.type === 'aria') {
+    const parsed = parseAriaLocator(loc.locator);
+    if (!parsed) return `${base}.getByRole('???')`;
+    return `${base}.getByRole('${parsed.role}', { name: '${parsed.text}' })`;
+  }
+  return `${base}.locator('${loc.locator}')`;
+}
+
+export function WithLocators<T extends Constructor>(Base: T) {
+  return class extends Base {
+    declare explorer: Explorer;
+    declare provider: Provider;
+    declare actionResult: ActionResult | undefined;
+
+    async testLocators(locators: Locator[]): Promise<void> {
+      let broken = 0;
+      for (const loc of locators) {
+        if (executionController.isInterrupted()) break;
+        if (loc.type !== 'aria' && isForbiddenLocator(loc.locator)) {
+          loc.valid = false;
+          loc.error = 'dynamic ID';
+          loc.pwLocator = buildPwLocatorString(loc);
+          debugLog(`DYNAMIC ID [${loc.section}] ${loc.type} "${loc.element}": ${loc.locator}`);
+          broken++;
+          continue;
+        }
+        try {
+          const count = await this.explorer.playwrightLocatorCount((page) => {
+            const base = loc.container ? page.locator(loc.container) : page;
+            if (loc.type === 'aria') {
+              const parsed = parseAriaLocator(loc.locator);
+              if (!parsed) return page.locator('__invalid__');
+              return base.getByRole(parsed.role as any, { name: parsed.text });
+            }
+            const converted = loc.locator.replace(/:contains\(/g, ':has-text(');
+            if (converted !== loc.locator) {
+              loc.locator = converted;
+            }
+            return base.locator(loc.locator);
+          });
+          loc.valid = count === 1;
+          loc.pwLocator = buildPwLocatorString(loc);
+          if (!loc.valid) {
+            loc.error = count === 0 ? '0 elements' : `${count} elements`;
+            debugLog(`BROKEN [${loc.section}] ${loc.type} "${loc.element}": ${loc.locator} (${loc.error})`);
+            broken++;
+          }
+        } catch (err) {
+          loc.valid = false;
+          loc.error = err instanceof Error ? err.message : String(err);
+          loc.pwLocator = buildPwLocatorString(loc);
+          debugLog(`ERROR [${loc.section}] ${loc.type} "${loc.element}": ${loc.locator} — ${loc.error}`);
+          broken++;
+        }
+      }
+
+      tag('substep').log(`Validated ${locators.length} locators: ${locators.length - broken} valid, ${broken} broken`);
+    }
+
+    async fixBrokenSections(result: ResearchResult, conversation: Conversation): Promise<void> {
+      const broken = result.locators.filter((l) => l.valid === false);
+      if (broken.length === 0) return;
+
+      const bySection = new Map<string, Locator[]>();
+      for (const loc of broken) {
+        const list = bySection.get(loc.section) || [];
+        list.push(loc);
+        bySection.set(loc.section, list);
+      }
+
+      const allLocsBySection = new Map<string, Locator[]>();
+      for (const loc of result.locators) {
+        const list = allLocsBySection.get(loc.section) || [];
+        list.push(loc);
+        allLocsBySection.set(loc.section, list);
+      }
+
+      const sectionParts: string[] = [];
+      const parsedSections = parseResearchSections(result.text);
+      for (const [name, sectionBroken] of bySection) {
+        const allLocs = allLocsBySection.get(name) || [];
+        const section = parsedSections.find((s) => s.name === name);
+        const container = section?.containerCss;
+
+        const isContainerBroken = sectionBroken.some((l) => l.error === 'container broken');
+
+        let header = `## ${name}\n`;
+        if (container) {
+          header += isContainerBroken ? `\n> Container: '${container}'  ← BROKEN (container not found)\n` : `\n> Container: '${container}'\n`;
+        }
+
+        const testedLines = allLocs.map((loc) => {
+          const status = loc.valid === false ? `← BROKEN (${loc.error || 'unknown'})` : '← OK';
+          return `- '${loc.element}': ${loc.pwLocator || loc.locator} ${status}`;
+        });
+
+        sectionParts.push(`${header}\nTested Elements:\n${testedLines.join('\n')}`);
+      }
+
+      const prompt = dedent`
+        Some locators in your research are broken. Please fix the broken sections.
+
+        ${sectionParts.join('\n\n')}
+
+        Return corrected sections in the same format as the original research.
+        Fix broken containers and locators. Keep working ones unchanged.
+        ${generalLocatorRuleText}
+      `;
+
+      tag('substep').log(`Fixing ${broken.length} broken locators via AI conversation...`);
+
+      try {
+        conversation.addUserText(prompt);
+        const invocationResult = await this.provider.invokeConversation(conversation, undefined, { agentName: 'researcher' });
+        if (!invocationResult) return;
+
+        const fixedSections = parseResearchSections(invocationResult.response.text);
+        if (fixedSections.length === 0) return;
+
+        for (const fixedSection of fixedSections) {
+          const originalSections = parseResearchSections(result.text);
+          const original = originalSections.find((s) => s.name === fixedSection.name);
+          if (!original || original.elements.length === 0) continue;
+
+          if (fixedSection.containerCss && fixedSection.containerCss !== original.containerCss) {
+            debugLog(`Fixed container for "${fixedSection.name}": '${original.containerCss}' → '${fixedSection.containerCss}'`);
+            original.containerCss = fixedSection.containerCss;
+          }
+
+          const fixedByName = new Map(fixedSection.elements.map((el) => [el.name, el]));
+          for (const el of original.elements) {
+            const fix = fixedByName.get(el.name);
+            if (!fix) continue;
+            if (fix.css) el.css = fix.css;
+            if (fix.aria) el.aria = fix.aria;
+          }
+          result.rebuildSectionInText(original);
+        }
+
+        result.parseLocators();
+        await this.testLocators(result.locators);
+      } catch (err) {
+        tag('substep').log(`AI fix failed: ${err instanceof Error ? err.message : err}`);
+      }
+    }
+
+    async backfillBrokenLocators(result: ResearchResult): Promise<void> {
+      result.parseLocators();
+      await this.testLocators(result.locators);
+
+      const sections = parseResearchSections(result.text);
+      const brokenCss = new Set(result.locators.filter((l) => l.type === 'css' && l.valid === false).map((l) => `${l.section}::${l.element}`));
+
+      const needsXpath: string[] = [];
+      const needsXpathEls = new Map<string, { section: (typeof sections)[0]; el: (typeof sections)[0]['elements'][0] }>();
+
+      for (const section of sections) {
+        for (const el of section.elements) {
+          if (el.aria && !/\w/.test(el.aria.text)) el.aria = null;
+          if (!el.eidx || el.xpath) continue;
+          const hasWorkingCss = el.css && !brokenCss.has(`${section.name}::${el.name}`);
+          const hasWorkingAria = el.aria && /\w/.test(el.aria.text);
+          if (!hasWorkingCss && !hasWorkingAria) {
+            needsXpath.push(el.eidx);
+            needsXpathEls.set(el.eidx, { section, el });
+          }
+        }
+      }
+
+      if (needsXpath.length > 0) {
+        const page = this.explorer.playwrightHelper.page;
+        const webElements = await WebElement.fromEidxList(page, needsXpath);
+        const changedSections = new Set<(typeof sections)[0]>();
+        for (const w of webElements) {
+          const entry = needsXpathEls.get(w.eidx!);
+          if (!entry || !w.clickXPath) continue;
+          entry.el.xpath = w.clickXPath;
+          changedSections.add(entry.section);
+        }
+        for (const section of changedSections) result.rebuildSectionInText(section);
+        tag('substep').log(`Backfilled XPath for ${webElements.length} elements missing working locators`);
+      }
+
+      await this.validateContainers(result);
+    }
+
+    private async validateContainers(result: ResearchResult): Promise<void> {
+      const sections = parseResearchSections(result.text);
+
+      for (const section of sections) {
+        if (!section.containerCss) continue;
+
+        let count = 0;
+        try {
+          count = await this.explorer.playwrightLocatorCount((page) => page.locator(section.containerCss!));
+        } catch {}
+
+        if (count >= 1) continue;
+
+        const simplified = firstCssSegment(section.containerCss);
+        if (simplified) {
+          let simplifiedCount = 0;
+          try {
+            simplifiedCount = await this.explorer.playwrightLocatorCount((page) => page.locator(simplified));
+          } catch {}
+
+          if (simplifiedCount >= 1) {
+            debugLog(`Simplified container: '${section.containerCss}' → '${simplified}'`);
+            this.updateSectionContainer(result, section, simplified);
+            continue;
+          }
+        }
+
+        debugLog(`Nullified broken container: '${section.containerCss}' in "${section.name}"`);
+        this.updateSectionContainer(result, section, null);
+      }
+    }
+
+    private updateSectionContainer(result: ResearchResult, section: ReturnType<typeof parseResearchSections>[0], newCss: string | null): void {
+      const oldCss = section.containerCss;
+      const escaped = section.name.replace(/"/g, '\\"');
+      let sectionQuery = mdq(result.text).query(`section2(~"${escaped}")`);
+      if (sectionQuery.count() === 0) sectionQuery = mdq(result.text).query(`section3(~"${escaped}")`);
+
+      if (newCss) {
+        result.text = sectionQuery.query('blockquote[0]').replace(`Container: '${newCss}'`);
+      } else {
+        result.text = sectionQuery.query('blockquote[0]').replace('');
+        result.text = result.text.replace(`${FOCUSED_MARKER}\n`, '');
+      }
+
+      for (const loc of result.locators) {
+        if (loc.container === oldCss && loc.section === section.name) {
+          loc.container = newCss;
+        }
+      }
+    }
+  };
+}
+
+export interface Locator {
+  section: string;
+  container: string | null;
+  element: string;
+  type: 'css' | 'xpath' | 'aria';
+  locator: string;
+  valid: boolean | null;
+  error: string | null;
+  pwLocator: string | null;
+}
+
+export interface LocatorMethods {
+  testLocators(locators: Locator[]): Promise<void>;
+  fixBrokenSections(result: ResearchResult, conversation: Conversation): Promise<void>;
+  backfillBrokenLocators(result: ResearchResult): Promise<void>;
+}

package/src/ai/researcher/mixin.ts

@@ -0,0 +1,4 @@
+import { createDebug } from '../../utils/logger.js';
+
+export type Constructor<T = object> = new (...args: any[]) => T;
+export const debugLog = createDebug('explorbot:researcher');

package/src/ai/researcher/parser.ts

@@ -0,0 +1,186 @@
+import { parseAriaLocator } from '../../utils/aria.ts';
+import { pluralize } from '../../utils/logger.ts';
+import { jsonToTable, parseSections, tableToJson } from '../../utils/markdown-parser.ts';
+import { mdq } from '../../utils/markdown-query.ts';
+import { FOCUSED_MARKER } from './focus.ts';
+
+export interface ResearchElement {
+  name: string;
+  type: string | null;
+  aria: { role: string; text: string } | null;
+  css: string | null;
+  xpath: string | null;
+  coordinates: string | null;
+  color: string | null;
+  icon: string | null;
+  eidx: string | null;
+}
+
+export interface ResearchSection {
+  name: string;
+  containerCss: string | null;
+  elements: ResearchElement[];
+  rawMarkdown: string;
+  isExtended: boolean;
+}
+
+const SKIP_SECTIONS = new Set(['summary', 'screenshot analysis', 'data', 'primary actions']);
+
+export const RESEARCH_COLUMN_ORDER = ['Element', 'Type', 'ARIA', 'CSS', 'XPath', 'Coordinates', 'Color', 'Icon', 'eidx'];
+
+function stripQuotes(str: string): string {
+  let trimmed = str.trim();
+  if (trimmed.startsWith('**') && trimmed.endsWith('**')) {
+    trimmed = trimmed.slice(2, -2);
+  }
+  if ((trimmed.startsWith("'") && trimmed.endsWith("'")) || (trimmed.startsWith('"') && trimmed.endsWith('"'))) {
+    return trimmed.slice(1, -1);
+  }
+  if (trimmed.startsWith('`') && trimmed.endsWith('`')) {
+    return trimmed.slice(1, -1);
+  }
+  return trimmed;
+}
+
+function normalizeLocatorValue(val: string): string | null {
+  let s = val.trim();
+  let prev = '';
+  while (prev !== s) {
+    prev = s;
+    s = stripQuotes(s).trim();
+  }
+  if (s === '-' || s === '') return null;
+  return s;
+}
+
+export function mapRowToElement(row: Record<string, string>): ResearchElement | null {
+  const colMap: Record<string, string> = {};
+  for (const [key, value] of Object.entries(row)) {
+    colMap[key.toLowerCase()] = value;
+  }
+
+  const name = stripQuotes(colMap.element || '');
+  if (!name) return null;
+
+  let eidxRaw = (colMap.eidx || '').trim();
+  if (eidxRaw && /^\d+$/.test(eidxRaw)) eidxRaw = `e${eidxRaw}`;
+
+  const aria = parseAriaLocator(colMap.aria || '-');
+
+  return {
+    name,
+    type: colMap.type?.trim() || aria?.role || null,
+    aria,
+    css: normalizeLocatorValue(colMap.css || '-'),
+    xpath: normalizeLocatorValue(colMap.xpath || '-'),
+    coordinates: (colMap.coordinates || '-').trim() === '-' ? null : colMap.coordinates.trim(),
+    color: (colMap.color || '-').trim() === '-' || (colMap.color || '').trim() === '' ? null : colMap.color.trim(),
+    icon: (colMap.icon || '-').trim() === '-' || (colMap.icon || '').trim() === '' ? null : colMap.icon.trim(),
+    eidx: eidxRaw && eidxRaw !== '-' ? eidxRaw : null,
+  };
+}
+
+export function extractContainerFromBlockquote(sectionMarkdown: string): string | null {
+  const bq = mdq(sectionMarkdown).query('blockquote[0]').text().trim();
+  if (!bq) return null;
+  const match = bq.match(/Container:\s*(.+)/i);
+  if (!match) return null;
+  const css = normalizeLocatorValue(match[1]);
+  if (!css || !/^[.#\[\w]/.test(css)) return null;
+  return css;
+}
+
+export function parseResearchSections(markdown: string): ResearchSection[] {
+  const hasExtendedResearch = markdown.includes('\n# Extended Research') || markdown.startsWith('# Extended Research');
+
+  return parseSections(markdown)
+    .filter((s) => !SKIP_SECTIONS.has(s.name.toLowerCase()) && !s.name.toLowerCase().includes('data:'))
+    .map((section) => {
+      const containerCss = extractContainerFromBlockquote(section.rawMarkdown);
+      const rows = tableToJson(section.rawMarkdown);
+      const elements = rows.map(mapRowToElement).filter(Boolean) as ResearchElement[];
+      const isExtended = hasExtendedResearch && section.depth === 3;
+
+      return { name: section.name, containerCss, elements, rawMarkdown: section.rawMarkdown, isExtended };
+    });
+}
+
+export function extractValidContainers(researchText: string, opts?: { exclude?: string[] }): Array<{ css: string; label: string }> {
+  const exclude = opts?.exclude || [];
+  return parseResearchSections(researchText)
+    .filter((s) => s.containerCss && !exclude.includes(s.containerCss))
+    .map((s) => ({ css: s.containerCss!, label: s.name }));
+}
+
+export function rebuildSectionMarkdown(section: ResearchSection): string {
+  const hasEidx = section.elements.some((e) => e.eidx);
+  const hasXpath = section.elements.some((e) => e.xpath);
+  const hasCoordinates = section.elements.some((e) => e.coordinates);
+  const hasColor = section.elements.some((e) => e.color);
+  const hasIcon = section.elements.some((e) => e.icon);
+
+  const presentColumns = new Set(['Element', 'ARIA', 'CSS']);
+  if (section.elements.some((e) => e.type || e.aria)) presentColumns.add('Type');
+  if (hasXpath) presentColumns.add('XPath');
+  if (hasCoordinates) presentColumns.add('Coordinates');
+  if (hasColor) presentColumns.add('Color');
+  if (hasIcon) presentColumns.add('Icon');
+  if (hasEidx) presentColumns.add('eidx');
+
+  const columns = RESEARCH_COLUMN_ORDER.filter((c) => presentColumns.has(c));
+
+  const rows = section.elements.map((el) => {
+    const row: Record<string, string> = {
+      Element: `'${el.name}'`,
+    };
+    if (presentColumns.has('Type')) row.Type = el.type || el.aria?.role || '-';
+    row.ARIA = el.aria ? `{ role: '${el.aria.role}', text: '${el.aria.text}' }` : '-';
+    row.CSS = el.css ? `'${el.css}'` : '-';
+    if (hasXpath) row.XPath = el.xpath ? `'${el.xpath}'` : '-';
+    if (hasCoordinates) row.Coordinates = el.coordinates || '-';
+    if (hasColor) row.Color = el.color || '-';
+    if (hasIcon) row.Icon = el.icon || '-';
+    if (hasEidx) row.eidx = el.eidx ? String(el.eidx) : '-';
+    return row;
+  });
+
+  return jsonToTable(rows, columns);
+}
+
+export function formatResearchSummary(text: string, opts?: { visionUsed?: boolean }): string {
+  const sections = parseResearchSections(text);
+  const coordCount = sections.reduce((sum, s) => sum + s.elements.filter((e) => e.coordinates !== null).length, 0);
+
+  const mainSections = sections.filter((s) => !s.isExtended);
+  const extendedSections = sections.filter((s) => s.isExtended);
+
+  const lines: string[] = [];
+
+  for (const s of mainSections) {
+    if (s.elements.length === 0 && !s.containerCss) continue;
+    lines.push(formatSectionLine(s));
+  }
+
+  if (extendedSections.length > 0) {
+    lines.push('', 'Extended Research', '');
+    for (const s of extendedSections) {
+      if (s.elements.length === 0 && !s.containerCss) continue;
+      lines.push(formatSectionLine(s));
+    }
+  }
+
+  lines.push('', `Chars: ${text.length}`);
+
+  if (opts?.visionUsed || coordCount > 0) {
+    lines.push(`Vision: ${coordCount} elements with coordinates`);
+  }
+
+  return lines.join('\n');
+}
+
+function formatSectionLine(s: ResearchSection): string {
+  const parts = [`* ${s.name} (${s.elements.length} ${pluralize(s.elements.length, 'element')})`];
+  if (s.containerCss) parts.push(`\`${s.containerCss}\``);
+  if (s.rawMarkdown.includes(FOCUSED_MARKER)) parts.push('**Focused**');
+  return parts.join(' ');
+}

package/src/ai/researcher/research-result.ts

@@ -0,0 +1,116 @@
+import { parseAriaLocator } from '../../utils/aria.ts';
+import { jsonToTable } from '../../utils/markdown-parser.ts';
+import { mdq } from '../../utils/markdown-query.ts';
+import type { Locator } from './locators.ts';
+import { RESEARCH_COLUMN_ORDER, type ResearchSection, parseResearchSections, rebuildSectionMarkdown } from './parser.ts';
+
+export class ResearchResult {
+  text: string;
+  url: string;
+  createdAt: Date;
+  locators: Locator[] = [];
+
+  constructor(text: string, url: string) {
+    this.text = text;
+    this.url = url;
+    this.createdAt = new Date();
+  }
+
+  parseLocators(): void {
+    const sections = parseResearchSections(this.text);
+    const locators: Locator[] = [];
+    for (const section of sections) {
+      for (const el of section.elements) {
+        if (el.css) locators.push({ section: section.name, container: section.containerCss, element: el.name, type: 'css', locator: el.css, valid: null, error: null, pwLocator: null });
+        if (el.xpath) locators.push({ section: section.name, container: section.containerCss, element: el.name, type: 'xpath', locator: el.xpath, valid: null, error: null, pwLocator: null });
+        if (el.aria && /\w/.test(el.aria.text)) locators.push({ section: section.name, container: section.containerCss, element: el.name, type: 'aria', locator: `{ role: '${el.aria.role}', text: '${el.aria.text}' }`, valid: null, error: null, pwLocator: null });
+      }
+    }
+    this.locators = locators;
+  }
+
+  get containers(): string[] {
+    return [...new Set(this.locators.map((l) => l.container).filter(Boolean))] as string[];
+  }
+
+  get containerLocators(): Locator[] {
+    return this.containers.map((css) => ({
+      section: '',
+      container: null,
+      element: css,
+      type: 'css' as const,
+      locator: css,
+      valid: null,
+      error: null,
+      pwLocator: null,
+    }));
+  }
+
+  updateSection(sectionName: string, locators: Locator[]): void {
+    const sections = parseResearchSections(this.text);
+    const section = sections.find((s) => s.name === sectionName);
+    if (!section) return;
+
+    for (const el of section.elements) {
+      const elLocators = locators.filter((l) => l.element === el.name);
+      for (const loc of elLocators) {
+        const value = loc.valid === false ? null : loc.locator || null;
+        if (loc.type === 'css') el.css = value;
+        if (loc.type === 'xpath') el.xpath = value;
+        if (loc.type === 'aria') el.aria = value ? parseAriaLocator(value) : null;
+      }
+    }
+
+    this.rebuildSectionInText(section);
+  }
+
+  rebuildSectionInText(section: ResearchSection): void {
+    if (section.elements.length === 0) return;
+    const newTable = rebuildSectionMarkdown(section);
+    const escaped = section.name.replace(/"/g, '\\"');
+    let sectionQuery = mdq(this.text).query(`section2(~"${escaped}")`);
+    if (sectionQuery.count() === 0) sectionQuery = mdq(this.text).query(`section3(~"${escaped}")`);
+    const updated = sectionQuery.query('table').replace(`${newTable.trimEnd()}\n`);
+    if (updated === this.text) return;
+    section.rawMarkdown = mdq(section.rawMarkdown).query('table').replace(`${newTable.trimEnd()}\n`);
+    this.text = updated;
+  }
+
+  cleanup(): void {
+    for (const table of mdq(this.text).query('table').each()) {
+      const rows = table.toJson();
+      if (rows.length === 0) continue;
+
+      let changed = false;
+
+      if (!('Type' in rows[0]) && 'ARIA' in rows[0]) {
+        for (const row of rows) {
+          row.Type = parseAriaLocator(row.ARIA || '-')?.role || '-';
+        }
+        changed = true;
+      }
+
+      for (const row of rows) {
+        if (row.ARIA && !parseAriaLocator(row.ARIA)) {
+          row.ARIA = '-';
+          changed = true;
+        }
+      }
+
+      const hasEidx = 'eidx' in rows[0];
+      if (!changed && !hasEidx) continue;
+
+      const rawTable = table.text();
+      const baseColumns = Object.keys(rows[0]).filter((c) => c !== 'eidx');
+      const columns = this.reorderColumns(baseColumns);
+      const cleaned = rows.map(({ eidx, ...rest }) => rest);
+      this.text = this.text.replace(rawTable, jsonToTable(cleaned, columns));
+    }
+  }
+
+  private reorderColumns(columns: string[]): string[] {
+    const ordered = RESEARCH_COLUMN_ORDER.filter((c) => columns.includes(c));
+    const rest = columns.filter((c) => !RESEARCH_COLUMN_ORDER.includes(c));
+    return [...ordered, ...rest];
+  }
+}