explorbot 0.0.1 → 0.0.5

package/src/ai/quartermaster.ts

@@ -0,0 +1,286 @@
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { z } from 'zod';
+import type { ActionResult } from '../action-result.ts';
+import { ConfigParser } from '../config.ts';
+import type { StateManager, StateTransition, WebPageState } from '../state-manager.ts';
+import type { Task } from '../test-plan.ts';
+import { createDebug, tag } from '../utils/logger.ts';
+import type { Conversation, ToolExecution } from './conversation.ts';
+import type { Provider } from './provider.ts';
+import { CODECEPT_TOOLS } from './tools.ts';
+
+const debugLog = createDebug('explorbot:quartermaster');
+
+interface AxeViolation {
+  id: string;
+  impact: 'critical' | 'serious' | 'moderate' | 'minor';
+  description: string;
+  helpUrl: string;
+  nodes: Array<{ html: string; target: string[] }>;
+}
+
+interface PageAnalysis {
+  url: string;
+  stateHash: string;
+  timestamp: string;
+  axeViolations: AxeViolation[];
+}
+
+interface SemanticIssue {
+  type: 'unclear_intention' | 'confusing_naming' | 'hard_to_interact';
+  element: string;
+  issue: string;
+  suggestion: string;
+}
+
+interface AnalysisReport {
+  scenario: string;
+  timestamp: string;
+  url: string;
+  axeViolations: AxeViolation[];
+  semanticIssues: SemanticIssue[];
+}
+
+export class Quartermaster {
+  private provider: Provider;
+  private model?: any;
+  private outputDir: string;
+  private pageAnalyses: Map<string, PageAnalysis> = new Map();
+  private unsubscribe: (() => void) | null = null;
+  private playwrightHelper: any = null;
+  private pendingAnalyses: Promise<void>[] = [];
+
+  constructor(provider: Provider, options?: { model?: any }) {
+    this.provider = provider;
+    this.model = options?.model;
+
+    const configParser = ConfigParser.getInstance();
+    this.outputDir = join(configParser.getOutputDir(), 'a11y');
+  }
+
+  start(playwrightHelper: any, stateManager: StateManager): void {
+    this.playwrightHelper = playwrightHelper;
+    this.ensureDirectory();
+
+    this.unsubscribe = stateManager.onStateChange((event) => {
+      this.onPageChange(event);
+    });
+
+    debugLog('Quartermaster started, listening for page changes');
+  }
+
+  stop(): void {
+    if (this.unsubscribe) {
+      this.unsubscribe();
+      this.unsubscribe = null;
+    }
+    debugLog('Quartermaster stopped');
+  }
+
+  private ensureDirectory(): void {
+    if (!existsSync(this.outputDir)) {
+      mkdirSync(this.outputDir, { recursive: true });
+    }
+  }
+
+  private onPageChange(event: StateTransition): void {
+    const state = event.toState;
+    if (!state?.hash) return;
+    if (this.pageAnalyses.has(state.hash)) return;
+
+    const analysisPromise = this.runAxeAnalysis(state).catch((err) => {
+      debugLog('Axe analysis failed:', err);
+    });
+
+    this.pendingAnalyses.push(analysisPromise);
+  }
+
+  private async runAxeAnalysis(state: WebPageState): Promise<void> {
+    const page = this.playwrightHelper?.page;
+    if (!page || !state.hash) {
+      debugLog('No page available for axe analysis');
+      return;
+    }
+
+    try {
+      const { AxeBuilder } = await import('@axe-core/playwright');
+      const results = await new AxeBuilder({ page }).analyze();
+
+      const violations: AxeViolation[] = results.violations.map((v) => ({
+        id: v.id,
+        impact: v.impact as AxeViolation['impact'],
+        description: v.description,
+        helpUrl: v.helpUrl,
+        nodes: v.nodes.map((n) => ({
+          html: n.html,
+          target: n.target as string[],
+        })),
+      }));
+
+      this.pageAnalyses.set(state.hash, {
+        url: state.url,
+        stateHash: state.hash,
+        timestamp: new Date().toISOString(),
+        axeViolations: violations,
+      });
+
+      debugLog(`Axe analysis complete for ${state.url}: ${violations.length} violations`);
+    } catch (error) {
+      debugLog('Axe analysis error:', error);
+    }
+  }
+
+  async analyzeSession(task: Task, initialState: ActionResult, conversation: Conversation): Promise<AnalysisReport | null> {
+    await Promise.all(this.pendingAnalyses);
+    this.pendingAnalyses = [];
+
+    try {
+      const stateHash = initialState.getStateHash();
+      const pageAnalysis = this.pageAnalyses.get(stateHash);
+
+      const toolExecutions = conversation.getToolExecutions();
+      const codeceptExecutions = toolExecutions.filter((e) => CODECEPT_TOOLS.includes(e.toolName as any));
+
+      if (codeceptExecutions.length === 0 && !pageAnalysis?.axeViolations.length) {
+        debugLog('No interactions or violations to analyze');
+        return null;
+      }
+
+      const axeViolations = pageAnalysis?.axeViolations || [];
+      const semanticIssues = await this.generateSemanticAnalysis(axeViolations, codeceptExecutions, initialState);
+
+      if (axeViolations.length === 0 && semanticIssues.length === 0) {
+        debugLog('No issues found');
+        return null;
+      }
+
+      const report: AnalysisReport = {
+        scenario: task.description,
+        timestamp: new Date().toISOString(),
+        url: initialState.url || '',
+        axeViolations,
+        semanticIssues,
+      };
+
+      this.saveReport(stateHash, report);
+      this.addNotesToTask(task, report);
+
+      tag('substep').log(`Quartermaster: ${axeViolations.length} technical + ${semanticIssues.length} semantic issues`);
+      return report;
+    } catch (error) {
+      debugLog('Quartermaster analysis failed:', error);
+      return null;
+    }
+  }
+
+  private async generateSemanticAnalysis(axeViolations: AxeViolation[], executions: ToolExecution[], initialState: ActionResult): Promise<SemanticIssue[]> {
+    const failedExecs = executions.filter((e) => !e.wasSuccessful);
+    if (failedExecs.length === 0 && axeViolations.length === 0) return [];
+
+    const axeSummary = axeViolations
+      .slice(0, 10)
+      .map((v) => `[${v.impact}] ${v.id}: ${v.description}`)
+      .join('\n');
+
+    const failedActions = failedExecs
+      .slice(0, 10)
+      .map((e) => {
+        const locator = e.input?.locator || e.output?.locator || '';
+        const error = e.output?.message || 'Failed';
+        return `${e.toolName}("${locator}"): ${error}`;
+      })
+      .join('\n');
+
+    const ariaSnapshot = initialState.getCompactARIA().slice(0, 3000);
+
+    const schema = z.object({
+      issues: z.array(
+        z.object({
+          type: z.enum(['unclear_intention', 'confusing_naming', 'hard_to_interact']).describe('Type of semantic issue'),
+          element: z.string().describe('Short element identifier: tag with key attributes, e.g. <button class="nav-toggle"> or <a href="/login" role="link">'),
+          issue: z.string().describe('What is confusing or problematic'),
+          suggestion: z.string().describe('Actionable improvement suggestion'),
+        })
+      ),
+    });
+
+    const prompt = `Analyze this page for semantic UX issues that automated tools cannot detect.
+
+## Technical Violations (axe-core)
+${axeSummary || 'None'}
+
+## Failed Agent Interactions
+${failedActions || 'None'}
+
+## Page Structure (ARIA)
+${ariaSnapshot}
+
+Focus on issues that require human judgment:
+- **unclear_intention**: Button/link text doesn't match actual behavior
+- **confusing_naming**: Ambiguous labels, inconsistent terminology
+- **hard_to_interact**: Controls requiring non-obvious sequences
+
+Provide 0-5 high-signal issues. Skip obvious technical violations already covered by axe-core.
+Focus on what would confuse a real user or caused the agent to make mistakes.`;
+
+    const response = await this.provider.generateObject(
+      [
+        { role: 'system', content: 'You are a UX expert analyzing pages for semantic issues that confuse users.' },
+        { role: 'user', content: prompt },
+      ],
+      schema,
+      this.model
+    );
+
+    return response?.object?.issues || [];
+  }
+
+  private addNotesToTask(task: Task, report: AnalysisReport): void {
+    const criticalViolations = report.axeViolations.filter((v) => v.impact === 'critical' || v.impact === 'serious');
+    for (const v of criticalViolations.slice(0, 3)) {
+      const nodeHtml = v.nodes[0]?.html.slice(0, 100) || '';
+      task.addNote(`🔴 A11Y [${v.impact}] ${v.id}: ${v.description} — ${nodeHtml}`);
+    }
+
+    for (const issue of report.semanticIssues.slice(0, 3)) {
+      task.addNote(`💡 UX [${issue.type}] ${issue.element}: ${issue.suggestion}`);
+    }
+  }
+
+  private saveReport(stateHash: string, report: AnalysisReport): void {
+    const filePath = join(this.outputDir, `${stateHash}.md`);
+    const content = this.formatReportMarkdown(report);
+    writeFileSync(filePath, content, 'utf8');
+    debugLog(`Saved a11y report to ${filePath}`);
+  }
+
+  private formatReportMarkdown(report: AnalysisReport): string {
+    let content = `## A11Y Analysis: ${report.url}\n\n`;
+    content += `**Scenario**: ${report.scenario}\n`;
+    content += `**Date**: ${report.timestamp}\n\n`;
+
+    if (report.axeViolations.length > 0) {
+      content += '### Technical Violations (axe-core)\n\n';
+      for (const v of report.axeViolations) {
+        content += `- [${v.impact}] **${v.id}**: ${v.description}\n`;
+        content += `  [Learn more](${v.helpUrl})\n`;
+        for (const node of v.nodes.slice(0, 2)) {
+          content += `  - \`${node.target.join(' > ')}\` — \`${node.html.slice(0, 100)}\`\n`;
+        }
+        content += '\n';
+      }
+    }
+
+    if (report.semanticIssues.length > 0) {
+      content += '### Semantic Issues (LLM analysis)\n\n';
+      for (const issue of report.semanticIssues) {
+        content += `- **${issue.type}** - ${issue.element}\n`;
+        content += `  Issue: ${issue.issue}\n`;
+        content += `  → ${issue.suggestion}\n\n`;
+      }
+    }
+
+    return content;
+  }
+}

package/src/ai/researcher/cache.ts

@@ -0,0 +1,103 @@
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { outputPath } from '../../config.ts';
+import { computeHtmlFingerprint } from '../../utils/html-diff.ts';
+import { debugLog } from './mixin.ts';
+
+const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
+const FINGERPRINT_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
+const FINGERPRINT_WORKER_TIMEOUT_MS = 10_000;
+const SIMILARITY_THRESHOLD = 90;
+
+const memoryCache: Record<string, string> = {};
+const memoryCacheTimestamps: Record<string, number> = {};
+
+let fingerprintWorker: Worker | null = null;
+
+function getStatesDir(): string {
+  return outputPath('states');
+}
+
+function getFingerprintWorker(): Worker {
+  if (!fingerprintWorker) {
+    const ext = import.meta.url.endsWith('.ts') ? '.ts' : '.js';
+    fingerprintWorker = new Worker(new URL(`./fingerprint-worker${ext}`, import.meta.url).href);
+  }
+  return fingerprintWorker;
+}
+
+export function getCachedResearch(hash: string): string {
+  if (!hash) return '';
+  const now = Date.now();
+  const timestamp = memoryCacheTimestamps[hash];
+  if (timestamp && now - timestamp <= CACHE_TTL_MS) {
+    return memoryCache[hash] || '';
+  }
+  const researchFile = outputPath('research', `${hash}.md`);
+  if (!existsSync(researchFile)) return '';
+  const stats = statSync(researchFile);
+  if (now - stats.mtimeMs > CACHE_TTL_MS) return '';
+  const cached = readFileSync(researchFile, 'utf8');
+  memoryCache[hash] = cached;
+  memoryCacheTimestamps[hash] = now;
+  return cached;
+}
+
+export function saveResearch(hash: string, text: string, combinedHtml?: string): string {
+  const researchDir = outputPath('research');
+  const researchFile = join(researchDir, `${hash}.md`);
+  if (!existsSync(researchDir)) mkdirSync(researchDir, { recursive: true });
+  writeFileSync(researchFile, text);
+  memoryCache[hash] = text;
+  memoryCacheTimestamps[hash] = Date.now();
+  debugLog(`Research saved to ${researchFile}`);
+
+  if (combinedHtml) {
+    const statesDir = getStatesDir();
+    if (!existsSync(statesDir)) mkdirSync(statesDir, { recursive: true });
+    const fingerprint = computeHtmlFingerprint(combinedHtml);
+    const fingerprintFile = join(statesDir, `${hash}.fingerprint`);
+    writeFileSync(fingerprintFile, fingerprint.join('\n'));
+    debugLog(`Fingerprint saved to ${fingerprintFile}`);
+  }
+
+  return researchFile;
+}
+
+export function findSimilarResearch(combinedHtml: string): Promise<string | null> {
+  const statesDir = getStatesDir();
+  if (!existsSync(statesDir)) return Promise.resolve(null);
+
+  const worker = getFingerprintWorker();
+
+  return new Promise((resolve) => {
+    const timeout = setTimeout(() => {
+      debugLog('Fingerprint worker timed out');
+      resolve(null);
+    }, FINGERPRINT_WORKER_TIMEOUT_MS);
+
+    worker.onmessage = (event: MessageEvent) => {
+      clearTimeout(timeout);
+      const { matchHash, similarity } = event.data as { matchHash: string | null; similarity: number };
+      if (!matchHash) {
+        resolve(null);
+        return;
+      }
+
+      debugLog(`Similar research found: ${matchHash} (${similarity}% similar)`);
+      const research = getCachedResearch(matchHash);
+      if (research) {
+        resolve(research);
+        return;
+      }
+      resolve(null);
+    };
+
+    worker.postMessage({
+      html: combinedHtml,
+      statesDir,
+      maxAgeMs: FINGERPRINT_MAX_AGE_MS,
+      threshold: SIMILARITY_THRESHOLD,
+    });
+  });
+}

package/src/ai/researcher/coordinates.ts

@@ -0,0 +1,238 @@
+import dedent from 'dedent';
+import type { Page } from 'playwright';
+import type { ActionResult } from '../../action-result.js';
+import type Explorer from '../../explorer.ts';
+import { tag } from '../../utils/logger.js';
+import { mdq } from '../../utils/markdown-query.ts';
+import { WebElement } from '../../utils/web-element.ts';
+import type { Provider } from '../provider.js';
+import { type Constructor, debugLog } from './mixin.ts';
+import { parseResearchSections } from './parser.ts';
+import type { ResearchResult } from './research-result.ts';
+
+export async function visuallyAnnotateContainers(page: Page, containers: Array<{ css: string; label: string }>): Promise<number> {
+  return page.evaluate((ctrs: Array<{ css: string; label: string }>) => {
+    const containerColors = ['#9b59b6', '#16a085', '#c0392b', '#2980b9'];
+    const drawnContainers: Array<{ label: string; color: string }> = [];
+    for (let i = 0; i < ctrs.length; i++) {
+      let el: Element | null;
+      try {
+        el = document.querySelector(ctrs[i].css);
+      } catch {
+        continue;
+      }
+      if (!el) continue;
+      const rect = el.getBoundingClientRect();
+      if (rect.width === 0 && rect.height === 0) continue;
+
+      const color = containerColors[i % containerColors.length];
+      const box = document.createElement('div');
+      box.setAttribute('data-explorbot-annotation', 'true');
+      box.style.cssText = `position:absolute;left:${rect.left + window.scrollX}px;top:${rect.top + window.scrollY}px;width:${rect.width}px;height:${rect.height}px;border:1px dashed ${color};z-index:99998;pointer-events:none;`;
+      document.body.appendChild(box);
+      drawnContainers.push({ label: ctrs[i].label, color });
+    }
+
+    if (drawnContainers.length > 0) {
+      const legend = document.createElement('div');
+      legend.setAttribute('data-explorbot-annotation', 'true');
+      legend.style.cssText = 'position:fixed;right:10px;bottom:10px;background:white;color:black;font-size:14px;font-family:sans-serif;padding:10px 14px;z-index:100001;pointer-events:none;border:3px solid #e63946;border-radius:6px;line-height:22px;';
+      const title = '<div style="font-weight:bold;margin-bottom:4px;color:#e63946;">Legend</div>';
+      const items = drawnContainers.map((c) => `<div><span style="display:inline-block;width:24px;border-top:3px dashed ${c.color};margin-right:8px;vertical-align:middle;"></span>${c.label}</div>`).join('');
+      legend.innerHTML = title + items;
+      document.body.appendChild(legend);
+    }
+
+    const colors = ['#e63946', '#2a9d8f', '#e9c46a', '#264653', '#f4a261', '#7b2cbf', '#0077b6', '#d62828'];
+    const elements = document.querySelectorAll('[data-explorbot-eidx]');
+    let count = 0;
+    for (const el of elements) {
+      const eidx = el.getAttribute('data-explorbot-eidx');
+      if (!eidx) continue;
+      const rect = el.getBoundingClientRect();
+      if (rect.width === 0 && rect.height === 0) continue;
+
+      const color = colors[count % colors.length];
+
+      const box = document.createElement('div');
+      box.setAttribute('data-explorbot-annotation', 'true');
+      box.style.cssText = `position:absolute;left:${rect.left + window.scrollX}px;top:${rect.top + window.scrollY}px;width:${rect.width}px;height:${rect.height}px;border:2px solid ${color};z-index:99999;pointer-events:none;`;
+
+      const label = document.createElement('div');
+      label.textContent = eidx;
+      label.style.cssText = `position:absolute;top:-14px;right:-2px;background:${color};color:white;font-size:10px;padding:0 3px;line-height:14px;font-family:monospace;z-index:100000;pointer-events:none;`;
+      box.appendChild(label);
+
+      document.body.appendChild(box);
+      count++;
+    }
+    return count;
+  }, containers);
+}
+
+export function WithCoordinates<T extends Constructor>(Base: T) {
+  return class extends Base {
+    declare explorer: Explorer;
+    declare provider: Provider;
+    declare actionResult: ActionResult | undefined;
+
+    analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult> {
+      return this._analyzeScreenshotForVisualProps();
+    }
+
+    async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise<number> {
+      return visuallyAnnotateContainers(this.explorer.playwrightHelper.page, opts?.containers || []);
+    }
+
+    private async _analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult> {
+      const elements = new Map<number, { coordinates: string | null; color: string | null; icon: string | null }>();
+      const emptyResult: VisualAnalysisResult = { elements, pagePurpose: null, primaryActions: null, focusedSection: null };
+      if (!this.actionResult) return emptyResult;
+
+      const image = this.actionResult.screenshot;
+      if (!image) return emptyResult;
+      tag('step').log('Analyzing annotated screenshot for visual properties');
+
+      const prompt = dedent`
+        This screenshot has two types of annotations:
+        - **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-left maps dashed line colors to section names. Ignore containers for this task.
+        - **Interactive elements**: solid bordered boxes with eidx numbers in the top-right corner above the box. Adjacent elements use different colors.
+
+        For each interactive element (solid border, eidx number), report:
+        | eidx | Coordinates | Color | Icon |
+
+        Column definitions:
+        - eidx: the number shown in the colored label above the top-right corner of each solid-bordered box
+        - Coordinates: (X, Y) center point of the element
+        - Color: accent color if distinctive (red, green, blue, orange, yellow, purple, gray), otherwise -
+        - Icon: one-word icon description (plus, x, trash, pencil, gear, search, hamburger, ellipsis, chevron, star, check, filter), otherwise -
+
+        Ignore the legend block and dashed container borders.
+
+        Then add:
+
+        ## Page Purpose
+        One sentence: what this page is for from the user's perspective.
+
+        ## Primary Actions
+        List 3-5 most prominent interactive elements (accent buttons, CTAs, main form actions).
+        Format: - action description
+
+        ## Focused Section
+        Which section from the legend appears to be the user's primary focus area?
+        Look for: modal overlays, drawers, panels with shadows or elevated z-index, highlighted/active areas.
+        If a dialog or overlay is visible, that is the focused section.
+        Otherwise, pick the section with the most prominent interactive content.
+        Reply with the exact section name from the legend. If no legend is shown, reply with "-".
+      `;
+
+      try {
+        const aiResult = await this.provider.processImage(prompt, image.toString('base64'));
+        const text = aiResult.text || '';
+        const rows = mdq(text).query('table').toJson();
+        for (const row of rows) {
+          const eidx = Number.parseInt(row.eidx, 10);
+          if (Number.isNaN(eidx)) continue;
+          const val = (v: string) => (v && v !== '-' ? v : null);
+          elements.set(eidx, {
+            coordinates: val(row.Coordinates),
+            color: val(row.Color),
+            icon: val(row.Icon),
+          });
+        }
+
+        const pagePurposeSection = mdq(text).query('section2("Page Purpose")').text().trim();
+        const primaryActionsSection = mdq(text).query('section2("Primary Actions")').text().trim();
+        const focusedSectionRaw = mdq(text).query('section2("Focused Section")').text().trim();
+        const focusedSection = focusedSectionRaw && focusedSectionRaw !== '-' ? focusedSectionRaw.split('\n')[0].trim() : null;
+
+        debugLog(`Parsed visual props for ${elements.size} elements`);
+        return {
+          elements,
+          pagePurpose: pagePurposeSection || null,
+          primaryActions: primaryActionsSection
+            ? primaryActionsSection
+                .split('\n')
+                .filter((l) => l.trim().startsWith('-'))
+                .map((l) => l.trim())
+            : null,
+          focusedSection,
+        };
+      } catch (err) {
+        debugLog(`Screenshot visual analysis failed: ${err instanceof Error ? err.message : err}`);
+      }
+
+      debugLog(`Parsed visual props for ${elements.size} elements`);
+      return emptyResult;
+    }
+
+    async mergeVisualData(result: ResearchResult, visualData: Map<number, { coordinates: string | null; color: string | null; icon: string | null }>): Promise<void> {
+      const sections = parseResearchSections(result.text);
+      let merged = 0;
+
+      for (const section of sections) {
+        let sectionMerged = false;
+        for (const el of section.elements) {
+          let eidx = el.eidx || null;
+          if (!eidx) {
+            const locator = el.css || el.xpath || (el.aria ? `role=${el.aria.role}[name="${el.aria.text}"]` : null);
+            if (locator) eidx = await this.explorer.getEidxByLocator(locator, section.containerCss);
+          }
+          if (!eidx) continue;
+
+          const vis = visualData.get(eidx);
+          if (!vis) continue;
+          Object.assign(el, Object.fromEntries(Object.entries(vis).filter(([, v]) => v)));
+          sectionMerged = true;
+          merged++;
+        }
+        if (sectionMerged) result.rebuildSectionInText(section);
+      }
+      debugLog(`Merged visual props for ${merged} elements`);
+    }
+
+    async backfillCoordinates(result: ResearchResult): Promise<void> {
+      const page = this.explorer.playwrightHelper.page;
+      const sections = parseResearchSections(result.text);
+      const eidxWithoutCoords: number[] = [];
+      for (const section of sections) {
+        for (const el of section.elements) {
+          if (el.eidx && !el.coordinates) eidxWithoutCoords.push(el.eidx);
+        }
+      }
+      if (eidxWithoutCoords.length === 0) return;
+
+      const webElements = await WebElement.fromEidxList(page, eidxWithoutCoords);
+      if (webElements.length === 0) return;
+
+      const rectMap = new Map(webElements.map((w) => [w.eidx!, w]));
+      for (const section of sections) {
+        let changed = false;
+        for (const el of section.elements) {
+          if (el.eidx && !el.coordinates) {
+            const w = rectMap.get(el.eidx);
+            if (w) {
+              el.coordinates = w.coordinates;
+              changed = true;
+            }
+          }
+        }
+        if (changed) result.rebuildSectionInText(section);
+      }
+    }
+  };
+}
+
+export interface VisualAnalysisResult {
+  elements: Map<number, { coordinates: string | null; color: string | null; icon: string | null }>;
+  pagePurpose: string | null;
+  primaryActions: string[] | null;
+  focusedSection: string | null;
+}
+
+export interface CoordinateMethods {
+  analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult>;
+  mergeVisualData(result: ResearchResult, visualData: Map<number, { coordinates: string | null; color: string | null; icon: string | null }>): Promise<void>;
+  backfillCoordinates(result: ResearchResult): Promise<void>;
+  visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise<number>;
+}