@auxiora/screen 1.0.0

package/dist/types.d.ts

@@ -0,0 +1,99 @@
+/** Rectangular region on screen. */
+export interface ScreenBounds {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+/** A captured screenshot or region. */
+export interface ScreenCapture {
+    /** Raw image data (PNG). */
+    image: Buffer;
+    /** When the capture was taken. */
+    timestamp: number;
+    /** Pixel dimensions. */
+    dimensions: {
+        width: number;
+        height: number;
+    };
+}
+/** Result of OCR text extraction. */
+export interface OCRResult {
+    /** Full extracted text. */
+    text: string;
+    /** Detected text regions with positions. */
+    regions: OCRRegion[];
+    /** Overall confidence score (0-1). */
+    confidence: number;
+}
+/** A single region of detected text. */
+export interface OCRRegion {
+    text: string;
+    bounds: ScreenBounds;
+    confidence: number;
+}
+/** A UI element detected on screen. */
+export interface ScreenElement {
+    /** Element type (button, input, link, text, image, icon, etc.). */
+    type: string;
+    /** Bounding box on screen. */
+    bounds: ScreenBounds;
+    /** Visible text or label. */
+    text?: string;
+    /** Whether the element appears interactable. */
+    interactable: boolean;
+}
+/** Types of desktop automation actions. */
+export type DesktopActionType = 'click' | 'type' | 'scroll' | 'keypress';
+/** A desktop automation action to perform. */
+export interface DesktopAction {
+    type: DesktopActionType;
+    /** Target location or element description. */
+    target?: {
+        x: number;
+        y: number;
+    } | string;
+    /** Action-specific parameters. */
+    params?: {
+        /** Text to type (for 'type' action). */
+        text?: string;
+        /** Key combo to press (for 'keypress' action). */
+        key?: string;
+        /** Scroll delta (for 'scroll' action). */
+        deltaX?: number;
+        deltaY?: number;
+        /** Mouse button (for 'click' action). */
+        button?: 'left' | 'right' | 'middle';
+        /** Number of clicks (for 'click' action). */
+        clickCount?: number;
+    };
+}
+/** Configuration for the screen system. */
+export interface ScreenConfig {
+    /** Whether screen capture is allowed. */
+    captureEnabled: boolean;
+    /** Whether OCR is enabled. */
+    ocrEnabled: boolean;
+    /** Whether desktop automation is enabled. */
+    automationEnabled: boolean;
+    /** Maximum capture width in pixels. */
+    maxCaptureWidth: number;
+    /** Maximum capture height in pixels. */
+    maxCaptureHeight: number;
+    /** Trust level required for screen capture. */
+    captureRequiredTrust: number;
+    /** Trust level required for automation actions. */
+    automationRequiredTrust: number;
+}
+export declare const DEFAULT_SCREEN_CONFIG: ScreenConfig;
+/** Interface for capture backends (Tauri bridge, mock, etc.). */
+export interface CaptureBackend {
+    captureScreen(): Promise<ScreenCapture>;
+    captureRegion(bounds: ScreenBounds): Promise<ScreenCapture>;
+    captureWindow(title: string): Promise<ScreenCapture>;
+}
+/** Interface for vision model backend used by OCR and analyzer. */
+export interface VisionBackend {
+    analyzeImage(image: Buffer, prompt: string): Promise<string>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,MAAM,WAAW,YAAY;IAC3B,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,uCAAuC;AACvC,MAAM,WAAW,aAAa;IAC5B,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,wBAAwB;IACxB,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/C;AAED,qCAAqC;AACrC,MAAM,WAAW,SAAS;IACxB,2BAA2B;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,wCAAwC;AACxC,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,uCAAuC;AACvC,MAAM,WAAW,aAAa;IAC5B,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,MAAM,EAAE,YAAY,CAAC;IACrB,6BAA6B;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,2CAA2C;AAC3C,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,MAAM,GAAG,QAAQ,GAAG,UAAU,CAAC;AAEzE,8CAA8C;AAC9C,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,iBAAiB,CAAC;IACxB,8CAA8C;IAC9C,MAAM,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC;IAC3C,kCAAkC;IAClC,MAAM,CAAC,EAAE;QACP,wCAAwC;QACxC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,kDAAkD;QAClD,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,0CAA0C;QAC1C,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,yCAAyC;QACzC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;QACrC,6CAA6C;QAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACH;AAED,2CAA2C;AAC3C,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,cAAc,EAAE,OAAO,CAAC;IACxB,8BAA8B;IAC9B,UAAU,EAAE,OAAO,CAAC;IACpB,6CAA6C;IAC7C,iBAAiB,EAAE,OAAO,CAAC;IAC3B,uCAAuC;IACvC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,+CAA+C;IAC/C,oBAAoB,EAAE,MAAM,CAAC;IAC7B,mDAAmD;IACnD,uBAAuB,EAAE,MAAM,CAAC;CACjC;AAED,eAAO,MAAM,qBAAqB,EAAE,YAQnC,CAAC;AAEF,iEAAiE;AACjE,MAAM,WAAW,cAAc;IAC7B,aAAa,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,aAAa,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC5D,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;CACtD;AAED,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9D"}

package/dist/types.js

@@ -0,0 +1,10 @@
+export const DEFAULT_SCREEN_CONFIG = {
+    captureEnabled: true,
+    ocrEnabled: true,
+    automationEnabled: false,
+    maxCaptureWidth: 3840,
+    maxCaptureHeight: 2160,
+    captureRequiredTrust: 2,
+    automationRequiredTrust: 3,
+};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAyFA,MAAM,CAAC,MAAM,qBAAqB,GAAiB;IACjD,cAAc,EAAE,IAAI;IACpB,UAAU,EAAE,IAAI;IAChB,iBAAiB,EAAE,KAAK;IACxB,eAAe,EAAE,IAAI;IACrB,gBAAgB,EAAE,IAAI;IACtB,oBAAoB,EAAE,CAAC;IACvB,uBAAuB,EAAE,CAAC;CAC3B,CAAC"}

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@auxiora/screen",
+  "version": "1.0.0",
+  "description": "Screen capture, OCR, desktop automation, and screen analysis",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@auxiora/core": "1.0.0",
+    "@auxiora/autonomy": "1.0.0"
+  },
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist",
+    "typecheck": "tsc --noEmit"
+  }
+}

package/src/analyzer.ts

@@ -0,0 +1,27 @@
+import type { VisionBackend } from './types.js';
+
+/**
+ * Screen analyzer — sends screenshots to a vision model for analysis.
+ */
+export class ScreenAnalyzer {
+  private vision: VisionBackend;
+
+  constructor(vision: VisionBackend) {
+    this.vision = vision;
+  }
+
+  /**
+   * Analyze a screenshot, optionally answering a specific question about it.
+   * @param image - PNG screenshot buffer.
+   * @param question - Optional question to answer about the screen content.
+   * @returns Natural language analysis of the screen.
+   */
+  async analyzeScreen(image: Buffer, question?: string): Promise<string> {
+    const prompt = question
+      ? `Look at this screenshot and answer: ${question}`
+      : 'Describe what is visible on this screen. Include any visible text, UI elements, ' +
+        'active applications, and notable content. Be concise but thorough.';
+
+    return this.vision.analyzeImage(image, prompt);
+  }
+}

package/src/automation.ts

@@ -0,0 +1,122 @@
+import type { TrustGate } from '@auxiora/autonomy';
+import type { DesktopAction, ScreenConfig } from './types.js';
+import { DEFAULT_SCREEN_CONFIG } from './types.js';
+
+/** Result of an automation action. */
+export interface AutomationResult {
+  success: boolean;
+  action: DesktopAction;
+  error?: string;
+}
+
+/** Backend interface for performing desktop automation. */
+export interface AutomationBackend {
+  click(x: number, y: number, button?: string, clickCount?: number): Promise<void>;
+  typeText(text: string): Promise<void>;
+  keypress(key: string): Promise<void>;
+  scroll(deltaX: number, deltaY: number): Promise<void>;
+}
+
+/**
+ * Desktop automation with trust-gated actions.
+ * Every action is checked against TrustGate before execution.
+ */
+export class DesktopAutomation {
+  private backend: AutomationBackend;
+  private gate: TrustGate;
+  private config: ScreenConfig;
+
+  constructor(backend: AutomationBackend, gate: TrustGate, config?: Partial<ScreenConfig>) {
+    this.backend = backend;
+    this.gate = gate;
+    this.config = { ...DEFAULT_SCREEN_CONFIG, ...config };
+  }
+
+  /** Execute a desktop action (trust-gated). */
+  async execute(action: DesktopAction): Promise<AutomationResult> {
+    if (!this.config.automationEnabled) {
+      return { success: false, action, error: 'Desktop automation is disabled' };
+    }
+
+    const gateResult = this.gate.gate(
+      'system',
+      `screen:${action.type}`,
+      this.config.automationRequiredTrust as 0 | 1 | 2 | 3 | 4,
+    );
+    if (!gateResult.allowed) {
+      return { success: false, action, error: gateResult.message };
+    }
+
+    try {
+      switch (action.type) {
+        case 'click': {
+          const target = action.target;
+          if (!target || typeof target === 'string') {
+            return { success: false, action, error: 'Click requires x,y coordinates' };
+          }
+          await this.backend.click(
+            target.x,
+            target.y,
+            action.params?.button ?? 'left',
+            action.params?.clickCount ?? 1,
+          );
+          break;
+        }
+        case 'type': {
+          const text = action.params?.text;
+          if (!text) {
+            return { success: false, action, error: 'Type action requires text' };
+          }
+          await this.backend.typeText(text);
+          break;
+        }
+        case 'keypress': {
+          const key = action.params?.key;
+          if (!key) {
+            return { success: false, action, error: 'Keypress action requires key' };
+          }
+          await this.backend.keypress(key);
+          break;
+        }
+        case 'scroll': {
+          await this.backend.scroll(
+            action.params?.deltaX ?? 0,
+            action.params?.deltaY ?? 0,
+          );
+          break;
+        }
+        default:
+          return { success: false, action, error: `Unknown action type: ${(action as any).type}` };
+      }
+      return { success: true, action };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : 'Unknown error';
+      return { success: false, action, error: message };
+    }
+  }
+
+  /** Click at coordinates (trust-gated). */
+  async click(x: number, y: number, button: 'left' | 'right' | 'middle' = 'left'): Promise<AutomationResult> {
+    return this.execute({ type: 'click', target: { x, y }, params: { button } });
+  }
+
+  /** Type text (trust-gated). */
+  async type(text: string): Promise<AutomationResult> {
+    return this.execute({ type: 'type', params: { text } });
+  }
+
+  /** Press a key combo (trust-gated). */
+  async keypress(key: string): Promise<AutomationResult> {
+    return this.execute({ type: 'keypress', params: { key } });
+  }
+
+  /** Scroll (trust-gated). */
+  async scroll(deltaX: number, deltaY: number): Promise<AutomationResult> {
+    return this.execute({ type: 'scroll', params: { deltaX, deltaY } });
+  }
+
+  /** Find an element by description and click it (trust-gated). */
+  async findAndClick(description: string): Promise<AutomationResult> {
+    return this.execute({ type: 'click', target: description });
+  }
+}

package/src/capture.ts

@@ -0,0 +1,53 @@
+import type { CaptureBackend, ScreenBounds, ScreenCapture, ScreenConfig } from './types.js';
+import { DEFAULT_SCREEN_CONFIG } from './types.js';
+
+/**
+ * Screen capturer with injectable backend.
+ * Uses a CaptureBackend for actual capture (Tauri bridge on desktop, mock in tests).
+ */
+export class ScreenCapturer {
+  private backend: CaptureBackend;
+  private config: ScreenConfig;
+
+  constructor(backend: CaptureBackend, config?: Partial<ScreenConfig>) {
+    this.backend = backend;
+    this.config = { ...DEFAULT_SCREEN_CONFIG, ...config };
+  }
+
+  /** Capture the entire screen. */
+  async captureScreen(): Promise<ScreenCapture> {
+    if (!this.config.captureEnabled) {
+      throw new Error('Screen capture is disabled');
+    }
+    return this.backend.captureScreen();
+  }
+
+  /** Capture a specific region of the screen. */
+  async captureRegion(bounds: ScreenBounds): Promise<ScreenCapture> {
+    if (!this.config.captureEnabled) {
+      throw new Error('Screen capture is disabled');
+    }
+    if (bounds.width > this.config.maxCaptureWidth || bounds.height > this.config.maxCaptureHeight) {
+      throw new Error(
+        `Region exceeds max capture dimensions (${this.config.maxCaptureWidth}x${this.config.maxCaptureHeight})`
+      );
+    }
+    return this.backend.captureRegion(bounds);
+  }
+
+  /** Capture a window by title. */
+  async captureWindow(title: string): Promise<ScreenCapture> {
+    if (!this.config.captureEnabled) {
+      throw new Error('Screen capture is disabled');
+    }
+    if (!title || title.trim().length === 0) {
+      throw new Error('Window title must not be empty');
+    }
+    return this.backend.captureWindow(title);
+  }
+
+  /** Get current config. */
+  getConfig(): ScreenConfig {
+    return { ...this.config };
+  }
+}

package/src/index.ts

@@ -0,0 +1,17 @@
+export type {
+  ScreenBounds,
+  ScreenCapture,
+  OCRResult,
+  OCRRegion,
+  ScreenElement,
+  DesktopAction,
+  DesktopActionType,
+  ScreenConfig,
+  CaptureBackend,
+  VisionBackend,
+} from './types.js';
+export { DEFAULT_SCREEN_CONFIG } from './types.js';
+export { ScreenCapturer } from './capture.js';
+export { OCREngine } from './ocr.js';
+export { DesktopAutomation, type AutomationResult, type AutomationBackend } from './automation.js';
+export { ScreenAnalyzer } from './analyzer.js';

package/src/ocr.ts

@@ -0,0 +1,111 @@
+import type { OCRResult, ScreenElement, VisionBackend } from './types.js';
+
+/**
+ * OCR engine that uses a vision model to extract text and detect UI elements.
+ */
+export class OCREngine {
+  private vision: VisionBackend;
+
+  constructor(vision: VisionBackend) {
+    this.vision = vision;
+  }
+
+  /** Extract text from an image using the vision model. */
+  async extractText(image: Buffer): Promise<OCRResult> {
+    const response = await this.vision.analyzeImage(
+      image,
+      'Extract all visible text from this screenshot. Return the text content organized by position, ' +
+      'from top-left to bottom-right. For each distinct text block, provide the text and its approximate ' +
+      'position as a percentage of the image dimensions (x%, y%, width%, height%). ' +
+      'Format: TEXT|||x,y,w,h|||confidence\nOne per line.'
+    );
+
+    return this.parseTextResponse(response);
+  }
+
+  /** Find UI elements in an image using the vision model. */
+  async findElements(image: Buffer): Promise<ScreenElement[]> {
+    const response = await this.vision.analyzeImage(
+      image,
+      'Identify all UI elements in this screenshot. For each element, provide:\n' +
+      '- type (button, input, link, text, image, icon, checkbox, dropdown, menu, tab)\n' +
+      '- approximate bounding box as percentages (x%, y%, width%, height%)\n' +
+      '- visible text or label\n' +
+      '- whether it appears interactable (true/false)\n' +
+      'Format: type|||x,y,w,h|||text|||interactable\nOne per line.'
+    );
+
+    return this.parseElementsResponse(response);
+  }
+
+  private parseTextResponse(response: string): OCRResult {
+    const lines = response.split('\n').filter(l => l.trim().length > 0);
+    const regions: OCRResult['regions'] = [];
+    let totalConfidence = 0;
+
+    for (const line of lines) {
+      const parts = line.split('|||').map(s => s.trim());
+      if (parts.length < 2) continue;
+
+      const text = parts[0];
+      const boundsStr = parts[1];
+      const confidence = parts[2] ? parseFloat(parts[2]) : 0.8;
+
+      const coords = boundsStr.split(',').map(s => parseFloat(s.trim()));
+      if (coords.length < 4 || coords.some(n => isNaN(n))) continue;
+
+      regions.push({
+        text,
+        bounds: {
+          x: Math.round(coords[0]),
+          y: Math.round(coords[1]),
+          width: Math.round(coords[2]),
+          height: Math.round(coords[3]),
+        },
+        confidence: isNaN(confidence) ? 0.8 : Math.max(0, Math.min(1, confidence)),
+      });
+      totalConfidence += confidence;
+    }
+
+    const fullText = regions.map(r => r.text).join('\n');
+    const avgConfidence = regions.length > 0 ? totalConfidence / regions.length : 0;
+
+    return {
+      text: fullText,
+      regions,
+      confidence: Math.max(0, Math.min(1, avgConfidence)),
+    };
+  }
+
+  private parseElementsResponse(response: string): ScreenElement[] {
+    const lines = response.split('\n').filter(l => l.trim().length > 0);
+    const elements: ScreenElement[] = [];
+
+    for (const line of lines) {
+      const parts = line.split('|||').map(s => s.trim());
+      if (parts.length < 4) continue;
+
+      const type = parts[0];
+      const boundsStr = parts[1];
+      const text = parts[2] || undefined;
+      const interactable = parts[3] === 'true';
+
+      const coords = boundsStr.split(',').map(s => parseFloat(s.trim()));
+      if (coords.length < 4 || coords.some(n => isNaN(n))) continue;
+
+      elements.push({
+        type,
+        bounds: {
+          x: Math.round(coords[0]),
+          y: Math.round(coords[1]),
+          width: Math.round(coords[2]),
+          height: Math.round(coords[3]),
+        },
+        text,
+        interactable,
+      });
+    }
+
+    return elements;
+  }
+}

package/src/types.ts

@@ -0,0 +1,110 @@
+/** Rectangular region on screen. */
+export interface ScreenBounds {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+/** A captured screenshot or region. */
+export interface ScreenCapture {
+  /** Raw image data (PNG). */
+  image: Buffer;
+  /** When the capture was taken. */
+  timestamp: number;
+  /** Pixel dimensions. */
+  dimensions: { width: number; height: number };
+}
+
+/** Result of OCR text extraction. */
+export interface OCRResult {
+  /** Full extracted text. */
+  text: string;
+  /** Detected text regions with positions. */
+  regions: OCRRegion[];
+  /** Overall confidence score (0-1). */
+  confidence: number;
+}
+
+/** A single region of detected text. */
+export interface OCRRegion {
+  text: string;
+  bounds: ScreenBounds;
+  confidence: number;
+}
+
+/** A UI element detected on screen. */
+export interface ScreenElement {
+  /** Element type (button, input, link, text, image, icon, etc.). */
+  type: string;
+  /** Bounding box on screen. */
+  bounds: ScreenBounds;
+  /** Visible text or label. */
+  text?: string;
+  /** Whether the element appears interactable. */
+  interactable: boolean;
+}
+
+/** Types of desktop automation actions. */
+export type DesktopActionType = 'click' | 'type' | 'scroll' | 'keypress';
+
+/** A desktop automation action to perform. */
+export interface DesktopAction {
+  type: DesktopActionType;
+  /** Target location or element description. */
+  target?: { x: number; y: number } | string;
+  /** Action-specific parameters. */
+  params?: {
+    /** Text to type (for 'type' action). */
+    text?: string;
+    /** Key combo to press (for 'keypress' action). */
+    key?: string;
+    /** Scroll delta (for 'scroll' action). */
+    deltaX?: number;
+    deltaY?: number;
+    /** Mouse button (for 'click' action). */
+    button?: 'left' | 'right' | 'middle';
+    /** Number of clicks (for 'click' action). */
+    clickCount?: number;
+  };
+}
+
+/** Configuration for the screen system. */
+export interface ScreenConfig {
+  /** Whether screen capture is allowed. */
+  captureEnabled: boolean;
+  /** Whether OCR is enabled. */
+  ocrEnabled: boolean;
+  /** Whether desktop automation is enabled. */
+  automationEnabled: boolean;
+  /** Maximum capture width in pixels. */
+  maxCaptureWidth: number;
+  /** Maximum capture height in pixels. */
+  maxCaptureHeight: number;
+  /** Trust level required for screen capture. */
+  captureRequiredTrust: number;
+  /** Trust level required for automation actions. */
+  automationRequiredTrust: number;
+}
+
+export const DEFAULT_SCREEN_CONFIG: ScreenConfig = {
+  captureEnabled: true,
+  ocrEnabled: true,
+  automationEnabled: false,
+  maxCaptureWidth: 3840,
+  maxCaptureHeight: 2160,
+  captureRequiredTrust: 2,
+  automationRequiredTrust: 3,
+};
+
+/** Interface for capture backends (Tauri bridge, mock, etc.). */
+export interface CaptureBackend {
+  captureScreen(): Promise<ScreenCapture>;
+  captureRegion(bounds: ScreenBounds): Promise<ScreenCapture>;
+  captureWindow(title: string): Promise<ScreenCapture>;
+}
+
+/** Interface for vision model backend used by OCR and analyzer. */
+export interface VisionBackend {
+  analyzeImage(image: Buffer, prompt: string): Promise<string>;
+}