@elizaos/plugin-vision 2.0.0-beta.1 → 2.0.3-beta.6

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shaw Walters and elizaOS Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,340 +1,112 @@
-# ElizaOS Vision Plugin
+# @elizaos/plugin-vision
 
-A powerful visual perception plugin for ElizaOS that provides agents with
-real-time camera integration and scene analysis capabilities. This plugin
-enables agents to "see" their environment, describe scenes, detect people and
-objects, and make decisions based on visual input.
+Visual perception plugin for elizaOS — gives Eliza agents real-time awareness of their camera feed and/or screen through scene analysis, object/person detection, OCR, face recognition, and entity tracking.
 
-## Features
+## What it does
 
-### Phase 1 (Implemented)
-
-- ✅ Camera detection and connection (platform-specific)
-- ✅ Real-time frame capture and processing
-- ✅ Scene description using Vision Language Models (VLM)
-- ✅ Motion-based object detection
-- ✅ Basic person detection with pose estimation
-- ✅ Configurable pixel change threshold
-- ✅ Image capture action with base64 attachments
-- ✅ Non-dynamic vision provider (always active)
-- ✅ Integration with autonomy plugin (kill switch)
-
-### Phase 2 (Implemented)
-
-- ✅ Enhanced object detection with COCO-like classification
-- ✅ Advanced pose detection with keypoint estimation
-- ✅ Improved person detection and tracking
-- ✅ Object classification (person, monitor, chair, keyboard, furniture, etc.)
-- ✅ Configurable computer vision models
-- ✅ Fallback to motion detection when CV is disabled
-
-### Phase 3 (Implemented)
-
-- ✅ Real-time object tracking with IDs
-- ✅ Face detection and recognition
-- ✅ Screen capture and OCR integration
-- ✅ Entity tracking with persistent IDs
-- ✅ Multi-display support
-- ✅ Circuit breaker pattern for error resilience
-- ✅ Florence2 model integration for advanced scene understanding
-- ✅ Worker-based processing for high-FPS operations
-
-### Phase 4 (Planned)
-
-- 🔄 WebAssembly (WASM) integration for browser compatibility
-- 🔄 Gesture recognition
-- 🔄 Emotion detection
-- 🔄 Advanced scene understanding and spatial relationships
+- Captures frames from a connected camera (macOS/Linux/Windows) or the host screen.
+- Describes scenes by routing images through `runtime.useModel(IMAGE_DESCRIPTION)` — compatible with any registered VLM (local or cloud).
+- Detects and tracks people, objects, and faces across frames with persistent entity IDs.
+- Reads text on screen through the generic Apple Vision/doCTR OCR service and the coordinate-aware OCR registry used by computeruse: Windows.Media.Ocr on Windows, Tesseract on Linux when available, and the RapidOCR adapter as the portable fallback.
+- Exposes all capabilities through a single `VISION` action and a `VISION_PERCEPTION` context provider.
 
 ## Installation
 
-### TypeScript (Primary)
-
 ```bash
 npm install @elizaos/plugin-vision
-# or
-cd plugins/plugin-vision
-bun install
-bun run build
 ```
-### Camera Tools Required
-
-The plugin requires platform-specific camera tools:
-
-- **macOS**: `brew install imagesnap`
-- **Linux**: `sudo apt-get install fswebcam`
-- **Windows**: Install ffmpeg and add to PATH
-
-## Configuration
-
-### Environment Variables
-
-```env
-# Camera selection (partial name match, case-insensitive)
-CAMERA_NAME=obsbot
 
-# Pixel change threshold (percentage, default: 50)
-PIXEL_CHANGE_THRESHOLD=30
+### Platform camera tools (required for camera mode)
 
-# Enable advanced computer vision features (default: false)
-ENABLE_OBJECT_DETECTION=true
-ENABLE_POSE_DETECTION=true
-ENABLE_FACE_RECOGNITION=false
+| Platform | Tool |
+|----------|------|
+| macOS | `brew install imagesnap` |
+| Linux | `sudo apt-get install fswebcam` |
+| Windows | Install ffmpeg and add to PATH |
 
-# Vision mode: OFF, CAMERA, SCREEN, BOTH
-VISION_MODE=CAMERA
+Screen capture and OCR work without these tools.
 
-# Update intervals (milliseconds)
-TF_UPDATE_INTERVAL=1000
-VLM_UPDATE_INTERVAL=10000
+## Enabling the plugin
 
-# Screen capture settings
-SCREEN_CAPTURE_INTERVAL=2000
-OCR_ENABLED=true
-```
-
-### Character Configuration
+Add it to your character's plugin list:
 
 ```json
 {
-  "name": "VisionAgent",
+  "name": "MyAgent",
   "plugins": ["@elizaos/plugin-vision"],
   "settings": {
     "CAMERA_NAME": "obsbot",
-    "PIXEL_CHANGE_THRESHOLD": "30",
-    "ENABLE_OBJECT_DETECTION": "true",
-    "ENABLE_POSE_DETECTION": "true"
+    "VISION_MODE": "CAMERA"
   }
 }
 ```
 
-## Actions
-
-### DESCRIBE_SCENE
-
-Analyzes the current visual scene and provides a detailed description.
-
-**Similes**: `ANALYZE_SCENE`, `WHAT_DO_YOU_SEE`, `VISION_CHECK`, `LOOK_AROUND`
-
-**Example**:
-
-```
-User: "What do you see?"
-Agent: "Looking through the camera, I see a home office setup with a person sitting at a desk. There are 2 monitors, a keyboard, and various desk accessories. I detected 5 objects total: 1 person, 2 monitors, 1 keyboard, and 1 chair."
-```
-
-### CAPTURE_IMAGE
-
-Captures the current frame and returns it as a base64 image attachment.
-
-**Similes**: `TAKE_PHOTO`, `SCREENSHOT`, `CAPTURE_FRAME`, `TAKE_PICTURE`
-
-**Example**:
-
-```
-User: "Take a photo"
-Agent: "I've captured an image from the camera." [Image attached]
-```
-
-### SET_VISION_MODE
-
-Changes the vision mode (OFF, CAMERA, SCREEN, or BOTH).
+The plugin auto-enables when `config.features.vision` is truthy or `config.media.vision.provider` is set.
 
-**Similes**: `CHANGE_VISION_MODE`, `SET_VISION`, `TOGGLE_VISION`
-
-### NAME_ENTITY
-
-Assigns a name to a detected entity for tracking.
-
-**Similes**: `LABEL_ENTITY`, `NAME_OBJECT`, `IDENTIFY_ENTITY`
-
-### IDENTIFY_PERSON
-
-Identifies a person using face recognition (requires face recognition to be enabled).
-
-**Similes**: `RECOGNIZE_PERSON`, `IDENTIFY_FACE`
-
-### TRACK_ENTITY
-
-Starts tracking an entity with a persistent ID.
+## Configuration
 
-**Similes**: `START_TRACKING`, `FOLLOW_ENTITY`
+| Setting | Default | Description |
+|---------|---------|-------------|
+| `CAMERA_NAME` | auto | Partial name match for camera device selection (case-insensitive) |
+| `VISION_MODE` | `CAMERA` | `OFF` / `CAMERA` / `SCREEN` / `BOTH` |
+| `PIXEL_CHANGE_THRESHOLD` | `50` | % pixel change required before triggering a VLM scene update |
+| `VLM_UPDATE_INTERVAL` | `10000` | ms between VLM scene-describe calls |
+| `SCREEN_CAPTURE_INTERVAL` | `2000` | ms between screen captures |
+| `OCR_ENABLED` | `true` | Enable OCR on screen tiles |
+| `ENABLE_OBJECT_DETECTION` | `false` | ggml YOLOv8n object detection (`native/yolo.cpp`) |
+| `ENABLE_POSE_DETECTION` | `false` | Heuristic person detection (ggml pose pending) |
+| `ENABLE_FACE_RECOGNITION` | `false` | Native ggml face recognition (BlazeFace + 128-d embed via `native/face-cpp`) |
+| `ENTITY_TIMEOUT` | `30000` | ms before an inactive entity is evicted from tracking |
+
+All settings can also be prefixed with `VISION_` (e.g. `VISION_CAMERA_NAME`).
 
-### KILL_AUTONOMOUS
+## Actions
 
-Stops the autonomous agent loop (useful for debugging with autonomy plugin).
+The plugin registers a single `VISION` action that routes to one of these sub-operations based on explicit `action` parameter or natural-language inference:
 
-**Similes**: `STOP_AUTONOMOUS`, `HALT_AUTONOMOUS`, `KILL_AUTO_LOOP`
+| Sub-operation | Trigger examples | What it does |
+|--------------|-----------------|-------------|
+| `describe` | "what do you see?", "describe the scene" | Returns the current VLM scene description |
+| `capture` | "take a photo", "screenshot" | Captures a frame and returns it as a base64 image attachment |
+| `set_mode` | "set vision mode to screen" | Switches between `OFF`, `CAMERA`, `SCREEN`, `BOTH` |
+| `enable_camera` / `disable_camera` | "turn on the camera" | Toggles camera input |
+| `enable_screen` / `disable_screen` | "enable screen capture" | Toggles screen input |
+| `name_entity` | "the person is named Alice" | Assigns a display name to the most prominent tracked entity |
+| `identify_person` | "who is that?" | Lists tracked people with names and presence duration |
+| `track_entity` | "track the person in the red shirt" | Refreshes entity tracking and reports statistics |
 
 ## Vision Provider
 
-The vision provider is **non-dynamic** (always active) and provides:
-
-- Current scene description
-- Camera connection status
-- Detected objects count and types
-- Detected people count with poses
-- Scene change percentage
-- Time since last update
-
-### Provider Data Structure
-
-```typescript
-{
-  visionAvailable: boolean,
-  sceneDescription: string,
-  cameraStatus: string,
-  cameraId?: string,
-  peopleCount?: number,
-  objectCount?: number,
-  sceneAge?: number,
-  lastChange?: number
-}
-```
-
-## Detection Modes
-
-### Motion-Based Detection (Default)
-
-- Lightweight and fast
-- Detects movement between frames
-- Groups motion blocks into objects
-- Basic size-based classification
-
-### Advanced Computer Vision (Optional)
-
-Enable with `ENABLE_OBJECT_DETECTION=true` and/or `ENABLE_POSE_DETECTION=true`
-
-- **Object Detection**: Enhanced object recognition with COCO-like classes
-- **Pose Detection**: 17-keypoint pose estimation
-- **Better Classification**: Distinguishes between person, monitor, chair,
-  keyboard, etc.
-- **Higher Accuracy**: Edge detection and color variance analysis
-
-## Integration with Autonomy
-
-- Continuous environmental monitoring
-- Autonomous responses to visual changes
-- Visual memory persistence
-- Scene-based decision making
-
-Example autonomous behavior:
-
-```typescript
-// Agent autonomously monitors environment
-"I notice someone just entered the room.";
-"The lighting has changed significantly.";
-"A new object has appeared on the desk.";
-```
-
-## Performance Considerations
-
-- Frame processing runs every 100ms by default
-- VLM is only called when pixel change exceeds threshold
-- Motion detection uses 64x64 pixel blocks with 50% overlap
-- Advanced CV models add ~50-100ms processing time per frame
-- Memory usage increases with resolution (1280x720 recommended)
-
-## Security & Privacy
-
-- Camera access requires system permissions
-- No images are stored permanently by default
-- All processing happens locally
-- Base64 images in messages are ephemeral
-- Consider privacy implications in your implementation
-
-## Architecture
-
-```
-plugin-vision/
-├── README.md                 # This file
-├── package.json              # TypeScript package config
-├── src/                      # TypeScript implementation (primary)
-│   ├── index.ts              # Plugin entry point
-│   ├── service.ts            # Vision service
-│   ├── provider.ts           # Vision provider
-│   ├── action.ts             # All actions
-│   ├── entity-tracker.ts     # Entity tracking
-│   ├── screen-capture.ts     # Screen capture
-│   ├── ocr-service.ts        # OCR service
-│   ├── face-recognition.ts   # Face recognition
-│   ├── florence2-model.ts    # Florence2 model integration
-│   ├── vision-worker-manager.ts # Worker management
-│   └── tests/                # E2E tests
-```
-
-## Development
-
-### Running Tests
-
-```bash
-# Run E2E tests
-cd plugins/plugin-vision
-npx vitest
-
-# Run local E2E tests
-bun run test:e2e:local
-```
-
-### Test Coverage
-
-- Service initialization
-- Camera detection and connection
-- Scene description generation
-- Object and person detection
-- Image capture
-- Provider integration
-- Autonomy integration
-
-## Troubleshooting
-
-### No Camera Detected
-
-1. Ensure camera tools are installed (imagesnap/fswebcam/ffmpeg)
-2. Check camera permissions in system settings
-3. Try without CAMERA_NAME to use default camera
-4. Verify camera is not in use by another application
-
-### Poor Object Detection
-
-1. Ensure good lighting conditions
-2. Adjust PIXEL_CHANGE_THRESHOLD (lower = more sensitive)
-3. Enable advanced CV with ENABLE_OBJECT_DETECTION=true
-4. Check camera resolution (higher is better for detection)
-
-### High CPU Usage
-
-1. Increase frame processing interval in code
-2. Disable advanced CV features if not needed
-3. Reduce camera resolution
-4. Increase pixel change threshold
-
-## Future Roadmap
-
-### Phase 3: WebAssembly Integration
-
-- TensorFlow.js WASM backend
-- Browser-compatible vision processing
-- Real-time object tracking
-- Face detection and recognition
-
-### Phase 4: Advanced Features
+`VISION_PERCEPTION` is injected into agent context during turns in the `media` and `browser` contexts. It provides:
 
-- Gesture recognition
-- Emotion detection
-- Scene understanding
-- Spatial relationship mapping
-- Multi-camera support
+- Current scene description text
+- Camera / screen connection status and mode
+- Detected people (count, poses, facings)
+- Detected objects (types)
+- Active tracked entities with duration
+- Recently-departed entities
+- Screen tile OCR text and UI element list (when screen mode is active)
 
-## Contributing
+## Detection backends
 
-Contributions are welcome! Please see the main ElizaOS repository for
-contribution guidelines.
+| Capability | Default backend | Optional / alternative |
+|-----------|-----------------|----------------------|
+| Scene description | VLM via `runtime.useModel(IMAGE_DESCRIPTION)` | Any registered IMAGE_DESCRIPTION provider |
+| Object detection | YOLOv8n ggml via `native/yolo.cpp` (`src/yolo-detector.ts`); build with `bun run build:native` + `bun run build:weights`. Service degrades to motion/heuristic + VLM when the lib/GGUF are absent. | — (TensorFlow.js path removed) |
+| Pose detection | Heuristic person detection (motion-derived) | Planned ggml MoveNet port |
+| OCR | Generic OCR uses Apple Vision (darwin, when a provider is registered) → doCTR ggml (`native/doctr.cpp`). Coordinate OCR for computeruse prefers Windows.Media.Ocr (Windows) → Tesseract CLI or vendored bundle (Linux) → RapidOCR adapter. | Native/mobile bridges can register platform OCR providers; no ONNX OCR path. |
+| Set-of-Marks grounding | `src/som.ts` fuses GGUF YOLO icon boxes + OCR text boxes into a deduplicated, 1-indexed numbered set (icon-over-text suppression + NMS) and renders a numbered-overlay PNG via `sharp`. `src/set-of-marks-provider.ts` registers it into plugin-computeruse's `detect_elements` seam at boot (best-effort; degrades to text-only marks when the GGUF detector is absent). | trycua/cua OmniParser parity (#9170 M9) |
+| Face recognition | Native ggml BlazeFace + 128-d embed (`face-detector-ggml.ts`, `face-recognition-ggml.ts`, `native/face-cpp`); disabled until the lib/GGUF artifacts land. No tfjs/face-api.js path. | MediaPipe BlazeFace migration shim is deprecated. |
 
-## License
+## Platform notes
 
-MIT
+- **Node.js only.** Mobile (iOS, Android) registers a `MobileCameraSource` (`src/mobile/capacitor-camera.ts`) bridged by plugin-ios / plugin-aosp.
+- **Camera tools** (`imagesnap` / `fswebcam` / `ffmpeg`) are required for camera mode; screen capture and OCR work without them.
+- **Native detectors and OCR** (`native/yolo.cpp`, `native/doctr.cpp`, and the coordinate-OCR providers) run through the available host backend. YOLO/doCTR require compiled libraries and GGUF artifacts; Tesseract requires a binary plus traineddata resolved from the vendored bundle or PATH.
 
-## Support
+## Privacy
 
-For issues and feature requests, please use the GitHub issue tracker.
+- Camera access requires OS-level permissions.
+- No frames are written to disk by default.
+- All inference runs locally unless a remote IMAGE_DESCRIPTION provider is registered.
+- Consider access implications before enabling in shared or sensitive environments.

package/dist/action.d.ts

@@ -0,0 +1,3 @@
+import { type Action } from "@elizaos/core";
+export declare const visionAction: Action;
+//# sourceMappingURL=action.d.ts.map

package/dist/action.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"action.d.ts","sourceRoot":"","sources":["../src/action.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,MAAM,EAYZ,MAAM,eAAe,CAAC;AA29CvB,eAAO,MAAM,YAAY,EAAE,MAmP1B,CAAC"}

package/dist/audio-capture-stream.d.ts

@@ -0,0 +1,42 @@
+import { EventEmitter } from "node:events";
+import { type IAgentRuntime } from "@elizaos/core";
+export interface StreamingAudioConfig {
+    enabled: boolean;
+    device?: string;
+    sampleRate?: number;
+    channels?: number;
+    vadThreshold?: number;
+    silenceTimeout?: number;
+    responseDelay?: number;
+    chunkSize?: number;
+}
+export declare class StreamingAudioCaptureService extends EventEmitter {
+    private runtime;
+    private config;
+    private captureProcess;
+    private isCapturing;
+    private audioBuffer;
+    private isSpeaking;
+    private silenceTimer;
+    private transcriptionInProgress;
+    private currentTranscription;
+    private responseTimer;
+    constructor(runtime: IAgentRuntime, config: StreamingAudioConfig);
+    initialize(): Promise<void>;
+    private startContinuousCapture;
+    private processAudioChunk;
+    private calculateEnergy;
+    private startStreamingTranscription;
+    private endSpeech;
+    private processFinalTranscription;
+    private getRecentAudioData;
+    private transcribeAudio;
+    private rawToWav;
+    private generateResponse;
+    private createAudioMemory;
+    stop(): Promise<void>;
+    isActive(): boolean;
+    getCurrentTranscription(): string;
+    isSpeechActive(): boolean;
+}
+//# sourceMappingURL=audio-capture-stream.d.ts.map

package/dist/audio-capture-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio-capture-stream.d.ts","sourceRoot":"","sources":["../src/audio-capture-stream.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EACL,KAAK,aAAa,EAInB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAQD,qBAAa,4BAA6B,SAAQ,YAAY;IAC5D,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAAoB;IACvC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,uBAAuB,CAAS;IACxC,OAAO,CAAC,oBAAoB,CAAM;IAClC,OAAO,CAAC,aAAa,CAA+B;gBAExC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,oBAAoB;IAc1D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAmBnB,sBAAsB;IA2FpC,OAAO,CAAC,iBAAiB;IAuDzB,OAAO,CAAC,eAAe;YAcT,2BAA2B;IAqCzC,OAAO,CAAC,SAAS;YAcH,yBAAyB;IA8BvC,OAAO,CAAC,kBAAkB;YA2BZ,eAAe;IAyB7B,OAAO,CAAC,QAAQ;YAkCF,gBAAgB;YAchB,iBAAiB;IAwBzB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAyB3B,QAAQ,IAAI,OAAO;IAInB,uBAAuB,IAAI,MAAM;IAIjC,cAAc,IAAI,OAAO;CAG1B"}

package/dist/audio-capture.d.ts

@@ -0,0 +1,25 @@
+import { type IAgentRuntime } from "@elizaos/core";
+export interface AudioConfig {
+    enabled: boolean;
+    transcriptionInterval: number;
+    device?: string;
+    sampleRate?: number;
+    channels?: number;
+}
+export declare class AudioCaptureService {
+    private runtime;
+    private config;
+    private isRecording;
+    private recordingInterval;
+    constructor(runtime: IAgentRuntime, config: AudioConfig);
+    initialize(): Promise<void>;
+    private checkAudioTools;
+    private startTranscriptionLoop;
+    recordAndTranscribe(): Promise<string | null>;
+    private recordAudio;
+    private createAudioMemory;
+    listAudioDevices(): Promise<string[]>;
+    isActive(): boolean;
+    stop(): Promise<void>;
+}
+//# sourceMappingURL=audio-capture.d.ts.map

package/dist/audio-capture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio-capture.d.ts","sourceRoot":"","sources":["../src/audio-capture.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,aAAa,EAInB,MAAM,eAAe,CAAC;AAIvB,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,iBAAiB,CAA+B;gBAE5C,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,WAAW;IASjD,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAgCnB,eAAe;IAgD7B,OAAO,CAAC,sBAAsB;IAexB,mBAAmB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;YA4DrC,WAAW;YAqCX,iBAAiB;IAuBzB,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IA+D3C,QAAQ,IAAI,OAAO;IAIb,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAe5B"}

package/dist/computeruse-ocr-bridge.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Bridge plugin-vision's hierarchical OCR into plugin-computeruse's
+ * `CoordOcrProvider` registry seam.
+ *
+ * plugin-vision owns the OCR implementations; plugin-computeruse's
+ * scene-builder + GET_SCREEN want coordinate-aware OCR but must NOT take a
+ * hard dependency on plugin-vision (that would create a cycle and force the
+ * vision OCR stack onto every computeruse consumer). So plugin-vision
+ * registers a bridge into computeruse's seam at boot via a best-effort dynamic
+ * import (see `index.ts`).
+ *
+ * The two interfaces are structurally identical — vision's
+ * `OcrWithCoordsService.describe(OcrWithCoordsInput) -> OcrWithCoordsResult`
+ * and computeruse's `CoordOcrProvider.describe(CoordOcrInput) -> CoordOcrResult`
+ * share field shapes (displayId/sourceX/sourceY/pngBytes in; blocks with
+ * bbox+words+semantic_position out) — so the bridge is a thin pass-through.
+ * The types live in different packages, so we describe computeruse's side
+ * structurally here rather than importing it (keeps the no-hard-dep rule).
+ *
+ * Pure + injectable so the wiring is unit-testable without a real
+ * plugin-computeruse present.
+ */
+import { type OcrWithCoordsResult, type OcrWithCoordsService } from "./ocr-with-coords.js";
+/** Structural shape of `@elizaos/plugin-computeruse`'s `CoordOcrInput`. */
+export interface CoordOcrInputLike {
+    readonly displayId: string;
+    readonly sourceX: number;
+    readonly sourceY: number;
+    readonly pngBytes: Uint8Array;
+}
+/** Structural shape of `@elizaos/plugin-computeruse`'s `CoordOcrProvider`. */
+export interface CoordOcrProviderLike {
+    readonly name: string;
+    describe(input: CoordOcrInputLike): Promise<OcrWithCoordsResult>;
+}
+export type RegisterCoordOcrProvider = (provider: CoordOcrProviderLike | null) => void;
+export declare const VISION_COORD_OCR_BRIDGE_NAME = "vision-coord-ocr-bridge";
+/**
+ * Build a `CoordOcrProvider`-shaped bridge that delegates to whatever vision
+ * `OcrWithCoordsService` is currently registered. Resolving the service lazily
+ * (per call) means a later `registerOcrWithCoordsService()` (e.g. swapping in a
+ * native Windows.Media.Ocr / Apple Vision provider) is picked up automatically.
+ */
+export declare function buildVisionCoordOcrBridge(resolve?: () => OcrWithCoordsService | null): CoordOcrProviderLike;
+/**
+ * Register the vision OCR bridge into computeruse's CoordOcrProvider seam.
+ * Idempotent (the seam is last-call-wins). Returns true once registered.
+ */
+export declare function wireComputerUseOcrBridge(register: RegisterCoordOcrProvider, resolve?: () => OcrWithCoordsService | null): boolean;
+//# sourceMappingURL=computeruse-ocr-bridge.d.ts.map

package/dist/computeruse-ocr-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"computeruse-ocr-bridge.d.ts","sourceRoot":"","sources":["../src/computeruse-ocr-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAEL,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EAC1B,MAAM,sBAAsB,CAAC;AAE9B,2EAA2E;AAC3E,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;CAC/B;AAED,8EAA8E;AAC9E,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CAClE;AAED,MAAM,MAAM,wBAAwB,GAAG,CACrC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,KAClC,IAAI,CAAC;AAEV,eAAO,MAAM,4BAA4B,4BAA4B,CAAC;AAEtE;;;;;GAKG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,GAAE,MAAM,oBAAoB,GAAG,IAA8B,GACnE,oBAAoB,CAetB;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,wBAAwB,EAClC,OAAO,CAAC,EAAE,MAAM,oBAAoB,GAAG,IAAI,GAC1C,OAAO,CAGT"}

package/dist/config.d.ts

@@ -0,0 +1,68 @@
+import { z } from "zod";
+import type { VisionConfig } from "./types";
+export declare const defaultVisionConfig: VisionConfig;
+export declare const VisionConfigSchema: z.ZodObject<{
+    cameraName: z.ZodOptional<z.ZodString>;
+    enableCamera: z.ZodDefault<z.ZodBoolean>;
+    pixelChangeThreshold: z.ZodDefault<z.ZodNumber>;
+    updateInterval: z.ZodDefault<z.ZodNumber>;
+    enableObjectDetection: z.ZodDefault<z.ZodBoolean>;
+    objectConfidenceThreshold: z.ZodDefault<z.ZodNumber>;
+    enablePoseDetection: z.ZodDefault<z.ZodBoolean>;
+    poseConfidenceThreshold: z.ZodDefault<z.ZodNumber>;
+    tfUpdateInterval: z.ZodDefault<z.ZodNumber>;
+    vlmUpdateInterval: z.ZodDefault<z.ZodNumber>;
+    tfChangeThreshold: z.ZodDefault<z.ZodNumber>;
+    vlmChangeThreshold: z.ZodDefault<z.ZodNumber>;
+    visionMode: z.ZodDefault<z.ZodEnum<{
+        OFF: "OFF";
+        CAMERA: "CAMERA";
+        SCREEN: "SCREEN";
+        BOTH: "BOTH";
+    }>>;
+    screenCaptureInterval: z.ZodDefault<z.ZodNumber>;
+    tileSize: z.ZodDefault<z.ZodNumber>;
+    tileProcessingOrder: z.ZodDefault<z.ZodEnum<{
+        sequential: "sequential";
+        priority: "priority";
+        random: "random";
+    }>>;
+    maxConcurrentTiles: z.ZodDefault<z.ZodNumber>;
+    ocrEnabled: z.ZodDefault<z.ZodBoolean>;
+    ocrLanguage: z.ZodDefault<z.ZodString>;
+    ocrConfidenceThreshold: z.ZodDefault<z.ZodNumber>;
+    enableFaceRecognition: z.ZodDefault<z.ZodBoolean>;
+    faceMatchThreshold: z.ZodDefault<z.ZodNumber>;
+    maxFaceProfiles: z.ZodDefault<z.ZodNumber>;
+    entityTimeout: z.ZodDefault<z.ZodNumber>;
+    maxTrackedEntities: z.ZodDefault<z.ZodNumber>;
+    enableGPUAcceleration: z.ZodDefault<z.ZodBoolean>;
+    maxMemoryUsageMB: z.ZodDefault<z.ZodNumber>;
+    debugMode: z.ZodDefault<z.ZodBoolean>;
+    logLevel: z.ZodDefault<z.ZodEnum<{
+        info: "info";
+        error: "error";
+        warn: "warn";
+        debug: "debug";
+    }>>;
+}, z.core.$strip>;
+export type VisionConfigInput = z.input<typeof VisionConfigSchema>;
+export type VisionConfigOutput = z.output<typeof VisionConfigSchema>;
+interface RuntimeWithSettings {
+    getSetting(key: string): string | undefined;
+}
+export declare class ConfigurationManager {
+    private config;
+    private runtime;
+    constructor(runtime: RuntimeWithSettings);
+    private loadConfiguration;
+    private getSetting;
+    private getBooleanSetting;
+    private getNumberSetting;
+    private getEnumSetting;
+    get(): VisionConfigOutput;
+    update(updates: Partial<VisionConfigInput>): void;
+    static getPreset(name: string): Partial<VisionConfigInput>;
+}
+export {};
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,YAAY,EAAc,MAAM,SAAS,CAAC;AAExD,eAAO,MAAM,mBAAmB,EAAE,YAcjC,CAAC;AAEF,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgC7B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AACnE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAErE,UAAU,mBAAmB;IAC3B,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CAC7C;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,OAAO,CAAsB;gBAEzB,OAAO,EAAE,mBAAmB;IAKxC,OAAO,CAAC,iBAAiB;IAiGzB,OAAO,CAAC,UAAU;IAOlB,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,cAAc;IAetB,GAAG,IAAI,kBAAkB;IAIzB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,IAAI;IAejD,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;CAmC3D"}