@elizaos/plugin-vision 1.2.1 → 2.0.0-alpha.1

package/dist/ocr-service.js

@@ -1,216 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
-    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.OCRService = void 0;
-var core_1 = require("@elizaos/core");
-var ocr_service_real_1 = require("./ocr-service-real");
-var OCRService = /** @class */ (function () {
-    function OCRService() {
-        this.realOCR = null;
-        this.initialized = false;
-        this.useFallback = false;
-    }
-    OCRService.prototype.initialize = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var error_1;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (this.initialized) {
-                            return [2 /*return*/];
-                        }
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 3, , 4]);
-                        core_1.logger.info('[OCR] Initializing OCR service...');
-                        // Try to use real OCR first
-                        this.realOCR = new ocr_service_real_1.RealOCRService();
-                        return [4 /*yield*/, this.realOCR.initialize()];
-                    case 2:
-                        _a.sent();
-                        this.initialized = true;
-                        this.useFallback = false;
-                        core_1.logger.info('[OCR] Real OCR service initialized successfully');
-                        return [3 /*break*/, 4];
-                    case 3:
-                        error_1 = _a.sent();
-                        core_1.logger.error('[OCR] Failed to initialize real OCR:', error_1);
-                        // Fallback to basic OCR simulation
-                        core_1.logger.warn('[OCR] Using fallback OCR implementation');
-                        this.useFallback = true;
-                        this.initialized = true;
-                        return [3 /*break*/, 4];
-                    case 4: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    OCRService.prototype.extractText = function (imageBuffer) {
-        return __awaiter(this, void 0, void 0, function () {
-            var error_2;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (!!this.initialized) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.initialize()];
-                    case 1:
-                        _a.sent();
-                        _a.label = 2;
-                    case 2:
-                        if (!(this.realOCR && !this.useFallback)) return [3 /*break*/, 6];
-                        _a.label = 3;
-                    case 3:
-                        _a.trys.push([3, 5, , 6]);
-                        return [4 /*yield*/, this.realOCR.extractText(imageBuffer)];
-                    case 4: return [2 /*return*/, _a.sent()];
-                    case 5:
-                        error_2 = _a.sent();
-                        core_1.logger.error('[OCR] Real OCR failed, falling back:', error_2);
-                        this.useFallback = true;
-                        return [3 /*break*/, 6];
-                    case 6: 
-                    // Fallback implementation
-                    return [2 /*return*/, this.fallbackOCR(imageBuffer)];
-                }
-            });
-        });
-    };
-    OCRService.prototype.extractFromTile = function (tile) {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                if (!tile.data) {
-                    return [2 /*return*/, {
-                            text: '',
-                            blocks: [],
-                            fullText: '',
-                        }];
-                }
-                return [2 /*return*/, this.extractText(tile.data)];
-            });
-        });
-    };
-    OCRService.prototype.extractFromImage = function (imageBuffer) {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                return [2 /*return*/, this.extractText(imageBuffer)];
-            });
-        });
-    };
-    OCRService.prototype.fallbackOCR = function (_imageBuffer) {
-        return __awaiter(this, void 0, void 0, function () {
-            var blocks, mockTexts, _i, mockTexts_1, mock, fullText;
-            return __generator(this, function (_a) {
-                // Fallback implementation for when Tesseract is not available
-                core_1.logger.debug('[OCR] Using fallback OCR implementation');
-                blocks = [];
-                mockTexts = [
-                    { text: 'File Edit View Window Help', x: 10, y: 5, width: 300, height: 20 },
-                    { text: 'Welcome to the application', x: 100, y: 100, width: 400, height: 40 },
-                    { text: 'Click here to continue', x: 200, y: 300, width: 200, height: 30 },
-                ];
-                for (_i = 0, mockTexts_1 = mockTexts; _i < mockTexts_1.length; _i++) {
-                    mock = mockTexts_1[_i];
-                    blocks.push({
-                        text: mock.text,
-                        bbox: { x: mock.x, y: mock.y, width: mock.width, height: mock.height },
-                        confidence: 0.85 + Math.random() * 0.1,
-                    });
-                }
-                fullText = blocks.map(function (b) { return b.text; }).join('\n');
-                return [2 /*return*/, {
-                        text: fullText,
-                        blocks: blocks,
-                        fullText: fullText,
-                    }];
-            });
-        });
-    };
-    OCRService.prototype.extractStructuredData = function (imageBuffer) {
-        return __awaiter(this, void 0, void 0, function () {
-            var error_3;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (!(this.realOCR && !this.useFallback)) return [3 /*break*/, 4];
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 3, , 4]);
-                        return [4 /*yield*/, this.realOCR.extractStructuredData(imageBuffer)];
-                    case 2: return [2 /*return*/, _a.sent()];
-                    case 3:
-                        error_3 = _a.sent();
-                        core_1.logger.error('[OCR] Structured data extraction failed:', error_3);
-                        return [3 /*break*/, 4];
-                    case 4: 
-                    // Fallback: return empty structured data
-                    return [2 /*return*/, {
-                            tables: [],
-                            forms: [],
-                            lists: [],
-                        }];
-                }
-            });
-        });
-    };
-    OCRService.prototype.isInitialized = function () {
-        return this.initialized;
-    };
-    OCRService.prototype.dispose = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (!this.realOCR) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.realOCR.dispose()];
-                    case 1:
-                        _a.sent();
-                        this.realOCR = null;
-                        _a.label = 2;
-                    case 2:
-                        this.initialized = false;
-                        this.useFallback = false;
-                        core_1.logger.info('[OCR] Service disposed');
-                        return [2 /*return*/];
-                }
-            });
-        });
-    };
-    return OCRService;
-}());
-exports.OCRService = OCRService;
-//# sourceMappingURL=ocr-service.js.map

package/dist/ocr-service.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ocr-service.js","sourceRoot":"","sources":["../src/ocr-service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sCAAuC;AAEvC,uDAAoD;AAEpD;IAAA;QACU,YAAO,GAA0B,IAAI,CAAC;QACtC,gBAAW,GAAG,KAAK,CAAC;QACpB,gBAAW,GAAG,KAAK,CAAC;IAuI9B,CAAC;IArIO,+BAAU,GAAhB;;;;;;wBACE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,sBAAO;wBACT,CAAC;;;;wBAGC,aAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;wBAEjD,4BAA4B;wBAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,iCAAc,EAAE,CAAC;wBACpC,qBAAM,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAA;;wBAA/B,SAA+B,CAAC;wBAEhC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBACxB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;wBACzB,aAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;;;;wBAE/D,aAAM,CAAC,KAAK,CAAC,sCAAsC,EAAE,OAAK,CAAC,CAAC;wBAE5D,mCAAmC;wBACnC,aAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;wBACvD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBACxB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;;;;;;KAE3B;IAEK,gCAAW,GAAjB,UAAkB,WAAmB;;;;;;6BAC/B,CAAC,IAAI,CAAC,WAAW,EAAjB,wBAAiB;wBACnB,qBAAM,IAAI,CAAC,UAAU,EAAE,EAAA;;wBAAvB,SAAuB,CAAC;;;6BAItB,CAAA,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,CAAA,EAAjC,wBAAiC;;;;wBAE1B,qBAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,EAAA;4BAAlD,sBAAO,SAA2C,EAAC;;;wBAEnD,aAAM,CAAC,KAAK,CAAC,sCAAsC,EAAE,OAAK,CAAC,CAAC;wBAC5D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;;;oBAI5B,0BAA0B;oBAC1B,sBAAO,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,EAAC;;;;KACtC;IAEK,oCAAe,GAArB,UAAsB,IAAgB;;;gBACpC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACf,sBAAO;4BACL,IAAI,EAAE,EAAE;4BACR,MAAM,EAAE,EAAE;4BACV,QAAQ,EAAE,EAAE;yBACb,EAAC;gBACJ,CAAC;gBAED,sBAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAC;;;KACpC;IAEK,qCAAgB,GAAtB,UAAuB,WAAmB;;;gBACxC,sBAAO,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,EAAC;;;KACtC;IAEa,gCAAW,GAAzB,UAA0B,YAAoB;;;;gBAC5C,8DAA8D;gBAC9D,aAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBAElD,MAAM,GASP,EAAE,CAAC;gBAGF,SAAS,GAAG;oBAChB,EAAE,IAAI,EAAE,4BAA4B,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE;oBAC3E,EAAE,IAAI,EAAE,4BAA4B,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE;oBAC9E,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE;iBAC3E,CAAC;gBAEF,WAA4B,EAAT,uBAAS,EAAT,uBAAS,EAAT,IAAS,EAAE,CAAC;oBAApB,IAAI;oBACb,MAAM,CAAC,IAAI,CAAC;wBACV,IAAI,EAAE,IAAI,CAAC,IAAI;wBACf,IAAI,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;wBACtE,UAAU,EAAE,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG;qBACvC,CAAC,CAAC;gBACL,CAAC;gBAEK,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,IAAI,EAAN,CAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEtD,sBAAO;wBACL,IAAI,EAAE,QAAQ;wBACd,MAAM,QAAA;wBACN,QAAQ,UAAA;qBACT,EAAC;;;KACH;IAEK,0CAAqB,GAA3B,UAA4B,WAAmB;;;;;;6BAKzC,CAAA,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,CAAA,EAAjC,wBAAiC;;;;wBAE1B,qBAAM,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,WAAW,CAAC,EAAA;4BAA5D,sBAAO,SAAqD,EAAC;;;wBAE7D,aAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,OAAK,CAAC,CAAC;;;oBAIpE,yCAAyC;oBACzC,sBAAO;4BACL,MAAM,EAAE,EAAE;4BACV,KAAK,EAAE,EAAE;4BACT,KAAK,EAAE,EAAE;yBACV,EAAC;;;;KACH;IAED,kCAAa,GAAb;QACE,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAEK,4BAAO,GAAb;;;;;6BACM,IAAI,CAAC,OAAO,EAAZ,wBAAY;wBACd,qBAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAA;;wBAA5B,SAA4B,CAAC;wBAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;;;wBAEtB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;wBACzB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;wBACzB,aAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;;;;;KACvC;IACH,iBAAC;AAAD,CAAC,AA1ID,IA0IC;AA1IY,gCAAU","sourcesContent":["import { logger } from '@elizaos/core';\nimport type { OCRResult, ScreenTile, BoundingBox } from './types';\nimport { RealOCRService } from './ocr-service-real';\n\nexport class OCRService {\n  private realOCR: RealOCRService | null = null;\n  private initialized = false;\n  private useFallback = false;\n\n  async initialize(): Promise<void> {\n    if (this.initialized) {\n      return;\n    }\n\n    try {\n      logger.info('[OCR] Initializing OCR service...');\n\n      // Try to use real OCR first\n      this.realOCR = new RealOCRService();\n      await this.realOCR.initialize();\n\n      this.initialized = true;\n      this.useFallback = false;\n      logger.info('[OCR] Real OCR service initialized successfully');\n    } catch (error) {\n      logger.error('[OCR] Failed to initialize real OCR:', error);\n\n      // Fallback to basic OCR simulation\n      logger.warn('[OCR] Using fallback OCR implementation');\n      this.useFallback = true;\n      this.initialized = true;\n    }\n  }\n\n  async extractText(imageBuffer: Buffer): Promise<OCRResult> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    // Use real OCR if available\n    if (this.realOCR && !this.useFallback) {\n      try {\n        return await this.realOCR.extractText(imageBuffer);\n      } catch (error) {\n        logger.error('[OCR] Real OCR failed, falling back:', error);\n        this.useFallback = true;\n      }\n    }\n\n    // Fallback implementation\n    return this.fallbackOCR(imageBuffer);\n  }\n\n  async extractFromTile(tile: ScreenTile): Promise<OCRResult> {\n    if (!tile.data) {\n      return {\n        text: '',\n        blocks: [],\n        fullText: '',\n      };\n    }\n\n    return this.extractText(tile.data);\n  }\n\n  async extractFromImage(imageBuffer: Buffer): Promise<OCRResult> {\n    return this.extractText(imageBuffer);\n  }\n\n  private async fallbackOCR(_imageBuffer: Buffer): Promise<OCRResult> {\n    // Fallback implementation for when Tesseract is not available\n    logger.debug('[OCR] Using fallback OCR implementation');\n\n    const blocks: Array<{\n      text: string;\n      bbox: BoundingBox;\n      confidence: number;\n      words?: Array<{\n        text: string;\n        bbox: BoundingBox;\n        confidence: number;\n      }>;\n    }> = [];\n\n    // Simulate finding common UI text\n    const mockTexts = [\n      { text: 'File Edit View Window Help', x: 10, y: 5, width: 300, height: 20 },\n      { text: 'Welcome to the application', x: 100, y: 100, width: 400, height: 40 },\n      { text: 'Click here to continue', x: 200, y: 300, width: 200, height: 30 },\n    ];\n\n    for (const mock of mockTexts) {\n      blocks.push({\n        text: mock.text,\n        bbox: { x: mock.x, y: mock.y, width: mock.width, height: mock.height },\n        confidence: 0.85 + Math.random() * 0.1,\n      });\n    }\n\n    const fullText = blocks.map((b) => b.text).join('\\n');\n\n    return {\n      text: fullText,\n      blocks,\n      fullText,\n    };\n  }\n\n  async extractStructuredData(imageBuffer: Buffer): Promise<{\n    tables?: Array<{ rows: string[][]; bbox: BoundingBox }>;\n    forms?: Array<{ label: string; value: string; bbox: BoundingBox }>;\n    lists?: Array<{ items: string[]; bbox: BoundingBox }>;\n  }> {\n    if (this.realOCR && !this.useFallback) {\n      try {\n        return await this.realOCR.extractStructuredData(imageBuffer);\n      } catch (error) {\n        logger.error('[OCR] Structured data extraction failed:', error);\n      }\n    }\n\n    // Fallback: return empty structured data\n    return {\n      tables: [],\n      forms: [],\n      lists: [],\n    };\n  }\n\n  isInitialized(): boolean {\n    return this.initialized;\n  }\n\n  async dispose(): Promise<void> {\n    if (this.realOCR) {\n      await this.realOCR.dispose();\n      this.realOCR = null;\n    }\n    this.initialized = false;\n    this.useFallback = false;\n    logger.info('[OCR] Service disposed');\n  }\n}\n"]}

package/dist/provider.d.ts

@@ -1,2 +0,0 @@
-import { type Provider } from '@elizaos/core';
-export declare const visionProvider: Provider;

package/dist/provider.js

@@ -1,285 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
-    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
-    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
-        if (ar || !(i in from)) {
-            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
-            ar[i] = from[i];
-        }
-    }
-    return to.concat(ar || Array.prototype.slice.call(from));
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.visionProvider = void 0;
-// Vision provider - provides current visual perception data to the agent
-var core_1 = require("@elizaos/core");
-exports.visionProvider = {
-    name: 'VISION_PERCEPTION',
-    description: 'Provides current visual perception data including scene description, detected objects, people, and entity tracking. This provider is always active and provides real-time visual awareness.',
-    position: 99, // High priority for vision context
-    dynamic: false, // Always included - vision is a constant sense
-    get: function (runtime, message, _state) { return __awaiter(void 0, void 0, void 0, function () {
-        var visionService, sceneDescription, _a, cameraInfo, isActive, visionMode, screenCapture, _worldId, entityTracker, entityData, activeEntities, recentlyLeft, stats, perceptionText, values, data, ageInSeconds, secondsAgo, poses, facings, poseCounts, facingCounts, objectTypes, uniqueObjects, _i, _b, entity, name_1, duration, _c, _d, departed, name_2, timeStr, screenAge, screenSecondsAgo, enhanced, tileAnalysis, uiElements, uniqueElements;
-        var _e;
-        return __generator(this, function (_f) {
-            switch (_f.label) {
-                case 0:
-                    visionService = runtime.getService('VISION');
-                    if (!visionService) {
-                        core_1.logger.warn('[visionProvider] VisionService not found.');
-                        return [2 /*return*/, {
-                                values: {
-                                    visionAvailable: false,
-                                    sceneDescription: 'Vision service is not available.',
-                                    cameraStatus: 'No camera connected',
-                                },
-                                text: (0, core_1.addHeader)('# Visual Perception', 'Vision service is not available.'),
-                                data: { hasVision: false },
-                            }];
-                    }
-                    return [4 /*yield*/, visionService.getEnhancedSceneDescription()];
-                case 1:
-                    _a = (_f.sent());
-                    if (_a) return [3 /*break*/, 3];
-                    return [4 /*yield*/, visionService.getSceneDescription()];
-                case 2:
-                    _a = (_f.sent());
-                    _f.label = 3;
-                case 3:
-                    sceneDescription = _a;
-                    cameraInfo = visionService.getCameraInfo();
-                    isActive = visionService.isActive();
-                    visionMode = visionService.getVisionMode();
-                    return [4 /*yield*/, visionService.getScreenCapture()];
-                case 4:
-                    screenCapture = _f.sent();
-                    _worldId = message.worldId || 'default-world';
-                    entityTracker = visionService.getEntityTracker();
-                    entityData = null;
-                    if (!(sceneDescription && entityTracker)) return [3 /*break*/, 6];
-                    // Update entities with current detections
-                    return [4 /*yield*/, entityTracker.updateEntities(sceneDescription.objects, sceneDescription.people, undefined, runtime)];
-                case 5:
-                    // Update entities with current detections
-                    _f.sent();
-                    activeEntities = entityTracker.getActiveEntities();
-                    recentlyLeft = entityTracker.getRecentlyLeft();
-                    stats = entityTracker.getStatistics();
-                    entityData = {
-                        activeEntities: activeEntities.map(function (e) { return ({
-                            id: e.id,
-                            type: e.entityType,
-                            name: e.attributes.name,
-                            firstSeen: e.firstSeen,
-                            duration: Date.now() - e.firstSeen,
-                            position: e.lastPosition,
-                            attributes: e.attributes,
-                        }); }),
-                        recentlyLeft: recentlyLeft.map(function (_a) {
-                            var entity = _a.entity, leftAt = _a.leftAt;
-                            return ({
-                                id: entity.id,
-                                name: entity.attributes.name,
-                                leftAt: leftAt,
-                                timeAgo: Date.now() - leftAt,
-                            });
-                        }),
-                        statistics: stats,
-                    };
-                    _f.label = 6;
-                case 6:
-                    perceptionText = '';
-                    values = {};
-                    data = {};
-                    if (!isActive) {
-                        perceptionText = "Vision mode: ".concat(visionMode, "\n");
-                        if (visionMode === 'OFF') {
-                            perceptionText += 'Vision is disabled.';
-                        }
-                        else {
-                            perceptionText += 'Vision service is initializing...';
-                        }
-                        values = {
-                            visionAvailable: false,
-                            visionMode: visionMode,
-                            sceneDescription: 'Vision not active',
-                            cameraStatus: cameraInfo
-                                ? "Camera \"".concat(cameraInfo.name, "\" detected but not active")
-                                : 'No camera',
-                        };
-                    }
-                    else {
-                        perceptionText = "Vision mode: ".concat(visionMode, "\n\n");
-                        // Camera vision data
-                        if ((visionMode === 'CAMERA' || visionMode === 'BOTH') && sceneDescription) {
-                            ageInSeconds = (Date.now() - sceneDescription.timestamp) / 1000;
-                            secondsAgo = Math.round(ageInSeconds);
-                            perceptionText += "Camera view (".concat(secondsAgo, "s ago):\n").concat(sceneDescription.description);
-                            if (sceneDescription.people.length > 0) {
-                                perceptionText += "\n\nPeople detected: ".concat(sceneDescription.people.length);
-                                poses = sceneDescription.people.map(function (p) { return p.pose; }).filter(function (p) { return p !== 'unknown'; });
-                                facings = sceneDescription.people
-                                    .map(function (p) { return p.facing; })
-                                    .filter(function (f) { return f !== 'unknown'; });
-                                if (poses.length > 0) {
-                                    poseCounts = poses.reduce(function (acc, pose) {
-                                        acc[pose] = (acc[pose] || 0) + 1;
-                                        return acc;
-                                    }, {});
-                                    perceptionText += "\n  Poses: ".concat(Object.entries(poseCounts)
-                                        .map(function (_a) {
-                                        var pose = _a[0], count = _a[1];
-                                        return "".concat(count, " ").concat(pose);
-                                    })
-                                        .join(', '));
-                                }
-                                if (facings.length > 0) {
-                                    facingCounts = facings.reduce(function (acc, facing) {
-                                        acc[facing] = (acc[facing] || 0) + 1;
-                                        return acc;
-                                    }, {});
-                                    perceptionText += "\n  Facing: ".concat(Object.entries(facingCounts)
-                                        .map(function (_a) {
-                                        var facing = _a[0], count = _a[1];
-                                        return "".concat(count, " facing ").concat(facing);
-                                    })
-                                        .join(', '));
-                                }
-                            }
-                            if (sceneDescription.objects.length > 0) {
-                                objectTypes = sceneDescription.objects.map(function (o) { return o.type; });
-                                uniqueObjects = __spreadArray([], new Set(objectTypes), true);
-                                perceptionText += "\n\nObjects detected: ".concat(uniqueObjects.join(', '));
-                            }
-                            if (sceneDescription.sceneChanged) {
-                                perceptionText += "\n\nScene change: ".concat(sceneDescription.changePercentage.toFixed(1), "% of pixels changed");
-                            }
-                            // Add entity tracking information
-                            if (entityData) {
-                                if (entityData.activeEntities.length > 0) {
-                                    perceptionText += '\n\nCurrently tracking:';
-                                    for (_i = 0, _b = entityData.activeEntities; _i < _b.length; _i++) {
-                                        entity = _b[_i];
-                                        name_1 = entity.name || "Unknown ".concat(entity.type);
-                                        duration = entity.duration < 60000
-                                            ? "".concat(Math.round(entity.duration / 1000), "s")
-                                            : "".concat(Math.round(entity.duration / 60000), "m");
-                                        perceptionText += "\n- ".concat(name_1, " (present for ").concat(duration, ")");
-                                    }
-                                }
-                                if (entityData.recentlyLeft.length > 0) {
-                                    perceptionText += '\n\nRecently left:';
-                                    for (_c = 0, _d = entityData.recentlyLeft; _c < _d.length; _c++) {
-                                        departed = _d[_c];
-                                        name_2 = departed.name || 'Unknown person';
-                                        timeStr = departed.timeAgo < 60000
-                                            ? "".concat(Math.round(departed.timeAgo / 1000), "s ago")
-                                            : "".concat(Math.round(departed.timeAgo / 60000), "m ago");
-                                        perceptionText += "\n- ".concat(name_2, " left ").concat(timeStr);
-                                    }
-                                }
-                            }
-                        }
-                        // Screen vision data
-                        if ((visionMode === 'SCREEN' || visionMode === 'BOTH') && screenCapture) {
-                            screenAge = (Date.now() - screenCapture.timestamp) / 1000;
-                            screenSecondsAgo = Math.round(screenAge);
-                            if (visionMode === 'BOTH') {
-                                perceptionText += '\n\n---\n\n';
-                            }
-                            perceptionText += "Screen capture (".concat(screenSecondsAgo, "s ago):\n");
-                            perceptionText += "Resolution: ".concat(screenCapture.width, "x").concat(screenCapture.height, "\n");
-                            enhanced = sceneDescription;
-                            if (enhanced === null || enhanced === void 0 ? void 0 : enhanced.screenAnalysis) {
-                                if ((_e = enhanced.screenAnalysis.activeTile) === null || _e === void 0 ? void 0 : _e.analysis) {
-                                    tileAnalysis = enhanced.screenAnalysis.activeTile.analysis;
-                                    if (tileAnalysis.summary) {
-                                        perceptionText += "\nActive area: ".concat(tileAnalysis.summary);
-                                    }
-                                    if (tileAnalysis.text) {
-                                        perceptionText += "\n\nVisible text:\n\"".concat(tileAnalysis.text.substring(0, 200)).concat(tileAnalysis.text.length > 200 ? '...' : '', "\"");
-                                    }
-                                    if (tileAnalysis.objects && tileAnalysis.objects.length > 0) {
-                                        uiElements = tileAnalysis.objects.map(function (o) { return o.type; });
-                                        uniqueElements = __spreadArray([], new Set(uiElements), true);
-                                        perceptionText += "\n\nUI elements: ".concat(uniqueElements.join(', '));
-                                    }
-                                }
-                                if (enhanced.screenAnalysis.focusedApp) {
-                                    perceptionText += "\n\nActive application: ".concat(enhanced.screenAnalysis.focusedApp);
-                                }
-                            }
-                        }
-                        values = {
-                            visionAvailable: true,
-                            visionMode: visionMode,
-                            sceneDescription: (sceneDescription === null || sceneDescription === void 0 ? void 0 : sceneDescription.description) || 'Processing...',
-                            cameraStatus: cameraInfo ? "Connected to ".concat(cameraInfo.name) : 'No camera',
-                            cameraId: cameraInfo === null || cameraInfo === void 0 ? void 0 : cameraInfo.id,
-                            peopleCount: (sceneDescription === null || sceneDescription === void 0 ? void 0 : sceneDescription.people.length) || 0,
-                            objectCount: (sceneDescription === null || sceneDescription === void 0 ? void 0 : sceneDescription.objects.length) || 0,
-                            sceneAge: sceneDescription
-                                ? Math.round((Date.now() - sceneDescription.timestamp) / 1000)
-                                : null,
-                            lastChange: (sceneDescription === null || sceneDescription === void 0 ? void 0 : sceneDescription.sceneChanged) ? sceneDescription.changePercentage : 0,
-                            hasScreenCapture: !!screenCapture,
-                            screenResolution: screenCapture ? "".concat(screenCapture.width, "x").concat(screenCapture.height) : null,
-                            // Entity tracking values
-                            activeEntities: (entityData === null || entityData === void 0 ? void 0 : entityData.activeEntities) || [],
-                            recentlyLeft: (entityData === null || entityData === void 0 ? void 0 : entityData.recentlyLeft) || [],
-                            entityStatistics: (entityData === null || entityData === void 0 ? void 0 : entityData.statistics) || null,
-                        };
-                        data = {
-                            objects: (sceneDescription === null || sceneDescription === void 0 ? void 0 : sceneDescription.objects) || [],
-                            people: (sceneDescription === null || sceneDescription === void 0 ? void 0 : sceneDescription.people) || [],
-                            screenCapture: screenCapture || null,
-                            enhancedData: (sceneDescription === null || sceneDescription === void 0 ? void 0 : sceneDescription.screenAnalysis) || null,
-                            trackedEntities: (entityData === null || entityData === void 0 ? void 0 : entityData.activeEntities) || [],
-                            worldState: entityData || null,
-                        };
-                    }
-                    return [2 /*return*/, {
-                            values: values,
-                            text: (0, core_1.addHeader)('# Visual Perception', perceptionText),
-                            data: data,
-                        }];
-            }
-        });
-    }); },
-};
-//# sourceMappingURL=provider.js.map

package/dist/provider.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"provider.js","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yEAAyE;AACzE,sCAOuB;AAIV,QAAA,cAAc,GAAa;IACtC,IAAI,EAAE,mBAAmB;IACzB,WAAW,EACT,6LAA6L;IAC/L,QAAQ,EAAE,EAAE,EAAE,mCAAmC;IACjD,OAAO,EAAE,KAAK,EAAE,+CAA+C;IAE/D,GAAG,EAAE,UAAO,OAAsB,EAAE,OAAe,EAAE,MAAa;;;;;;oBAC1D,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;oBAElE,IAAI,CAAC,aAAa,EAAE,CAAC;wBACnB,aAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;wBACzD,sBAAO;gCACL,MAAM,EAAE;oCACN,eAAe,EAAE,KAAK;oCACtB,gBAAgB,EAAE,kCAAkC;oCACpD,YAAY,EAAE,qBAAqB;iCACpC;gCACD,IAAI,EAAE,IAAA,gBAAS,EAAC,qBAAqB,EAAE,kCAAkC,CAAC;gCAC1E,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;6BAC3B,EAAC;oBACJ,CAAC;oBAIE,qBAAM,aAAa,CAAC,2BAA2B,EAAE,EAAA;;oBAAlD,KAAA,CAAC,SAAiD,CAAC,CAAA;4BAAnD,wBAAmD;oBAClD,qBAAM,aAAa,CAAC,mBAAmB,EAAE,EAAA;;oBAA1C,KAAA,CAAC,SAAyC,CAAC,CAAA;;;oBAFvC,gBAAgB,KAEuB;oBACvC,UAAU,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;oBAC3C,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,CAAC;oBACpC,UAAU,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;oBAC3B,qBAAM,aAAa,CAAC,gBAAgB,EAAE,EAAA;;oBAAtD,aAAa,GAAG,SAAsC;oBAGtD,QAAQ,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;oBAC9C,aAAa,GAAG,aAAa,CAAC,gBAAgB,EAAE,CAAC;oBAEnD,UAAU,GAuBH,IAAI,CAAC;yBAEZ,CAAA,gBAAgB,IAAI,aAAa,CAAA,EAAjC,wBAAiC;oBACnC,0CAA0C;oBAC1C,qBAAM,aAAa,CAAC,cAAc,CAChC,gBAAgB,CAAC,OAAO,EACxB,gBAAgB,CAAC,MAAM,EACvB,SAAS,EACT,OAAO,CACR,EAAA;;oBAND,0CAA0C;oBAC1C,SAKC,CAAC;oBAEI,cAAc,GAAG,aAAa,CAAC,iBAAiB,EAAE,CAAC;oBACnD,YAAY,GAAG,aAAa,CAAC,eAAe,EAAE,CAAC;oBAC/C,KAAK,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;oBAE5C,UAAU,GAAG;wBACX,cAAc,EAAE,cAAc,CAAC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC;4BACzC,EAAE,EAAE,CAAC,CAAC,EAAE;4BACR,IAAI,EAAE,CAAC,CAAC,UAAU;4BAClB,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI;4BACvB,SAAS,EAAE,CAAC,CAAC,SAAS;4BACtB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,SAAS;4BAClC,QAAQ,EAAE,CAAC,CAAC,YAAY;4BACxB,UAAU,EAAE,CAAC,CAAC,UAAU;yBACzB,CAAC,EARwC,CAQxC,CAAC;wBACH,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,UAAC,EAAkB;gCAAhB,MAAM,YAAA,EAAE,MAAM,YAAA;4BAAO,OAAA,CAAC;gCACtD,EAAE,EAAE,MAAM,CAAC,EAAE;gCACb,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;gCAC5B,MAAM,QAAA;gCACN,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM;6BAC7B,CAAC;wBALqD,CAKrD,CAAC;wBACH,UAAU,EAAE,KAAK;qBAClB,CAAC;;;oBAGA,cAAc,GAAG,EAAE,CAAC;oBACpB,MAAM,GAAG,EAAE,CAAC;oBACZ,IAAI,GAAG,EAAE,CAAC;oBAEd,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACd,cAAc,GAAG,uBAAgB,UAAU,OAAI,CAAC;wBAChD,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;4BACzB,cAAc,IAAI,qBAAqB,CAAC;wBAC1C,CAAC;6BAAM,CAAC;4BACN,cAAc,IAAI,mCAAmC,CAAC;wBACxD,CAAC;wBAED,MAAM,GAAG;4BACP,eAAe,EAAE,KAAK;4BACtB,UAAU,YAAA;4BACV,gBAAgB,EAAE,mBAAmB;4BACrC,YAAY,EAAE,UAAU;gCACtB,CAAC,CAAC,mBAAW,UAAU,CAAC,IAAI,+BAA2B;gCACvD,CAAC,CAAC,WAAW;yBAChB,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,cAAc,GAAG,uBAAgB,UAAU,SAAM,CAAC;wBAElD,qBAAqB;wBACrB,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,UAAU,KAAK,MAAM,CAAC,IAAI,gBAAgB,EAAE,CAAC;4BACrE,YAAY,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;4BAChE,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;4BAE5C,cAAc,IAAI,uBAAgB,UAAU,sBAAY,gBAAgB,CAAC,WAAW,CAAE,CAAC;4BAEvF,IAAI,gBAAgB,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gCACvC,cAAc,IAAI,+BAAwB,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAE,CAAC;gCACrE,KAAK,GAAG,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,IAAI,EAAN,CAAM,CAAC,CAAC,MAAM,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,KAAK,SAAS,EAAf,CAAe,CAAC,CAAC;gCAClF,OAAO,GAAG,gBAAgB,CAAC,MAAM;qCACpC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,MAAM,EAAR,CAAQ,CAAC;qCACpB,MAAM,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,KAAK,SAAS,EAAf,CAAe,CAAC,CAAC;gCAElC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACf,UAAU,GAAG,KAAK,CAAC,MAAM,CAC7B,UAAC,GAAG,EAAE,IAAI;wCACR,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;wCACjC,OAAO,GAAG,CAAC;oCACb,CAAC,EACD,EAA4B,CAC7B,CAAC;oCACF,cAAc,IAAI,qBAAc,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;yCACvD,GAAG,CAAC,UAAC,EAAa;4CAAZ,IAAI,QAAA,EAAE,KAAK,QAAA;wCAAM,OAAA,UAAG,KAAK,cAAI,IAAI,CAAE;oCAAlB,CAAkB,CAAC;yCAC1C,IAAI,CAAC,IAAI,CAAC,CAAE,CAAC;gCAClB,CAAC;gCAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACjB,YAAY,GAAG,OAAO,CAAC,MAAM,CACjC,UAAC,GAAG,EAAE,MAAM;wCACV,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;wCACrC,OAAO,GAAG,CAAC;oCACb,CAAC,EACD,EAA4B,CAC7B,CAAC;oCACF,cAAc,IAAI,sBAAe,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;yCAC1D,GAAG,CAAC,UAAC,EAAe;4CAAd,MAAM,QAAA,EAAE,KAAK,QAAA;wCAAM,OAAA,UAAG,KAAK,qBAAW,MAAM,CAAE;oCAA3B,CAA2B,CAAC;yCACrD,IAAI,CAAC,IAAI,CAAC,CAAE,CAAC;gCAClB,CAAC;4BACH,CAAC;4BAED,IAAI,gBAAgB,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gCAClC,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,IAAI,EAAN,CAAM,CAAC,CAAC;gCAC1D,aAAa,qBAAO,IAAI,GAAG,CAAC,WAAW,CAAC,OAAC,CAAC;gCAChD,cAAc,IAAI,gCAAyB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAE,CAAC;4BACxE,CAAC;4BAED,IAAI,gBAAgB,CAAC,YAAY,EAAE,CAAC;gCAClC,cAAc,IAAI,4BAAqB,gBAAgB,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAqB,CAAC;4BAC3G,CAAC;4BAED,kCAAkC;4BAClC,IAAI,UAAU,EAAE,CAAC;gCACf,IAAI,UAAU,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACzC,cAAc,IAAI,yBAAyB,CAAC;oCAC5C,WAA8C,EAAzB,KAAA,UAAU,CAAC,cAAc,EAAzB,cAAyB,EAAzB,IAAyB,EAAE,CAAC;wCAAtC,MAAM;wCACT,SAAO,MAAM,CAAC,IAAI,IAAI,kBAAW,MAAM,CAAC,IAAI,CAAE,CAAC;wCAC/C,QAAQ,GACZ,MAAM,CAAC,QAAQ,GAAG,KAAK;4CACrB,CAAC,CAAC,UAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAG;4CAC1C,CAAC,CAAC,UAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAG,CAAC;wCAChD,cAAc,IAAI,cAAO,MAAI,2BAAiB,QAAQ,MAAG,CAAC;oCAC5D,CAAC;gCACH,CAAC;gCAED,IAAI,UAAU,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACvC,cAAc,IAAI,oBAAoB,CAAC;oCACvC,WAA8C,EAAvB,KAAA,UAAU,CAAC,YAAY,EAAvB,cAAuB,EAAvB,IAAuB,EAAE,CAAC;wCAAtC,QAAQ;wCACX,SAAO,QAAQ,CAAC,IAAI,IAAI,gBAAgB,CAAC;wCACzC,OAAO,GACX,QAAQ,CAAC,OAAO,GAAG,KAAK;4CACtB,CAAC,CAAC,UAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,UAAO;4CAC/C,CAAC,CAAC,UAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,GAAG,KAAK,CAAC,UAAO,CAAC;wCACrD,cAAc,IAAI,cAAO,MAAI,mBAAS,OAAO,CAAE,CAAC;oCAClD,CAAC;gCACH,CAAC;4BACH,CAAC;wBACH,CAAC;wBAED,qBAAqB;wBACrB,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,UAAU,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;4BAClE,SAAS,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;4BAC1D,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;4BAE/C,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;gCAC1B,cAAc,IAAI,aAAa,CAAC;4BAClC,CAAC;4BAED,cAAc,IAAI,0BAAmB,gBAAgB,cAAW,CAAC;4BACjE,cAAc,IAAI,sBAAe,aAAa,CAAC,KAAK,cAAI,aAAa,CAAC,MAAM,OAAI,CAAC;4BAG3E,QAAQ,GAAG,gBAAuB,CAAC;4BACzC,IAAI,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,cAAc,EAAE,CAAC;gCAC7B,IAAI,MAAA,QAAQ,CAAC,cAAc,CAAC,UAAU,0CAAE,QAAQ,EAAE,CAAC;oCAC3C,YAAY,GAAG,QAAQ,CAAC,cAAc,CAAC,UAAU,CAAC,QAAQ,CAAC;oCAEjE,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;wCACzB,cAAc,IAAI,yBAAkB,YAAY,CAAC,OAAO,CAAE,CAAC;oCAC7D,CAAC;oCAED,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;wCACtB,cAAc,IAAI,+BAAuB,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,SAAG,YAAY,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAG,CAAC;oCAChI,CAAC;oCAED,IAAI,YAAY,CAAC,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wCACtD,UAAU,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,UAAC,CAAM,IAAK,OAAA,CAAC,CAAC,IAAI,EAAN,CAAM,CAAC,CAAC;wCAC1D,cAAc,qBAAO,IAAI,GAAG,CAAC,UAAU,CAAC,OAAC,CAAC;wCAChD,cAAc,IAAI,2BAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAE,CAAC;oCACpE,CAAC;gCACH,CAAC;gCAED,IAAI,QAAQ,CAAC,cAAc,CAAC,UAAU,EAAE,CAAC;oCACvC,cAAc,IAAI,kCAA2B,QAAQ,CAAC,cAAc,CAAC,UAAU,CAAE,CAAC;gCACpF,CAAC;4BACH,CAAC;wBACH,CAAC;wBAED,MAAM,GAAG;4BACP,eAAe,EAAE,IAAI;4BACrB,UAAU,YAAA;4BACV,gBAAgB,EAAE,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,WAAW,KAAI,eAAe;4BAClE,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC,uBAAgB,UAAU,CAAC,IAAI,CAAE,CAAC,CAAC,CAAC,WAAW;4BAC1E,QAAQ,EAAE,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,EAAE;4BACxB,WAAW,EAAE,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,MAAM,CAAC,MAAM,KAAI,CAAC;4BACjD,WAAW,EAAE,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,OAAO,CAAC,MAAM,KAAI,CAAC;4BAClD,QAAQ,EAAE,gBAAgB;gCACxB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;gCAC9D,CAAC,CAAC,IAAI;4BACR,UAAU,EAAE,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,YAAY,EAAC,CAAC,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;4BAClF,gBAAgB,EAAE,CAAC,CAAC,aAAa;4BACjC,gBAAgB,EAAE,aAAa,CAAC,CAAC,CAAC,UAAG,aAAa,CAAC,KAAK,cAAI,aAAa,CAAC,MAAM,CAAE,CAAC,CAAC,CAAC,IAAI;4BACzF,yBAAyB;4BACzB,cAAc,EAAE,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,cAAc,KAAI,EAAE;4BAChD,YAAY,EAAE,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,YAAY,KAAI,EAAE;4BAC5C,gBAAgB,EAAE,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,UAAU,KAAI,IAAI;yBACjD,CAAC;wBAEF,IAAI,GAAG;4BACL,OAAO,EAAE,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,OAAO,KAAI,EAAE;4BACxC,MAAM,EAAE,CAAA,gBAAgB,aAAhB,gBAAgB,uBAAhB,gBAAgB,CAAE,MAAM,KAAI,EAAE;4BACtC,aAAa,EAAE,aAAa,IAAI,IAAI;4BACpC,YAAY,EAAE,CAAC,gBAAwB,aAAxB,gBAAgB,uBAAhB,gBAAgB,CAAU,cAAc,KAAI,IAAI;4BAC/D,eAAe,EAAE,CAAA,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,cAAc,KAAI,EAAE;4BACjD,UAAU,EAAE,UAAU,IAAI,IAAI;yBAC/B,CAAC;oBACJ,CAAC;oBAED,sBAAO;4BACL,MAAM,QAAA;4BACN,IAAI,EAAE,IAAA,gBAAS,EAAC,qBAAqB,EAAE,cAAc,CAAC;4BACtD,IAAI,MAAA;yBACL,EAAC;;;SACH;CACF,CAAC","sourcesContent":["// Vision provider - provides current visual perception data to the agent\nimport {\n  type IAgentRuntime,\n  type Memory,\n  type Provider,\n  type State,\n  addHeader,\n  logger,\n} from '@elizaos/core';\nimport { type VisionService } from './service';\nimport { type BoundingBox, type EntityAttributes } from './types';\n\nexport const visionProvider: Provider = {\n  name: 'VISION_PERCEPTION',\n  description:\n    'Provides current visual perception data including scene description, detected objects, people, and entity tracking. This provider is always active and provides real-time visual awareness.',\n  position: 99, // High priority for vision context\n  dynamic: false, // Always included - vision is a constant sense\n\n  get: async (runtime: IAgentRuntime, message: Memory, _state: State) => {\n    const visionService = runtime.getService<VisionService>('VISION');\n\n    if (!visionService) {\n      logger.warn('[visionProvider] VisionService not found.');\n      return {\n        values: {\n          visionAvailable: false,\n          sceneDescription: 'Vision service is not available.',\n          cameraStatus: 'No camera connected',\n        },\n        text: addHeader('# Visual Perception', 'Vision service is not available.'),\n        data: { hasVision: false },\n      };\n    }\n\n    // Get current scene description (enhanced if screen is enabled)\n    const sceneDescription =\n      (await visionService.getEnhancedSceneDescription()) ||\n      (await visionService.getSceneDescription());\n    const cameraInfo = visionService.getCameraInfo();\n    const isActive = visionService.isActive();\n    const visionMode = visionService.getVisionMode();\n    const screenCapture = await visionService.getScreenCapture();\n\n    // Get entity tracking data\n    const _worldId = message.worldId || 'default-world';\n    const entityTracker = visionService.getEntityTracker();\n\n    let entityData: {\n      activeEntities: Array<{\n        id: string;\n        type: 'person' | 'object' | 'pet';\n        name: string | undefined;\n        firstSeen: number;\n        duration: number;\n        position: BoundingBox;\n        attributes: EntityAttributes;\n      }>;\n      recentlyLeft: Array<{\n        id: string;\n        name: string | undefined;\n        leftAt: number;\n        timeAgo: number;\n      }>;\n      statistics: {\n        totalEntities: number;\n        activeEntities: number;\n        recentlyLeft: number;\n        people: number;\n        objects: number;\n      };\n    } | null = null;\n\n    if (sceneDescription && entityTracker) {\n      // Update entities with current detections\n      await entityTracker.updateEntities(\n        sceneDescription.objects,\n        sceneDescription.people,\n        undefined,\n        runtime\n      );\n\n      const activeEntities = entityTracker.getActiveEntities();\n      const recentlyLeft = entityTracker.getRecentlyLeft();\n      const stats = entityTracker.getStatistics();\n\n      entityData = {\n        activeEntities: activeEntities.map((e) => ({\n          id: e.id,\n          type: e.entityType,\n          name: e.attributes.name,\n          firstSeen: e.firstSeen,\n          duration: Date.now() - e.firstSeen,\n          position: e.lastPosition,\n          attributes: e.attributes,\n        })),\n        recentlyLeft: recentlyLeft.map(({ entity, leftAt }) => ({\n          id: entity.id,\n          name: entity.attributes.name,\n          leftAt,\n          timeAgo: Date.now() - leftAt,\n        })),\n        statistics: stats,\n      };\n    }\n\n    let perceptionText = '';\n    let values = {};\n    let data = {};\n\n    if (!isActive) {\n      perceptionText = `Vision mode: ${visionMode}\\n`;\n      if (visionMode === 'OFF') {\n        perceptionText += 'Vision is disabled.';\n      } else {\n        perceptionText += 'Vision service is initializing...';\n      }\n\n      values = {\n        visionAvailable: false,\n        visionMode,\n        sceneDescription: 'Vision not active',\n        cameraStatus: cameraInfo\n          ? `Camera \"${cameraInfo.name}\" detected but not active`\n          : 'No camera',\n      };\n    } else {\n      perceptionText = `Vision mode: ${visionMode}\\n\\n`;\n\n      // Camera vision data\n      if ((visionMode === 'CAMERA' || visionMode === 'BOTH') && sceneDescription) {\n        const ageInSeconds = (Date.now() - sceneDescription.timestamp) / 1000;\n        const secondsAgo = Math.round(ageInSeconds);\n\n        perceptionText += `Camera view (${secondsAgo}s ago):\\n${sceneDescription.description}`;\n\n        if (sceneDescription.people.length > 0) {\n          perceptionText += `\\n\\nPeople detected: ${sceneDescription.people.length}`;\n          const poses = sceneDescription.people.map((p) => p.pose).filter((p) => p !== 'unknown');\n          const facings = sceneDescription.people\n            .map((p) => p.facing)\n            .filter((f) => f !== 'unknown');\n\n          if (poses.length > 0) {\n            const poseCounts = poses.reduce(\n              (acc, pose) => {\n                acc[pose] = (acc[pose] || 0) + 1;\n                return acc;\n              },\n              {} as Record<string, number>\n            );\n            perceptionText += `\\n  Poses: ${Object.entries(poseCounts)\n              .map(([pose, count]) => `${count} ${pose}`)\n              .join(', ')}`;\n          }\n\n          if (facings.length > 0) {\n            const facingCounts = facings.reduce(\n              (acc, facing) => {\n                acc[facing] = (acc[facing] || 0) + 1;\n                return acc;\n              },\n              {} as Record<string, number>\n            );\n            perceptionText += `\\n  Facing: ${Object.entries(facingCounts)\n              .map(([facing, count]) => `${count} facing ${facing}`)\n              .join(', ')}`;\n          }\n        }\n\n        if (sceneDescription.objects.length > 0) {\n          const objectTypes = sceneDescription.objects.map((o) => o.type);\n          const uniqueObjects = [...new Set(objectTypes)];\n          perceptionText += `\\n\\nObjects detected: ${uniqueObjects.join(', ')}`;\n        }\n\n        if (sceneDescription.sceneChanged) {\n          perceptionText += `\\n\\nScene change: ${sceneDescription.changePercentage.toFixed(1)}% of pixels changed`;\n        }\n\n        // Add entity tracking information\n        if (entityData) {\n          if (entityData.activeEntities.length > 0) {\n            perceptionText += '\\n\\nCurrently tracking:';\n            for (const entity of entityData.activeEntities) {\n              const name = entity.name || `Unknown ${entity.type}`;\n              const duration =\n                entity.duration < 60000\n                  ? `${Math.round(entity.duration / 1000)}s`\n                  : `${Math.round(entity.duration / 60000)}m`;\n              perceptionText += `\\n- ${name} (present for ${duration})`;\n            }\n          }\n\n          if (entityData.recentlyLeft.length > 0) {\n            perceptionText += '\\n\\nRecently left:';\n            for (const departed of entityData.recentlyLeft) {\n              const name = departed.name || 'Unknown person';\n              const timeStr =\n                departed.timeAgo < 60000\n                  ? `${Math.round(departed.timeAgo / 1000)}s ago`\n                  : `${Math.round(departed.timeAgo / 60000)}m ago`;\n              perceptionText += `\\n- ${name} left ${timeStr}`;\n            }\n          }\n        }\n      }\n\n      // Screen vision data\n      if ((visionMode === 'SCREEN' || visionMode === 'BOTH') && screenCapture) {\n        const screenAge = (Date.now() - screenCapture.timestamp) / 1000;\n        const screenSecondsAgo = Math.round(screenAge);\n\n        if (visionMode === 'BOTH') {\n          perceptionText += '\\n\\n---\\n\\n';\n        }\n\n        perceptionText += `Screen capture (${screenSecondsAgo}s ago):\\n`;\n        perceptionText += `Resolution: ${screenCapture.width}x${screenCapture.height}\\n`;\n\n        // Enhanced scene data if available\n        const enhanced = sceneDescription as any;\n        if (enhanced?.screenAnalysis) {\n          if (enhanced.screenAnalysis.activeTile?.analysis) {\n            const tileAnalysis = enhanced.screenAnalysis.activeTile.analysis;\n\n            if (tileAnalysis.summary) {\n              perceptionText += `\\nActive area: ${tileAnalysis.summary}`;\n            }\n\n            if (tileAnalysis.text) {\n              perceptionText += `\\n\\nVisible text:\\n\"${tileAnalysis.text.substring(0, 200)}${tileAnalysis.text.length > 200 ? '...' : ''}\"`;\n            }\n\n            if (tileAnalysis.objects && tileAnalysis.objects.length > 0) {\n              const uiElements = tileAnalysis.objects.map((o: any) => o.type);\n              const uniqueElements = [...new Set(uiElements)];\n              perceptionText += `\\n\\nUI elements: ${uniqueElements.join(', ')}`;\n            }\n          }\n\n          if (enhanced.screenAnalysis.focusedApp) {\n            perceptionText += `\\n\\nActive application: ${enhanced.screenAnalysis.focusedApp}`;\n          }\n        }\n      }\n\n      values = {\n        visionAvailable: true,\n        visionMode,\n        sceneDescription: sceneDescription?.description || 'Processing...',\n        cameraStatus: cameraInfo ? `Connected to ${cameraInfo.name}` : 'No camera',\n        cameraId: cameraInfo?.id,\n        peopleCount: sceneDescription?.people.length || 0,\n        objectCount: sceneDescription?.objects.length || 0,\n        sceneAge: sceneDescription\n          ? Math.round((Date.now() - sceneDescription.timestamp) / 1000)\n          : null,\n        lastChange: sceneDescription?.sceneChanged ? sceneDescription.changePercentage : 0,\n        hasScreenCapture: !!screenCapture,\n        screenResolution: screenCapture ? `${screenCapture.width}x${screenCapture.height}` : null,\n        // Entity tracking values\n        activeEntities: entityData?.activeEntities || [],\n        recentlyLeft: entityData?.recentlyLeft || [],\n        entityStatistics: entityData?.statistics || null,\n      };\n\n      data = {\n        objects: sceneDescription?.objects || [],\n        people: sceneDescription?.people || [],\n        screenCapture: screenCapture || null,\n        enhancedData: (sceneDescription as any)?.screenAnalysis || null,\n        trackedEntities: entityData?.activeEntities || [],\n        worldState: entityData || null,\n      };\n    }\n\n    return {\n      values,\n      text: addHeader('# Visual Perception', perceptionText),\n      data,\n    };\n  },\n};\n"]}

package/dist/screen-capture.d.ts

@@ -1,16 +0,0 @@
-import type { ScreenCapture, ScreenTile, VisionConfig } from './types';
-export declare class ScreenCaptureService {
-    private config;
-    private activeTileIndex;
-    private lastCapture;
-    constructor(config: VisionConfig);
-    getScreenInfo(): Promise<{
-        width: number;
-        height: number;
-    } | null>;
-    captureScreen(): Promise<ScreenCapture>;
-    private captureScreenToFile;
-    getActiveTile(): ScreenTile | null;
-    getAllTiles(): ScreenTile[];
-    getProcessedTiles(): ScreenTile[];
-}