@elizaos/plugin-vision 1.2.1

package/dist/tests/e2e/screen-vision.js

@@ -0,0 +1,384 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScreenVisionE2ETestSuite = void 0;
+var core_1 = require("@elizaos/core");
+var types_1 = require("../../types");
+var action_1 = require("../../action");
+var ScreenVisionE2ETestSuite = /** @class */ (function () {
+    function ScreenVisionE2ETestSuite() {
+        var _this = this;
+        this.name = 'plugin-vision-screen-e2e';
+        this.description = 'E2E tests for screen vision functionality including Florence-2 and OCR';
+        this.tests = [
+            {
+                name: 'Should initialize screen vision components',
+                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
+                    var visionService, mode, screenInfo;
+                    return __generator(this, function (_a) {
+                        switch (_a.label) {
+                            case 0:
+                                console.log('Testing screen vision initialization...');
+                                visionService = runtime.getService('VISION');
+                                if (!visionService) {
+                                    throw new Error('Vision service not available');
+                                }
+                                // Set vision mode to SCREEN
+                                return [4 /*yield*/, visionService.setVisionMode(types_1.VisionMode.SCREEN)];
+                            case 1:
+                                // Set vision mode to SCREEN
+                                _a.sent();
+                                // Wait for initialization
+                                return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 2000); })];
+                            case 2:
+                                // Wait for initialization
+                                _a.sent();
+                                mode = visionService.getVisionMode();
+                                if (mode !== types_1.VisionMode.SCREEN) {
+                                    throw new Error("Expected vision mode SCREEN but got ".concat(mode));
+                                }
+                                console.log('✓ Screen vision mode activated');
+                                return [4 /*yield*/, visionService.getScreenCapture()];
+                            case 3:
+                                screenInfo = _a.sent();
+                                if (screenInfo) {
+                                    console.log("\u2713 Screen capture available: ".concat(screenInfo.width, "x").concat(screenInfo.height));
+                                }
+                                else {
+                                    console.log('⚠️  Screen capture not yet available (may still be initializing)');
+                                }
+                                return [2 /*return*/];
+                        }
+                    });
+                }); },
+            },
+            {
+                name: 'Should capture and tile screen',
+                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
+                    var visionService, screenCapture, firstTile, tilesWithData;
+                    return __generator(this, function (_a) {
+                        switch (_a.label) {
+                            case 0:
+                                console.log('Testing screen capture and tiling...');
+                                visionService = runtime.getService('VISION');
+                                if (!visionService) {
+                                    throw new Error('Vision service not available');
+                                }
+                                // Ensure screen mode is active
+                                return [4 /*yield*/, visionService.setVisionMode(types_1.VisionMode.SCREEN)];
+                            case 1:
+                                // Ensure screen mode is active
+                                _a.sent();
+                                // Wait for capture
+                                return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 3000); })];
+                            case 2:
+                                // Wait for capture
+                                _a.sent();
+                                return [4 /*yield*/, visionService.getScreenCapture()];
+                            case 3:
+                                screenCapture = _a.sent();
+                                if (!screenCapture) {
+                                    console.warn('⚠️  No screen capture available - screen capture may not be supported in this environment');
+                                    return [2 /*return*/];
+                                }
+                                console.log("\u2713 Screen captured: ".concat(screenCapture.width, "x").concat(screenCapture.height));
+                                console.log("\u2713 Divided into ".concat(screenCapture.tiles.length, " tiles"));
+                                firstTile = screenCapture.tiles[0];
+                                if (!firstTile) {
+                                    throw new Error('No tiles created from screen capture');
+                                }
+                                console.log("  First tile: ".concat(firstTile.id, " at (").concat(firstTile.x, ", ").concat(firstTile.y, ")"));
+                                console.log("  Tile size: ".concat(firstTile.width, "x").concat(firstTile.height));
+                                tilesWithData = screenCapture.tiles.filter(function (t) { return t.data !== undefined; });
+                                console.log("  Tiles with data: ".concat(tilesWithData.length));
+                                if (tilesWithData.length === 0) {
+                                    console.warn('⚠️  No tiles have been processed yet');
+                                }
+                                return [2 /*return*/];
+                        }
+                    });
+                }); },
+            },
+            {
+                name: 'Should analyze screen content with Florence-2 and OCR',
+                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
+                    var visionService, enhancedScene, screenAnalysis, elementTypes, uniqueTypes;
+                    var _a, _b;
+                    return __generator(this, function (_c) {
+                        switch (_c.label) {
+                            case 0:
+                                console.log('Testing screen content analysis...');
+                                visionService = runtime.getService('VISION');
+                                if (!visionService) {
+                                    throw new Error('Vision service not available');
+                                }
+                                // Ensure screen mode is active
+                                return [4 /*yield*/, visionService.setVisionMode(types_1.VisionMode.SCREEN)];
+                            case 1:
+                                // Ensure screen mode is active
+                                _c.sent();
+                                // Wait for analysis
+                                return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 5000); })];
+                            case 2:
+                                // Wait for analysis
+                                _c.sent();
+                                return [4 /*yield*/, visionService.getEnhancedSceneDescription()];
+                            case 3:
+                                enhancedScene = _c.sent();
+                                if (!enhancedScene || !enhancedScene.screenAnalysis) {
+                                    console.warn('⚠️  No enhanced scene analysis available yet');
+                                    return [2 /*return*/];
+                                }
+                                console.log('✓ Enhanced scene analysis available');
+                                screenAnalysis = enhancedScene.screenAnalysis;
+                                // Check active tile analysis
+                                if (screenAnalysis.activeTile) {
+                                    console.log('✓ Active tile analyzed');
+                                    if (screenAnalysis.activeTile.florence2) {
+                                        console.log("  Florence-2 caption: ".concat(screenAnalysis.activeTile.florence2.caption));
+                                        console.log("  Objects detected: ".concat(((_a = screenAnalysis.activeTile.florence2.objects) === null || _a === void 0 ? void 0 : _a.length) || 0));
+                                        console.log("  Tags: ".concat(((_b = screenAnalysis.activeTile.florence2.tags) === null || _b === void 0 ? void 0 : _b.join(', ')) || 'none'));
+                                    }
+                                    if (screenAnalysis.activeTile.ocr) {
+                                        console.log("  OCR text blocks: ".concat(screenAnalysis.activeTile.ocr.blocks.length));
+                                        console.log("  OCR preview: \"".concat(screenAnalysis.activeTile.ocr.fullText.substring(0, 50), "...\""));
+                                    }
+                                }
+                                // Check full screen OCR
+                                if (screenAnalysis.fullScreenOCR) {
+                                    console.log("\u2713 Full screen OCR: ".concat(screenAnalysis.fullScreenOCR.length, " characters"));
+                                }
+                                // Check UI elements
+                                if (screenAnalysis.uiElements && screenAnalysis.uiElements.length > 0) {
+                                    console.log("\u2713 UI elements detected: ".concat(screenAnalysis.uiElements.length));
+                                    elementTypes = screenAnalysis.uiElements.map(function (e) { return e.type; });
+                                    uniqueTypes = __spreadArray([], new Set(elementTypes), true);
+                                    console.log("  Types: ".concat(uniqueTypes.join(', ')));
+                                }
+                                return [2 /*return*/];
+                        }
+                    });
+                }); },
+            },
+            {
+                name: 'Should switch between vision modes',
+                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
+                    var visionService, modes, _i, modes_1, mode, currentMode, message, callbackCalled, finalMode;
+                    var _this = this;
+                    return __generator(this, function (_a) {
+                        switch (_a.label) {
+                            case 0:
+                                console.log('Testing vision mode switching...');
+                                visionService = runtime.getService('VISION');
+                                if (!visionService) {
+                                    throw new Error('Vision service not available');
+                                }
+                                modes = [types_1.VisionMode.CAMERA, types_1.VisionMode.SCREEN, types_1.VisionMode.BOTH, types_1.VisionMode.OFF];
+                                _i = 0, modes_1 = modes;
+                                _a.label = 1;
+                            case 1:
+                                if (!(_i < modes_1.length)) return [3 /*break*/, 5];
+                                mode = modes_1[_i];
+                                console.log("  Switching to ".concat(mode, " mode..."));
+                                return [4 /*yield*/, visionService.setVisionMode(mode)];
+                            case 2:
+                                _a.sent();
+                                // Wait for mode switch
+                                return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 1000); })];
+                            case 3:
+                                // Wait for mode switch
+                                _a.sent();
+                                currentMode = visionService.getVisionMode();
+                                if (currentMode !== mode) {
+                                    throw new Error("Failed to switch to ".concat(mode, " mode, current mode is ").concat(currentMode));
+                                }
+                                console.log("  \u2713 Successfully switched to ".concat(mode, " mode"));
+                                _a.label = 4;
+                            case 4:
+                                _i++;
+                                return [3 /*break*/, 1];
+                            case 5:
+                                // Test with action
+                                console.log('  Testing SET_VISION_MODE action...');
+                                message = {
+                                    id: (0, core_1.createUniqueUuid)(runtime, 'test-msg'),
+                                    entityId: runtime.agentId,
+                                    content: { text: 'set vision mode to both' },
+                                    agentId: runtime.agentId,
+                                    roomId: (0, core_1.createUniqueUuid)(runtime, 'test-room'),
+                                    createdAt: Date.now(),
+                                };
+                                callbackCalled = false;
+                                return [4 /*yield*/, action_1.setVisionModeAction.handler(runtime, message, { values: {}, data: {}, text: '' }, {}, function (response) { return __awaiter(_this, void 0, void 0, function () {
+                                        return __generator(this, function (_a) {
+                                            callbackCalled = true;
+                                            console.log("  Action response: ".concat(response.text));
+                                            return [2 /*return*/, []];
+                                        });
+                                    }); })];
+                            case 6:
+                                _a.sent();
+                                if (!callbackCalled) {
+                                    throw new Error('SET_VISION_MODE action did not call callback');
+                                }
+                                finalMode = visionService.getVisionMode();
+                                if (finalMode !== types_1.VisionMode.BOTH) {
+                                    throw new Error("SET_VISION_MODE action failed, mode is ".concat(finalMode));
+                                }
+                                console.log('✓ Vision mode switching works correctly');
+                                return [2 /*return*/];
+                        }
+                    });
+                }); },
+            },
+            {
+                name: 'Should provide combined vision data in BOTH mode',
+                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
+                    var visionService, enhancedScene, hasCamera, hasScreen, state;
+                    return __generator(this, function (_a) {
+                        switch (_a.label) {
+                            case 0:
+                                console.log('Testing combined camera and screen vision...');
+                                visionService = runtime.getService('VISION');
+                                if (!visionService) {
+                                    throw new Error('Vision service not available');
+                                }
+                                // Set to BOTH mode
+                                return [4 /*yield*/, visionService.setVisionMode(types_1.VisionMode.BOTH)];
+                            case 1:
+                                // Set to BOTH mode
+                                _a.sent();
+                                // Wait for both systems to initialize
+                                return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 5000); })];
+                            case 2:
+                                // Wait for both systems to initialize
+                                _a.sent();
+                                return [4 /*yield*/, visionService.getEnhancedSceneDescription()];
+                            case 3:
+                                enhancedScene = _a.sent();
+                                hasCamera = visionService.getCameraInfo() !== null;
+                                return [4 /*yield*/, visionService.getScreenCapture()];
+                            case 4:
+                                hasScreen = (_a.sent()) !== null;
+                                console.log("  Camera available: ".concat(hasCamera));
+                                console.log("  Screen capture available: ".concat(hasScreen));
+                                if (!hasCamera && !hasScreen) {
+                                    console.warn('⚠️  Neither camera nor screen capture available in this environment');
+                                    return [2 /*return*/];
+                                }
+                                if (enhancedScene) {
+                                    // Check for camera data
+                                    if (hasCamera && enhancedScene.description) {
+                                        console.log('✓ Camera data present in combined mode');
+                                        console.log("  Scene: ".concat(enhancedScene.description.substring(0, 50), "..."));
+                                    }
+                                    // Check for screen data
+                                    if (hasScreen && enhancedScene.screenAnalysis) {
+                                        console.log('✓ Screen data present in combined mode');
+                                        console.log("  Grid: ".concat(enhancedScene.screenAnalysis.gridSummary));
+                                    }
+                                }
+                                return [4 /*yield*/, runtime.composeState({
+                                        id: (0, core_1.createUniqueUuid)(runtime, 'test-msg'),
+                                        entityId: runtime.agentId,
+                                        content: { text: 'test' },
+                                        agentId: runtime.agentId,
+                                        roomId: (0, core_1.createUniqueUuid)(runtime, 'test-room'),
+                                        createdAt: Date.now(),
+                                    })];
+                            case 5:
+                                state = _a.sent();
+                                if (state.text.includes('Vision mode: BOTH')) {
+                                    console.log('✓ Provider correctly reports BOTH mode');
+                                }
+                                return [2 /*return*/];
+                        }
+                    });
+                }); },
+            },
+            {
+                name: 'Should handle screen capture errors gracefully',
+                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
+                    var visionService, originalConfig, isActive;
+                    return __generator(this, function (_a) {
+                        switch (_a.label) {
+                            case 0:
+                                console.log('Testing error handling...');
+                                visionService = runtime.getService('VISION');
+                                if (!visionService) {
+                                    throw new Error('Vision service not available');
+                                }
+                                originalConfig = visionService.visionConfig;
+                                visionService.visionConfig.screenRegion = {
+                                    x: -100,
+                                    y: -100,
+                                    width: 50000,
+                                    height: 50000,
+                                };
+                                return [4 /*yield*/, visionService.setVisionMode(types_1.VisionMode.SCREEN)];
+                            case 1:
+                                _a.sent();
+                                return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 2000); })];
+                            case 2:
+                                _a.sent();
+                                isActive = visionService.isActive();
+                                console.log("  Service active after invalid config: ".concat(isActive));
+                                // Restore config
+                                visionService.visionConfig = originalConfig;
+                                console.log('✓ Error handling works correctly');
+                                return [2 /*return*/];
+                        }
+                    });
+                }); },
+            },
+        ];
+    }
+    return ScreenVisionE2ETestSuite;
+}());
+exports.ScreenVisionE2ETestSuite = ScreenVisionE2ETestSuite;
+exports.default = new ScreenVisionE2ETestSuite();
+//# sourceMappingURL=screen-vision.js.map

package/dist/tests/e2e/screen-vision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-vision.js","sourceRoot":"","sources":["../../../src/tests/e2e/screen-vision.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,sCAAiD;AAEjD,qCAAyC;AACzC,uCAAmD;AAEnD;IAAA;QAAA,iBAgTC;QA/SC,SAAI,GAAG,0BAA0B,CAAC;QAClC,gBAAW,GAAG,wEAAwE,CAAC;QAEvF,UAAK,GAAG;YACN;gBACE,IAAI,EAAE,4CAA4C;gBAClD,EAAE,EAAE,UAAO,OAAsB;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;gCAEjD,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAClE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;gCAClD,CAAC;gCAED,4BAA4B;gCAC5B,qBAAM,aAAa,CAAC,aAAa,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAA;;gCADpD,4BAA4B;gCAC5B,SAAoD,CAAC;gCAErD,0BAA0B;gCAC1B,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,EAAzB,CAAyB,CAAC,EAAA;;gCADzD,0BAA0B;gCAC1B,SAAyD,CAAC;gCAEpD,IAAI,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;gCAC3C,IAAI,IAAI,KAAK,kBAAU,CAAC,MAAM,EAAE,CAAC;oCAC/B,MAAM,IAAI,KAAK,CAAC,8CAAuC,IAAI,CAAE,CAAC,CAAC;gCACjE,CAAC;gCAED,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;gCAG3B,qBAAM,aAAa,CAAC,gBAAgB,EAAE,EAAA;;gCAAnD,UAAU,GAAG,SAAsC;gCACzD,IAAI,UAAU,EAAE,CAAC;oCACf,OAAO,CAAC,GAAG,CAAC,2CAA+B,UAAU,CAAC,KAAK,cAAI,UAAU,CAAC,MAAM,CAAE,CAAC,CAAC;gCACtF,CAAC;qCAAM,CAAC;oCACN,OAAO,CAAC,GAAG,CAAC,kEAAkE,CAAC,CAAC;gCAClF,CAAC;;;;qBACF;aACF;YAED;gBACE,IAAI,EAAE,gCAAgC;gBACtC,EAAE,EAAE,UAAO,OAAsB;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;gCAE9C,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAClE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;gCAClD,CAAC;gCAED,+BAA+B;gCAC/B,qBAAM,aAAa,CAAC,aAAa,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAA;;gCADpD,+BAA+B;gCAC/B,SAAoD,CAAC;gCAErD,mBAAmB;gCACnB,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,EAAzB,CAAyB,CAAC,EAAA;;gCADzD,mBAAmB;gCACnB,SAAyD,CAAC;gCAEpC,qBAAM,aAAa,CAAC,gBAAgB,EAAE,EAAA;;gCAAtD,aAAa,GAAG,SAAsC;gCAC5D,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,OAAO,CAAC,IAAI,CACV,2FAA2F,CAC5F,CAAC;oCACF,sBAAO;gCACT,CAAC;gCAED,OAAO,CAAC,GAAG,CAAC,kCAAsB,aAAa,CAAC,KAAK,cAAI,aAAa,CAAC,MAAM,CAAE,CAAC,CAAC;gCACjF,OAAO,CAAC,GAAG,CAAC,8BAAkB,aAAa,CAAC,KAAK,CAAC,MAAM,WAAQ,CAAC,CAAC;gCAG5D,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gCACzC,IAAI,CAAC,SAAS,EAAE,CAAC;oCACf,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;gCAC1D,CAAC;gCAED,OAAO,CAAC,GAAG,CAAC,wBAAiB,SAAS,CAAC,EAAE,kBAAQ,SAAS,CAAC,CAAC,eAAK,SAAS,CAAC,CAAC,MAAG,CAAC,CAAC;gCACjF,OAAO,CAAC,GAAG,CAAC,uBAAgB,SAAS,CAAC,KAAK,cAAI,SAAS,CAAC,MAAM,CAAE,CAAC,CAAC;gCAG7D,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,IAAI,KAAK,SAAS,EAApB,CAAoB,CAAC,CAAC;gCAC9E,OAAO,CAAC,GAAG,CAAC,6BAAsB,aAAa,CAAC,MAAM,CAAE,CAAC,CAAC;gCAE1D,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oCAC/B,OAAO,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;gCACvD,CAAC;;;;qBACF;aACF;YAED;gBACE,IAAI,EAAE,uDAAuD;gBAC7D,EAAE,EAAE,UAAO,OAAsB;;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;gCAE5C,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAClE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;gCAClD,CAAC;gCAED,+BAA+B;gCAC/B,qBAAM,aAAa,CAAC,aAAa,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAA;;gCADpD,+BAA+B;gCAC/B,SAAoD,CAAC;gCAErD,oBAAoB;gCACpB,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,EAAzB,CAAyB,CAAC,EAAA;;gCADzD,oBAAoB;gCACpB,SAAyD,CAAC;gCAEpC,qBAAM,aAAa,CAAC,2BAA2B,EAAE,EAAA;;gCAAjE,aAAa,GAAG,SAAiD;gCACvE,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE,CAAC;oCACpD,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;oCAC7D,sBAAO;gCACT,CAAC;gCAED,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;gCAE7C,cAAc,GAAG,aAAa,CAAC,cAAc,CAAC;gCAEpD,6BAA6B;gCAC7B,IAAI,cAAc,CAAC,UAAU,EAAE,CAAC;oCAC9B,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;oCAEtC,IAAI,cAAc,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;wCACxC,OAAO,CAAC,GAAG,CAAC,gCAAyB,cAAc,CAAC,UAAU,CAAC,SAAS,CAAC,OAAO,CAAE,CAAC,CAAC;wCACpF,OAAO,CAAC,GAAG,CACT,8BAAuB,CAAA,MAAA,cAAc,CAAC,UAAU,CAAC,SAAS,CAAC,OAAO,0CAAE,MAAM,KAAI,CAAC,CAAE,CAClF,CAAC;wCACF,OAAO,CAAC,GAAG,CACT,kBAAW,CAAA,MAAA,cAAc,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,0CAAE,IAAI,CAAC,IAAI,CAAC,KAAI,MAAM,CAAE,CAC5E,CAAC;oCACJ,CAAC;oCAED,IAAI,cAAc,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;wCAClC,OAAO,CAAC,GAAG,CAAC,6BAAsB,cAAc,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAE,CAAC,CAAC;wCACjF,OAAO,CAAC,GAAG,CACT,2BAAmB,cAAc,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,UAAM,CACjF,CAAC;oCACJ,CAAC;gCACH,CAAC;gCAED,wBAAwB;gCACxB,IAAI,cAAc,CAAC,aAAa,EAAE,CAAC;oCACjC,OAAO,CAAC,GAAG,CAAC,kCAAsB,cAAc,CAAC,aAAa,CAAC,MAAM,gBAAa,CAAC,CAAC;gCACtF,CAAC;gCAED,oBAAoB;gCACpB,IAAI,cAAc,CAAC,UAAU,IAAI,cAAc,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACtE,OAAO,CAAC,GAAG,CAAC,uCAA2B,cAAc,CAAC,UAAU,CAAC,MAAM,CAAE,CAAC,CAAC;oCACrE,YAAY,GAAG,cAAc,CAAC,UAAU,CAAC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,IAAI,EAAN,CAAM,CAAC,CAAC;oCAC5D,WAAW,qBAAO,IAAI,GAAG,CAAC,YAAY,CAAC,OAAC,CAAC;oCAC/C,OAAO,CAAC,GAAG,CAAC,mBAAY,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAE,CAAC,CAAC;gCACpD,CAAC;;;;qBACF;aACF;YAED;gBACE,IAAI,EAAE,oCAAoC;gBAC1C,EAAE,EAAE,UAAO,OAAsB;;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;gCAE1C,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAClE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;gCAClD,CAAC;gCAGK,KAAK,GAAG,CAAC,kBAAU,CAAC,MAAM,EAAE,kBAAU,CAAC,MAAM,EAAE,kBAAU,CAAC,IAAI,EAAE,kBAAU,CAAC,GAAG,CAAC,CAAC;sCAE9D,EAAL,eAAK;;;qCAAL,CAAA,mBAAK,CAAA;gCAAb,IAAI;gCACb,OAAO,CAAC,GAAG,CAAC,yBAAkB,IAAI,aAAU,CAAC,CAAC;gCAC9C,qBAAM,aAAa,CAAC,aAAa,CAAC,IAAI,CAAC,EAAA;;gCAAvC,SAAuC,CAAC;gCAExC,uBAAuB;gCACvB,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,EAAzB,CAAyB,CAAC,EAAA;;gCADzD,uBAAuB;gCACvB,SAAyD,CAAC;gCAEpD,WAAW,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;gCAClD,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;oCACzB,MAAM,IAAI,KAAK,CAAC,8BAAuB,IAAI,oCAA0B,WAAW,CAAE,CAAC,CAAC;gCACtF,CAAC;gCAED,OAAO,CAAC,GAAG,CAAC,4CAAgC,IAAI,UAAO,CAAC,CAAC;;;gCAZxC,IAAK,CAAA;;;gCAexB,mBAAmB;gCACnB,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;gCAE7C,OAAO,GAAW;oCACtB,EAAE,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,UAAU,CAAC;oCACzC,QAAQ,EAAE,OAAO,CAAC,OAAO;oCACzB,OAAO,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE;oCAC5C,OAAO,EAAE,OAAO,CAAC,OAAO;oCACxB,MAAM,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,WAAW,CAAC;oCAC9C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iCACtB,CAAC;gCAEE,cAAc,GAAG,KAAK,CAAC;gCAC3B,qBAAM,4BAAmB,CAAC,OAAO,CAC/B,OAAO,EACP,OAAO,EACP,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAClC,EAAE,EACF,UAAO,QAAQ;;4CACb,cAAc,GAAG,IAAI,CAAC;4CACtB,OAAO,CAAC,GAAG,CAAC,6BAAsB,QAAQ,CAAC,IAAI,CAAE,CAAC,CAAC;4CACnD,sBAAO,EAAE,EAAC;;yCACX,CACF,EAAA;;gCAVD,SAUC,CAAC;gCAEF,IAAI,CAAC,cAAc,EAAE,CAAC;oCACpB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;gCAClE,CAAC;gCAEK,SAAS,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;gCAChD,IAAI,SAAS,KAAK,kBAAU,CAAC,IAAI,EAAE,CAAC;oCAClC,MAAM,IAAI,KAAK,CAAC,iDAA0C,SAAS,CAAE,CAAC,CAAC;gCACzE,CAAC;gCAED,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;;;;qBACxD;aACF;YAED;gBACE,IAAI,EAAE,kDAAkD;gBACxD,EAAE,EAAE,UAAO,OAAsB;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;gCAEtD,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAClE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;gCAClD,CAAC;gCAED,mBAAmB;gCACnB,qBAAM,aAAa,CAAC,aAAa,CAAC,kBAAU,CAAC,IAAI,CAAC,EAAA;;gCADlD,mBAAmB;gCACnB,SAAkD,CAAC;gCAEnD,sCAAsC;gCACtC,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,EAAzB,CAAyB,CAAC,EAAA;;gCADzD,sCAAsC;gCACtC,SAAyD,CAAC;gCAEpC,qBAAM,aAAa,CAAC,2BAA2B,EAAE,EAAA;;gCAAjE,aAAa,GAAG,SAAiD;gCACjE,SAAS,GAAG,aAAa,CAAC,aAAa,EAAE,KAAK,IAAI,CAAC;gCACtC,qBAAM,aAAa,CAAC,gBAAgB,EAAE,EAAA;;gCAAnD,SAAS,GAAG,CAAC,SAAsC,CAAC,KAAK,IAAI;gCAEnE,OAAO,CAAC,GAAG,CAAC,8BAAuB,SAAS,CAAE,CAAC,CAAC;gCAChD,OAAO,CAAC,GAAG,CAAC,sCAA+B,SAAS,CAAE,CAAC,CAAC;gCAExD,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;oCAC7B,OAAO,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;oCACpF,sBAAO;gCACT,CAAC;gCAED,IAAI,aAAa,EAAE,CAAC;oCAClB,wBAAwB;oCACxB,IAAI,SAAS,IAAI,aAAa,CAAC,WAAW,EAAE,CAAC;wCAC3C,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;wCACtD,OAAO,CAAC,GAAG,CAAC,mBAAY,aAAa,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,QAAK,CAAC,CAAC;oCAC3E,CAAC;oCAED,wBAAwB;oCACxB,IAAI,SAAS,IAAI,aAAa,CAAC,cAAc,EAAE,CAAC;wCAC9C,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;wCACtD,OAAO,CAAC,GAAG,CAAC,kBAAW,aAAa,CAAC,cAAc,CAAC,WAAW,CAAE,CAAC,CAAC;oCACrE,CAAC;gCACH,CAAC;gCAGa,qBAAM,OAAO,CAAC,YAAY,CAAC;wCACvC,EAAE,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,UAAU,CAAC;wCACzC,QAAQ,EAAE,OAAO,CAAC,OAAO;wCACzB,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;wCACzB,OAAO,EAAE,OAAO,CAAC,OAAO;wCACxB,MAAM,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,WAAW,CAAC;wCAC9C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qCACtB,CAAC,EAAA;;gCAPI,KAAK,GAAG,SAOZ;gCAEF,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;oCAC7C,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;gCACxD,CAAC;;;;qBACF;aACF;YAED;gBACE,IAAI,EAAE,gDAAgD;gBACtD,EAAE,EAAE,UAAO,OAAsB;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;gCAEnC,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAClE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;gCAClD,CAAC;gCAGK,cAAc,GAAI,aAAqB,CAAC,YAAY,CAAC;gCAC1D,aAAqB,CAAC,YAAY,CAAC,YAAY,GAAG;oCACjD,CAAC,EAAE,CAAC,GAAG;oCACP,CAAC,EAAE,CAAC,GAAG;oCACP,KAAK,EAAE,KAAK;oCACZ,MAAM,EAAE,KAAK;iCACd,CAAC;gCAEF,qBAAM,aAAa,CAAC,aAAa,CAAC,kBAAU,CAAC,MAAM,CAAC,EAAA;;gCAApD,SAAoD,CAAC;gCACrD,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,EAAzB,CAAyB,CAAC,EAAA;;gCAAzD,SAAyD,CAAC;gCAGpD,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,CAAC;gCAC1C,OAAO,CAAC,GAAG,CAAC,iDAA0C,QAAQ,CAAE,CAAC,CAAC;gCAElE,iBAAiB;gCAChB,aAAqB,CAAC,YAAY,GAAG,cAAc,CAAC;gCAErD,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;;;;qBACjD;aACF;SACF,CAAC;IACJ,CAAC;IAAD,+BAAC;AAAD,CAAC,AAhTD,IAgTC;AAhTY,4DAAwB;AAkTrC,kBAAe,IAAI,wBAAwB,EAAE,CAAC","sourcesContent":["import type { TestSuite, IAgentRuntime, Memory } from '@elizaos/core';\nimport { createUniqueUuid } from '@elizaos/core';\nimport { VisionService } from '../../service';\nimport { VisionMode } from '../../types';\nimport { setVisionModeAction } from '../../action';\n\nexport class ScreenVisionE2ETestSuite implements TestSuite {\n  name = 'plugin-vision-screen-e2e';\n  description = 'E2E tests for screen vision functionality including Florence-2 and OCR';\n\n  tests = [\n    {\n      name: 'Should initialize screen vision components',\n      fn: async (runtime: IAgentRuntime) => {\n        console.log('Testing screen vision initialization...');\n\n        const visionService = runtime.getService<VisionService>('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not available');\n        }\n\n        // Set vision mode to SCREEN\n        await visionService.setVisionMode(VisionMode.SCREEN);\n\n        // Wait for initialization\n        await new Promise((resolve) => setTimeout(resolve, 2000));\n\n        const mode = visionService.getVisionMode();\n        if (mode !== VisionMode.SCREEN) {\n          throw new Error(`Expected vision mode SCREEN but got ${mode}`);\n        }\n\n        console.log('✓ Screen vision mode activated');\n\n        // Check if screen capture is available\n        const screenInfo = await visionService.getScreenCapture();\n        if (screenInfo) {\n          console.log(`✓ Screen capture available: ${screenInfo.width}x${screenInfo.height}`);\n        } else {\n          console.log('⚠️  Screen capture not yet available (may still be initializing)');\n        }\n      },\n    },\n\n    {\n      name: 'Should capture and tile screen',\n      fn: async (runtime: IAgentRuntime) => {\n        console.log('Testing screen capture and tiling...');\n\n        const visionService = runtime.getService<VisionService>('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not available');\n        }\n\n        // Ensure screen mode is active\n        await visionService.setVisionMode(VisionMode.SCREEN);\n\n        // Wait for capture\n        await new Promise((resolve) => setTimeout(resolve, 3000));\n\n        const screenCapture = await visionService.getScreenCapture();\n        if (!screenCapture) {\n          console.warn(\n            '⚠️  No screen capture available - screen capture may not be supported in this environment'\n          );\n          return;\n        }\n\n        console.log(`✓ Screen captured: ${screenCapture.width}x${screenCapture.height}`);\n        console.log(`✓ Divided into ${screenCapture.tiles.length} tiles`);\n\n        // Check tile structure\n        const firstTile = screenCapture.tiles[0];\n        if (!firstTile) {\n          throw new Error('No tiles created from screen capture');\n        }\n\n        console.log(`  First tile: ${firstTile.id} at (${firstTile.x}, ${firstTile.y})`);\n        console.log(`  Tile size: ${firstTile.width}x${firstTile.height}`);\n\n        // Check if priority tiles have data\n        const tilesWithData = screenCapture.tiles.filter((t) => t.data !== undefined);\n        console.log(`  Tiles with data: ${tilesWithData.length}`);\n\n        if (tilesWithData.length === 0) {\n          console.warn('⚠️  No tiles have been processed yet');\n        }\n      },\n    },\n\n    {\n      name: 'Should analyze screen content with Florence-2 and OCR',\n      fn: async (runtime: IAgentRuntime) => {\n        console.log('Testing screen content analysis...');\n\n        const visionService = runtime.getService<VisionService>('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not available');\n        }\n\n        // Ensure screen mode is active\n        await visionService.setVisionMode(VisionMode.SCREEN);\n\n        // Wait for analysis\n        await new Promise((resolve) => setTimeout(resolve, 5000));\n\n        const enhancedScene = await visionService.getEnhancedSceneDescription();\n        if (!enhancedScene || !enhancedScene.screenAnalysis) {\n          console.warn('⚠️  No enhanced scene analysis available yet');\n          return;\n        }\n\n        console.log('✓ Enhanced scene analysis available');\n\n        const screenAnalysis = enhancedScene.screenAnalysis;\n\n        // Check active tile analysis\n        if (screenAnalysis.activeTile) {\n          console.log('✓ Active tile analyzed');\n\n          if (screenAnalysis.activeTile.florence2) {\n            console.log(`  Florence-2 caption: ${screenAnalysis.activeTile.florence2.caption}`);\n            console.log(\n              `  Objects detected: ${screenAnalysis.activeTile.florence2.objects?.length || 0}`\n            );\n            console.log(\n              `  Tags: ${screenAnalysis.activeTile.florence2.tags?.join(', ') || 'none'}`\n            );\n          }\n\n          if (screenAnalysis.activeTile.ocr) {\n            console.log(`  OCR text blocks: ${screenAnalysis.activeTile.ocr.blocks.length}`);\n            console.log(\n              `  OCR preview: \"${screenAnalysis.activeTile.ocr.fullText.substring(0, 50)}...\"`\n            );\n          }\n        }\n\n        // Check full screen OCR\n        if (screenAnalysis.fullScreenOCR) {\n          console.log(`✓ Full screen OCR: ${screenAnalysis.fullScreenOCR.length} characters`);\n        }\n\n        // Check UI elements\n        if (screenAnalysis.uiElements && screenAnalysis.uiElements.length > 0) {\n          console.log(`✓ UI elements detected: ${screenAnalysis.uiElements.length}`);\n          const elementTypes = screenAnalysis.uiElements.map((e) => e.type);\n          const uniqueTypes = [...new Set(elementTypes)];\n          console.log(`  Types: ${uniqueTypes.join(', ')}`);\n        }\n      },\n    },\n\n    {\n      name: 'Should switch between vision modes',\n      fn: async (runtime: IAgentRuntime) => {\n        console.log('Testing vision mode switching...');\n\n        const visionService = runtime.getService<VisionService>('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not available');\n        }\n\n        // Test all modes\n        const modes = [VisionMode.CAMERA, VisionMode.SCREEN, VisionMode.BOTH, VisionMode.OFF];\n\n        for (const mode of modes) {\n          console.log(`  Switching to ${mode} mode...`);\n          await visionService.setVisionMode(mode);\n\n          // Wait for mode switch\n          await new Promise((resolve) => setTimeout(resolve, 1000));\n\n          const currentMode = visionService.getVisionMode();\n          if (currentMode !== mode) {\n            throw new Error(`Failed to switch to ${mode} mode, current mode is ${currentMode}`);\n          }\n\n          console.log(`  ✓ Successfully switched to ${mode} mode`);\n        }\n\n        // Test with action\n        console.log('  Testing SET_VISION_MODE action...');\n\n        const message: Memory = {\n          id: createUniqueUuid(runtime, 'test-msg'),\n          entityId: runtime.agentId,\n          content: { text: 'set vision mode to both' },\n          agentId: runtime.agentId,\n          roomId: createUniqueUuid(runtime, 'test-room'),\n          createdAt: Date.now(),\n        };\n\n        let callbackCalled = false;\n        await setVisionModeAction.handler(\n          runtime,\n          message,\n          { values: {}, data: {}, text: '' },\n          {},\n          async (response) => {\n            callbackCalled = true;\n            console.log(`  Action response: ${response.text}`);\n            return [];\n          }\n        );\n\n        if (!callbackCalled) {\n          throw new Error('SET_VISION_MODE action did not call callback');\n        }\n\n        const finalMode = visionService.getVisionMode();\n        if (finalMode !== VisionMode.BOTH) {\n          throw new Error(`SET_VISION_MODE action failed, mode is ${finalMode}`);\n        }\n\n        console.log('✓ Vision mode switching works correctly');\n      },\n    },\n\n    {\n      name: 'Should provide combined vision data in BOTH mode',\n      fn: async (runtime: IAgentRuntime) => {\n        console.log('Testing combined camera and screen vision...');\n\n        const visionService = runtime.getService<VisionService>('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not available');\n        }\n\n        // Set to BOTH mode\n        await visionService.setVisionMode(VisionMode.BOTH);\n\n        // Wait for both systems to initialize\n        await new Promise((resolve) => setTimeout(resolve, 5000));\n\n        const enhancedScene = await visionService.getEnhancedSceneDescription();\n        const hasCamera = visionService.getCameraInfo() !== null;\n        const hasScreen = (await visionService.getScreenCapture()) !== null;\n\n        console.log(`  Camera available: ${hasCamera}`);\n        console.log(`  Screen capture available: ${hasScreen}`);\n\n        if (!hasCamera && !hasScreen) {\n          console.warn('⚠️  Neither camera nor screen capture available in this environment');\n          return;\n        }\n\n        if (enhancedScene) {\n          // Check for camera data\n          if (hasCamera && enhancedScene.description) {\n            console.log('✓ Camera data present in combined mode');\n            console.log(`  Scene: ${enhancedScene.description.substring(0, 50)}...`);\n          }\n\n          // Check for screen data\n          if (hasScreen && enhancedScene.screenAnalysis) {\n            console.log('✓ Screen data present in combined mode');\n            console.log(`  Grid: ${enhancedScene.screenAnalysis.gridSummary}`);\n          }\n        }\n\n        // Check provider output\n        const state = await runtime.composeState({\n          id: createUniqueUuid(runtime, 'test-msg'),\n          entityId: runtime.agentId,\n          content: { text: 'test' },\n          agentId: runtime.agentId,\n          roomId: createUniqueUuid(runtime, 'test-room'),\n          createdAt: Date.now(),\n        });\n\n        if (state.text.includes('Vision mode: BOTH')) {\n          console.log('✓ Provider correctly reports BOTH mode');\n        }\n      },\n    },\n\n    {\n      name: 'Should handle screen capture errors gracefully',\n      fn: async (runtime: IAgentRuntime) => {\n        console.log('Testing error handling...');\n\n        const visionService = runtime.getService<VisionService>('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not available');\n        }\n\n        // Try to set invalid region (should handle gracefully)\n        const originalConfig = (visionService as any).visionConfig;\n        (visionService as any).visionConfig.screenRegion = {\n          x: -100,\n          y: -100,\n          width: 50000,\n          height: 50000,\n        };\n\n        await visionService.setVisionMode(VisionMode.SCREEN);\n        await new Promise((resolve) => setTimeout(resolve, 2000));\n\n        // Service should still be active despite invalid region\n        const isActive = visionService.isActive();\n        console.log(`  Service active after invalid config: ${isActive}`);\n\n        // Restore config\n        (visionService as any).visionConfig = originalConfig;\n\n        console.log('✓ Error handling works correctly');\n      },\n    },\n  ];\n}\n\nexport default new ScreenVisionE2ETestSuite();\n"]}

package/dist/tests/e2e/vision-autonomy.d.ts

@@ -0,0 +1,11 @@
+import type { TestSuite, IAgentRuntime } from '@elizaos/core';
+export declare class VisionAutonomyE2ETestSuite implements TestSuite {
+    name: string;
+    description: string;
+    tests: {
+        name: string;
+        fn: (runtime: IAgentRuntime) => Promise<void>;
+    }[];
+}
+declare const _default: VisionAutonomyE2ETestSuite;
+export default _default;