@elizaos/plugin-vision 1.2.1 → 2.0.0-alpha.1

package/dist/tests/e2e/vision-runtime.js

@@ -1,357 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
-    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.VisionRuntimeTestSuite = void 0;
-var core_1 = require("@elizaos/core");
-var VisionRuntimeTestSuite = /** @class */ (function () {
-    function VisionRuntimeTestSuite() {
-        var _this = this;
-        this.name = 'vision-runtime-tests';
-        this.description = 'Real runtime tests for vision plugin functionality';
-        this.tests = [
-            {
-                name: 'Vision service initialization',
-                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
-                    var visionService, isActive;
-                    return __generator(this, function (_a) {
-                        core_1.logger.info('[Test] Testing vision service initialization...');
-                        visionService = runtime.getService('VISION');
-                        if (!visionService) {
-                            throw new Error('Vision service not found in runtime');
-                        }
-                        // Check if service is properly initialized
-                        if (typeof visionService.isActive !== 'function') {
-                            throw new Error('Vision service missing isActive method');
-                        }
-                        isActive = visionService.isActive();
-                        core_1.logger.info("[Test] Vision service active: ".concat(isActive));
-                        // Service should be active after initialization
-                        if (!isActive && runtime.getSetting('VISION_MODE') !== 'OFF') {
-                            throw new Error('Vision service should be active but is not');
-                        }
-                        core_1.logger.info('[Test] ✅ Vision service initialization test passed');
-                        return [2 /*return*/];
-                    });
-                }); },
-            },
-            {
-                name: 'Scene description functionality',
-                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
-                    var visionService, scene;
-                    return __generator(this, function (_a) {
-                        switch (_a.label) {
-                            case 0:
-                                core_1.logger.info('[Test] Testing scene description...');
-                                visionService = runtime.getService('VISION');
-                                if (!visionService) {
-                                    throw new Error('Vision service not found');
-                                }
-                                return [4 /*yield*/, visionService.getSceneDescription()];
-                            case 1:
-                                scene = _a.sent();
-                                if (!scene) {
-                                    core_1.logger.warn('[Test] No scene description available (camera might not be connected)');
-                                    return [2 /*return*/]; // Don't fail if no camera
-                                }
-                                // Validate scene structure
-                                if (typeof scene.timestamp !== 'number') {
-                                    throw new Error('Scene description missing timestamp');
-                                }
-                                if (typeof scene.description !== 'string') {
-                                    throw new Error('Scene description missing description text');
-                                }
-                                if (!Array.isArray(scene.objects)) {
-                                    throw new Error('Scene description missing objects array');
-                                }
-                                if (!Array.isArray(scene.people)) {
-                                    throw new Error('Scene description missing people array');
-                                }
-                                core_1.logger.info("[Test] Scene: ".concat(scene.description.substring(0, 100), "..."));
-                                core_1.logger.info("[Test] Objects: ".concat(scene.objects.length, ", People: ").concat(scene.people.length));
-                                core_1.logger.info('[Test] ✅ Scene description test passed');
-                                return [2 /*return*/];
-                        }
-                    });
-                }); },
-            },
-            {
-                name: 'Vision mode switching',
-                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
-                    var visionService, originalMode, testModes, _i, testModes_1, mode, currentMode;
-                    return __generator(this, function (_a) {
-                        switch (_a.label) {
-                            case 0:
-                                core_1.logger.info('[Test] Testing vision mode switching...');
-                                visionService = runtime.getService('VISION');
-                                if (!visionService) {
-                                    throw new Error('Vision service not found');
-                                }
-                                originalMode = visionService.getVisionMode();
-                                core_1.logger.info("[Test] Original mode: ".concat(originalMode));
-                                testModes = ['CAMERA', 'SCREEN', 'BOTH', 'OFF'];
-                                _i = 0, testModes_1 = testModes;
-                                _a.label = 1;
-                            case 1:
-                                if (!(_i < testModes_1.length)) return [3 /*break*/, 4];
-                                mode = testModes_1[_i];
-                                core_1.logger.info("[Test] Switching to mode: ".concat(mode));
-                                return [4 /*yield*/, visionService.setVisionMode(mode)];
-                            case 2:
-                                _a.sent();
-                                currentMode = visionService.getVisionMode();
-                                if (currentMode !== mode) {
-                                    throw new Error("Failed to switch to mode ".concat(mode, ", current mode is ").concat(currentMode));
-                                }
-                                _a.label = 3;
-                            case 3:
-                                _i++;
-                                return [3 /*break*/, 1];
-                            case 4: 
-                            // Restore original mode
-                            return [4 /*yield*/, visionService.setVisionMode(originalMode)];
-                            case 5:
-                                // Restore original mode
-                                _a.sent();
-                                core_1.logger.info('[Test] ✅ Vision mode switching test passed');
-                                return [2 /*return*/];
-                        }
-                    });
-                }); },
-            },
-            {
-                name: 'DESCRIBE_SCENE action execution',
-                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
-                    var action, message, isValid, responseReceived, callback;
-                    var _this = this;
-                    return __generator(this, function (_a) {
-                        switch (_a.label) {
-                            case 0:
-                                core_1.logger.info('[Test] Testing DESCRIBE_SCENE action...');
-                                action = runtime.actions.find(function (a) { return a.name === 'DESCRIBE_SCENE'; });
-                                if (!action) {
-                                    throw new Error('DESCRIBE_SCENE action not found');
-                                }
-                                message = {
-                                    id: "test-msg-".concat(Date.now()),
-                                    entityId: 'test-entity',
-                                    roomId: 'test-room',
-                                    content: {
-                                        text: 'Describe what you see',
-                                        source: 'test',
-                                    },
-                                    createdAt: Date.now(),
-                                };
-                                return [4 /*yield*/, action.validate(runtime, message)];
-                            case 1:
-                                isValid = _a.sent();
-                                if (!isValid) {
-                                    throw new Error('DESCRIBE_SCENE action validation failed');
-                                }
-                                responseReceived = false;
-                                callback = function (response) { return __awaiter(_this, void 0, void 0, function () {
-                                    return __generator(this, function (_a) {
-                                        if (response.text && response.text.length > 0) {
-                                            responseReceived = true;
-                                            core_1.logger.info("[Test] Action response: ".concat(response.text.substring(0, 100), "..."));
-                                        }
-                                        return [2 /*return*/, []];
-                                    });
-                                }); };
-                                return [4 /*yield*/, action.handler(runtime, message, {}, {}, callback)];
-                            case 2:
-                                _a.sent();
-                                if (!responseReceived) {
-                                    throw new Error('DESCRIBE_SCENE action did not produce a response');
-                                }
-                                core_1.logger.info('[Test] ✅ DESCRIBE_SCENE action test passed');
-                                return [2 /*return*/];
-                        }
-                    });
-                }); },
-            },
-            {
-                name: 'Vision provider integration',
-                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
-                    var provider, message, state, result;
-                    return __generator(this, function (_a) {
-                        switch (_a.label) {
-                            case 0:
-                                core_1.logger.info('[Test] Testing vision provider...');
-                                provider = runtime.providers.find(function (p) { return p.name === 'visionProvider'; });
-                                if (!provider) {
-                                    throw new Error('Vision provider not found');
-                                }
-                                message = {
-                                    id: "test-msg-".concat(Date.now()),
-                                    entityId: 'test-entity',
-                                    roomId: 'test-room',
-                                    content: { text: 'test', source: 'test' },
-                                    createdAt: Date.now(),
-                                };
-                                state = {
-                                    values: {},
-                                    data: {},
-                                    text: '',
-                                };
-                                return [4 /*yield*/, provider.get(runtime, message, state)];
-                            case 1:
-                                result = _a.sent();
-                                if (!result || typeof result !== 'object') {
-                                    throw new Error('Vision provider returned invalid result');
-                                }
-                                // Check if provider returns vision data when available
-                                if (result.text && result.text.includes('I can see')) {
-                                    core_1.logger.info("[Test] Provider text: ".concat(result.text.substring(0, 100), "..."));
-                                }
-                                core_1.logger.info('[Test] ✅ Vision provider test passed');
-                                return [2 /*return*/];
-                        }
-                    });
-                }); },
-            },
-            {
-                name: 'Florence-2 model initialization',
-                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
-                    var visionService, florence2Enabled, mode, screenCapture;
-                    return __generator(this, function (_a) {
-                        switch (_a.label) {
-                            case 0:
-                                core_1.logger.info('[Test] Testing Florence-2 model...');
-                                visionService = runtime.getService('VISION');
-                                if (!visionService) {
-                                    throw new Error('Vision service not found');
-                                }
-                                florence2Enabled = runtime.getSetting('FLORENCE2_ENABLED') === 'true' ||
-                                    runtime.getSetting('VISION_FLORENCE2_ENABLED') === 'true';
-                                if (!florence2Enabled) {
-                                    core_1.logger.info('[Test] Florence-2 is disabled, skipping test');
-                                    return [2 /*return*/];
-                                }
-                                mode = visionService.getVisionMode();
-                                if (!(mode === 'SCREEN' || mode === 'BOTH')) return [3 /*break*/, 2];
-                                return [4 /*yield*/, visionService.getScreenCapture()];
-                            case 1:
-                                screenCapture = _a.sent();
-                                if (screenCapture) {
-                                    core_1.logger.info('[Test] Screen capture available');
-                                    core_1.logger.info("[Test] Screen size: ".concat(screenCapture.width, "x").concat(screenCapture.height));
-                                    core_1.logger.info("[Test] Tiles: ".concat(screenCapture.tiles.length));
-                                }
-                                _a.label = 2;
-                            case 2:
-                                core_1.logger.info('[Test] ✅ Florence-2 model test passed');
-                                return [2 /*return*/];
-                        }
-                    });
-                }); },
-            },
-            {
-                name: 'OCR service functionality',
-                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
-                    var visionService, ocrEnabled, mode, enhancedScene, ocrText;
-                    return __generator(this, function (_a) {
-                        switch (_a.label) {
-                            case 0:
-                                core_1.logger.info('[Test] Testing OCR service...');
-                                visionService = runtime.getService('VISION');
-                                if (!visionService) {
-                                    throw new Error('Vision service not found');
-                                }
-                                ocrEnabled = runtime.getSetting('OCR_ENABLED') === 'true' ||
-                                    runtime.getSetting('VISION_OCR_ENABLED') === 'true';
-                                if (!ocrEnabled) {
-                                    core_1.logger.info('[Test] OCR is disabled, skipping test');
-                                    return [2 /*return*/];
-                                }
-                                mode = visionService.getVisionMode();
-                                if (!(mode === 'SCREEN' || mode === 'BOTH')) return [3 /*break*/, 2];
-                                return [4 /*yield*/, visionService.getEnhancedSceneDescription()];
-                            case 1:
-                                enhancedScene = _a.sent();
-                                if (enhancedScene && enhancedScene.screenAnalysis) {
-                                    ocrText = enhancedScene.screenAnalysis.fullScreenOCR;
-                                    if (ocrText) {
-                                        core_1.logger.info("[Test] OCR extracted ".concat(ocrText.length, " characters"));
-                                        core_1.logger.info("[Test] OCR sample: ".concat(ocrText.substring(0, 100), "..."));
-                                    }
-                                }
-                                _a.label = 2;
-                            case 2:
-                                core_1.logger.info('[Test] ✅ OCR service test passed');
-                                return [2 /*return*/];
-                        }
-                    });
-                }); },
-            },
-            {
-                name: 'Entity tracking system',
-                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
-                    var visionService, entityTracker, entities, _i, entities_1, entity;
-                    return __generator(this, function (_a) {
-                        core_1.logger.info('[Test] Testing entity tracking...');
-                        visionService = runtime.getService('VISION');
-                        if (!visionService) {
-                            throw new Error('Vision service not found');
-                        }
-                        entityTracker = visionService.getEntityTracker();
-                        if (!entityTracker) {
-                            throw new Error('Entity tracker not found');
-                        }
-                        entities = entityTracker.getActiveEntities();
-                        core_1.logger.info("[Test] Active entities: ".concat(entities.length));
-                        // Check entity structure if any exist
-                        for (_i = 0, entities_1 = entities; _i < entities_1.length; _i++) {
-                            entity = entities_1[_i];
-                            if (!entity.id || !entity.type || !entity.lastSeen) {
-                                throw new Error('Entity missing required fields');
-                            }
-                            core_1.logger.info("[Test] Entity ".concat(entity.id, ": type=").concat(entity.type, ", tracked=").concat(entity.trackingDuration, "ms"));
-                        }
-                        core_1.logger.info('[Test] ✅ Entity tracking test passed');
-                        return [2 /*return*/];
-                    });
-                }); },
-            },
-        ];
-    }
-    return VisionRuntimeTestSuite;
-}());
-exports.VisionRuntimeTestSuite = VisionRuntimeTestSuite;
-// Export default instance for test runner
-exports.default = new VisionRuntimeTestSuite();
-//# sourceMappingURL=vision-runtime.js.map

package/dist/tests/e2e/vision-runtime.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"vision-runtime.js","sourceRoot":"","sources":["../../../src/tests/e2e/vision-runtime.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,sCAAuC;AAEvC;IAAA;QAAA,iBAiTC;QAhTC,SAAI,GAAG,sBAAsB,CAAC;QAC9B,gBAAW,GAAG,oDAAoD,CAAC;QAEnE,UAAK,GAAG;YACN;gBACE,IAAI,EAAE,+BAA+B;gBACrC,EAAE,EAAE,UAAO,OAAY;;;wBACrB,aAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;wBAEzD,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;wBACnD,IAAI,CAAC,aAAa,EAAE,CAAC;4BACnB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;wBACzD,CAAC;wBAED,2CAA2C;wBAC3C,IAAI,OAAO,aAAa,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;4BACjD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;wBAC5D,CAAC;wBAEK,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,CAAC;wBAC1C,aAAM,CAAC,IAAI,CAAC,wCAAiC,QAAQ,CAAE,CAAC,CAAC;wBAEzD,gDAAgD;wBAChD,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,KAAK,EAAE,CAAC;4BAC7D,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;wBAChE,CAAC;wBAED,aAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;;;qBACnE;aACF;YAED;gBACE,IAAI,EAAE,iCAAiC;gBACvC,EAAE,EAAE,UAAO,OAAY;;;;;gCACrB,aAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;gCAE7C,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gCACnD,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;gCAC9C,CAAC;gCAGa,qBAAM,aAAa,CAAC,mBAAmB,EAAE,EAAA;;gCAAjD,KAAK,GAAG,SAAyC;gCAEvD,IAAI,CAAC,KAAK,EAAE,CAAC;oCACX,aAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;oCACrF,sBAAO,CAAC,0BAA0B;gCACpC,CAAC;gCAED,2BAA2B;gCAC3B,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;oCACxC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;gCACzD,CAAC;gCAED,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;oCAC1C,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;gCAChE,CAAC;gCAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oCAClC,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gCAC7D,CAAC;gCAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;oCACjC,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;gCAC5D,CAAC;gCAED,aAAM,CAAC,IAAI,CAAC,wBAAiB,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,QAAK,CAAC,CAAC;gCACvE,aAAM,CAAC,IAAI,CAAC,0BAAmB,KAAK,CAAC,OAAO,CAAC,MAAM,uBAAa,KAAK,CAAC,MAAM,CAAC,MAAM,CAAE,CAAC,CAAC;gCACvF,aAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;;;;qBACvD;aACF;YAED;gBACE,IAAI,EAAE,uBAAuB;gBAC7B,EAAE,EAAE,UAAO,OAAY;;;;;gCACrB,aAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;gCAEjD,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gCACnD,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;gCAC9C,CAAC;gCAGK,YAAY,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;gCACnD,aAAM,CAAC,IAAI,CAAC,gCAAyB,YAAY,CAAE,CAAC,CAAC;gCAG/C,SAAS,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;sCAE1B,EAAT,uBAAS;;;qCAAT,CAAA,uBAAS,CAAA;gCAAjB,IAAI;gCACb,aAAM,CAAC,IAAI,CAAC,oCAA6B,IAAI,CAAE,CAAC,CAAC;gCACjD,qBAAM,aAAa,CAAC,aAAa,CAAC,IAAI,CAAC,EAAA;;gCAAvC,SAAuC,CAAC;gCAElC,WAAW,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;gCAClD,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;oCACzB,MAAM,IAAI,KAAK,CAAC,mCAA4B,IAAI,+BAAqB,WAAW,CAAE,CAAC,CAAC;gCACtF,CAAC;;;gCAPgB,IAAS,CAAA;;;4BAU5B,wBAAwB;4BACxB,qBAAM,aAAa,CAAC,aAAa,CAAC,YAAY,CAAC,EAAA;;gCAD/C,wBAAwB;gCACxB,SAA+C,CAAC;gCAChD,aAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;;;;qBAC3D;aACF;YAED;gBACE,IAAI,EAAE,iCAAiC;gBACvC,EAAE,EAAE,UAAO,OAAY;;;;;;gCACrB,aAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;gCAGjD,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAC,CAAM,IAAK,OAAA,CAAC,CAAC,IAAI,KAAK,gBAAgB,EAA3B,CAA2B,CAAC,CAAC;gCAC7E,IAAI,CAAC,MAAM,EAAE,CAAC;oCACZ,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;gCACrD,CAAC;gCAGK,OAAO,GAAG;oCACd,EAAE,EAAE,mBAAY,IAAI,CAAC,GAAG,EAAE,CAAE;oCAC5B,QAAQ,EAAE,aAAa;oCACvB,MAAM,EAAE,WAAW;oCACnB,OAAO,EAAE;wCACP,IAAI,EAAE,uBAAuB;wCAC7B,MAAM,EAAE,MAAM;qCACf;oCACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iCACtB,CAAC;gCAGc,qBAAM,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,EAAA;;gCAAjD,OAAO,GAAG,SAAuC;gCACvD,IAAI,CAAC,OAAO,EAAE,CAAC;oCACb,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gCAC7D,CAAC;gCAGG,gBAAgB,GAAG,KAAK,CAAC;gCACvB,QAAQ,GAAG,UAAO,QAAa;;wCACnC,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4CAC9C,gBAAgB,GAAG,IAAI,CAAC;4CACxB,aAAM,CAAC,IAAI,CAAC,kCAA2B,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,QAAK,CAAC,CAAC;wCAC/E,CAAC;wCACD,sBAAO,EAAE,EAAC;;qCACX,CAAC;gCAEF,qBAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAA;;gCAAxD,SAAwD,CAAC;gCAEzD,IAAI,CAAC,gBAAgB,EAAE,CAAC;oCACtB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;gCACtE,CAAC;gCAED,aAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;;;;qBAC3D;aACF;YAED;gBACE,IAAI,EAAE,6BAA6B;gBACnC,EAAE,EAAE,UAAO,OAAY;;;;;gCACrB,aAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;gCAG3C,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,UAAC,CAAM,IAAK,OAAA,CAAC,CAAC,IAAI,KAAK,gBAAgB,EAA3B,CAA2B,CAAC,CAAC;gCACjF,IAAI,CAAC,QAAQ,EAAE,CAAC;oCACd,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;gCAC/C,CAAC;gCAGK,OAAO,GAAG;oCACd,EAAE,EAAE,mBAAY,IAAI,CAAC,GAAG,EAAE,CAAE;oCAC5B,QAAQ,EAAE,aAAa;oCACvB,MAAM,EAAE,WAAW;oCACnB,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;oCACzC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iCACtB,CAAC;gCAEI,KAAK,GAAG;oCACZ,MAAM,EAAE,EAAE;oCACV,IAAI,EAAE,EAAE;oCACR,IAAI,EAAE,EAAE;iCACT,CAAC;gCAGa,qBAAM,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,EAAA;;gCAApD,MAAM,GAAG,SAA2C;gCAE1D,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;oCAC1C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gCAC7D,CAAC;gCAED,uDAAuD;gCACvD,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oCACrD,aAAM,CAAC,IAAI,CAAC,gCAAyB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,QAAK,CAAC,CAAC;gCAC3E,CAAC;gCAED,aAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;;;;qBACrD;aACF;YAED;gBACE,IAAI,EAAE,iCAAiC;gBACvC,EAAE,EAAE,UAAO,OAAY;;;;;gCACrB,aAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;gCAE5C,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gCACnD,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;gCAC9C,CAAC;gCAGK,gBAAgB,GACpB,OAAO,CAAC,UAAU,CAAC,mBAAmB,CAAC,KAAK,MAAM;oCAClD,OAAO,CAAC,UAAU,CAAC,0BAA0B,CAAC,KAAK,MAAM,CAAC;gCAE5D,IAAI,CAAC,gBAAgB,EAAE,CAAC;oCACtB,aAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;oCAC5D,sBAAO;gCACT,CAAC;gCAGK,IAAI,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;qCACvC,CAAA,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,MAAM,CAAA,EAApC,wBAAoC;gCAEhB,qBAAM,aAAa,CAAC,gBAAgB,EAAE,EAAA;;gCAAtD,aAAa,GAAG,SAAsC;gCAC5D,IAAI,aAAa,EAAE,CAAC;oCAClB,aAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;oCAC/C,aAAM,CAAC,IAAI,CAAC,8BAAuB,aAAa,CAAC,KAAK,cAAI,aAAa,CAAC,MAAM,CAAE,CAAC,CAAC;oCAClF,aAAM,CAAC,IAAI,CAAC,wBAAiB,aAAa,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC,CAAC;gCAC7D,CAAC;;;gCAGH,aAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;;;;qBACtD;aACF;YAED;gBACE,IAAI,EAAE,2BAA2B;gBACjC,EAAE,EAAE,UAAO,OAAY;;;;;gCACrB,aAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;gCAEvC,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gCACnD,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;gCAC9C,CAAC;gCAGK,UAAU,GACd,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,MAAM;oCAC5C,OAAO,CAAC,UAAU,CAAC,oBAAoB,CAAC,KAAK,MAAM,CAAC;gCAEtD,IAAI,CAAC,UAAU,EAAE,CAAC;oCAChB,aAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;oCACrD,sBAAO;gCACT,CAAC;gCAGK,IAAI,GAAG,aAAa,CAAC,aAAa,EAAE,CAAC;qCACvC,CAAA,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,MAAM,CAAA,EAApC,wBAAoC;gCAChB,qBAAM,aAAa,CAAC,2BAA2B,EAAE,EAAA;;gCAAjE,aAAa,GAAG,SAAiD;gCACvE,IAAI,aAAa,IAAI,aAAa,CAAC,cAAc,EAAE,CAAC;oCAC5C,OAAO,GAAG,aAAa,CAAC,cAAc,CAAC,aAAa,CAAC;oCAC3D,IAAI,OAAO,EAAE,CAAC;wCACZ,aAAM,CAAC,IAAI,CAAC,+BAAwB,OAAO,CAAC,MAAM,gBAAa,CAAC,CAAC;wCACjE,aAAM,CAAC,IAAI,CAAC,6BAAsB,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,QAAK,CAAC,CAAC;oCACpE,CAAC;gCACH,CAAC;;;gCAGH,aAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;;;;qBACjD;aACF;YAED;gBACE,IAAI,EAAE,wBAAwB;gBAC9B,EAAE,EAAE,UAAO,OAAY;;;wBACrB,aAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;wBAE3C,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;wBACnD,IAAI,CAAC,aAAa,EAAE,CAAC;4BACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;wBAC9C,CAAC;wBAGK,aAAa,GAAG,aAAa,CAAC,gBAAgB,EAAE,CAAC;wBACvD,IAAI,CAAC,aAAa,EAAE,CAAC;4BACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;wBAC9C,CAAC;wBAGK,QAAQ,GAAG,aAAa,CAAC,iBAAiB,EAAE,CAAC;wBACnD,aAAM,CAAC,IAAI,CAAC,kCAA2B,QAAQ,CAAC,MAAM,CAAE,CAAC,CAAC;wBAE1D,sCAAsC;wBACtC,WAA6B,EAAR,qBAAQ,EAAR,sBAAQ,EAAR,IAAQ,EAAE,CAAC;4BAArB,MAAM;4BACf,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gCACnD,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;4BACpD,CAAC;4BAED,aAAM,CAAC,IAAI,CACT,wBAAiB,MAAM,CAAC,EAAE,oBAAU,MAAM,CAAC,IAAI,uBAAa,MAAM,CAAC,gBAAgB,OAAI,CACxF,CAAC;wBACJ,CAAC;wBAED,aAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;;;qBACrD;aACF;SACF,CAAC;IACJ,CAAC;IAAD,6BAAC;AAAD,CAAC,AAjTD,IAiTC;AAjTY,wDAAsB;AAmTnC,0CAA0C;AAC1C,kBAAe,IAAI,sBAAsB,EAAE,CAAC","sourcesContent":["import type { TestSuite } from '@elizaos/core';\nimport { logger } from '@elizaos/core';\n\nexport class VisionRuntimeTestSuite implements TestSuite {\n  name = 'vision-runtime-tests';\n  description = 'Real runtime tests for vision plugin functionality';\n\n  tests = [\n    {\n      name: 'Vision service initialization',\n      fn: async (runtime: any) => {\n        logger.info('[Test] Testing vision service initialization...');\n\n        const visionService = runtime.getService('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not found in runtime');\n        }\n\n        // Check if service is properly initialized\n        if (typeof visionService.isActive !== 'function') {\n          throw new Error('Vision service missing isActive method');\n        }\n\n        const isActive = visionService.isActive();\n        logger.info(`[Test] Vision service active: ${isActive}`);\n\n        // Service should be active after initialization\n        if (!isActive && runtime.getSetting('VISION_MODE') !== 'OFF') {\n          throw new Error('Vision service should be active but is not');\n        }\n\n        logger.info('[Test] ✅ Vision service initialization test passed');\n      },\n    },\n\n    {\n      name: 'Scene description functionality',\n      fn: async (runtime: any) => {\n        logger.info('[Test] Testing scene description...');\n\n        const visionService = runtime.getService('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not found');\n        }\n\n        // Get current scene description\n        const scene = await visionService.getSceneDescription();\n\n        if (!scene) {\n          logger.warn('[Test] No scene description available (camera might not be connected)');\n          return; // Don't fail if no camera\n        }\n\n        // Validate scene structure\n        if (typeof scene.timestamp !== 'number') {\n          throw new Error('Scene description missing timestamp');\n        }\n\n        if (typeof scene.description !== 'string') {\n          throw new Error('Scene description missing description text');\n        }\n\n        if (!Array.isArray(scene.objects)) {\n          throw new Error('Scene description missing objects array');\n        }\n\n        if (!Array.isArray(scene.people)) {\n          throw new Error('Scene description missing people array');\n        }\n\n        logger.info(`[Test] Scene: ${scene.description.substring(0, 100)}...`);\n        logger.info(`[Test] Objects: ${scene.objects.length}, People: ${scene.people.length}`);\n        logger.info('[Test] ✅ Scene description test passed');\n      },\n    },\n\n    {\n      name: 'Vision mode switching',\n      fn: async (runtime: any) => {\n        logger.info('[Test] Testing vision mode switching...');\n\n        const visionService = runtime.getService('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not found');\n        }\n\n        // Get current mode\n        const originalMode = visionService.getVisionMode();\n        logger.info(`[Test] Original mode: ${originalMode}`);\n\n        // Test switching modes\n        const testModes = ['CAMERA', 'SCREEN', 'BOTH', 'OFF'];\n\n        for (const mode of testModes) {\n          logger.info(`[Test] Switching to mode: ${mode}`);\n          await visionService.setVisionMode(mode);\n\n          const currentMode = visionService.getVisionMode();\n          if (currentMode !== mode) {\n            throw new Error(`Failed to switch to mode ${mode}, current mode is ${currentMode}`);\n          }\n        }\n\n        // Restore original mode\n        await visionService.setVisionMode(originalMode);\n        logger.info('[Test] ✅ Vision mode switching test passed');\n      },\n    },\n\n    {\n      name: 'DESCRIBE_SCENE action execution',\n      fn: async (runtime: any) => {\n        logger.info('[Test] Testing DESCRIBE_SCENE action...');\n\n        // Find the action\n        const action = runtime.actions.find((a: any) => a.name === 'DESCRIBE_SCENE');\n        if (!action) {\n          throw new Error('DESCRIBE_SCENE action not found');\n        }\n\n        // Create test message\n        const message = {\n          id: `test-msg-${Date.now()}`,\n          entityId: 'test-entity',\n          roomId: 'test-room',\n          content: {\n            text: 'Describe what you see',\n            source: 'test',\n          },\n          createdAt: Date.now(),\n        };\n\n        // Validate action\n        const isValid = await action.validate(runtime, message);\n        if (!isValid) {\n          throw new Error('DESCRIBE_SCENE action validation failed');\n        }\n\n        // Execute action\n        let responseReceived = false;\n        const callback = async (response: any) => {\n          if (response.text && response.text.length > 0) {\n            responseReceived = true;\n            logger.info(`[Test] Action response: ${response.text.substring(0, 100)}...`);\n          }\n          return [];\n        };\n\n        await action.handler(runtime, message, {}, {}, callback);\n\n        if (!responseReceived) {\n          throw new Error('DESCRIBE_SCENE action did not produce a response');\n        }\n\n        logger.info('[Test] ✅ DESCRIBE_SCENE action test passed');\n      },\n    },\n\n    {\n      name: 'Vision provider integration',\n      fn: async (runtime: any) => {\n        logger.info('[Test] Testing vision provider...');\n\n        // Find the vision provider\n        const provider = runtime.providers.find((p: any) => p.name === 'visionProvider');\n        if (!provider) {\n          throw new Error('Vision provider not found');\n        }\n\n        // Create test message and state\n        const message = {\n          id: `test-msg-${Date.now()}`,\n          entityId: 'test-entity',\n          roomId: 'test-room',\n          content: { text: 'test', source: 'test' },\n          createdAt: Date.now(),\n        };\n\n        const state = {\n          values: {},\n          data: {},\n          text: '',\n        };\n\n        // Get provider data\n        const result = await provider.get(runtime, message, state);\n\n        if (!result || typeof result !== 'object') {\n          throw new Error('Vision provider returned invalid result');\n        }\n\n        // Check if provider returns vision data when available\n        if (result.text && result.text.includes('I can see')) {\n          logger.info(`[Test] Provider text: ${result.text.substring(0, 100)}...`);\n        }\n\n        logger.info('[Test] ✅ Vision provider test passed');\n      },\n    },\n\n    {\n      name: 'Florence-2 model initialization',\n      fn: async (runtime: any) => {\n        logger.info('[Test] Testing Florence-2 model...');\n\n        const visionService = runtime.getService('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not found');\n        }\n\n        // Check if Florence-2 is enabled\n        const florence2Enabled =\n          runtime.getSetting('FLORENCE2_ENABLED') === 'true' ||\n          runtime.getSetting('VISION_FLORENCE2_ENABLED') === 'true';\n\n        if (!florence2Enabled) {\n          logger.info('[Test] Florence-2 is disabled, skipping test');\n          return;\n        }\n\n        // If screen mode is enabled, Florence-2 should be initialized\n        const mode = visionService.getVisionMode();\n        if (mode === 'SCREEN' || mode === 'BOTH') {\n          // Try to get screen capture\n          const screenCapture = await visionService.getScreenCapture();\n          if (screenCapture) {\n            logger.info('[Test] Screen capture available');\n            logger.info(`[Test] Screen size: ${screenCapture.width}x${screenCapture.height}`);\n            logger.info(`[Test] Tiles: ${screenCapture.tiles.length}`);\n          }\n        }\n\n        logger.info('[Test] ✅ Florence-2 model test passed');\n      },\n    },\n\n    {\n      name: 'OCR service functionality',\n      fn: async (runtime: any) => {\n        logger.info('[Test] Testing OCR service...');\n\n        const visionService = runtime.getService('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not found');\n        }\n\n        // Check if OCR is enabled\n        const ocrEnabled =\n          runtime.getSetting('OCR_ENABLED') === 'true' ||\n          runtime.getSetting('VISION_OCR_ENABLED') === 'true';\n\n        if (!ocrEnabled) {\n          logger.info('[Test] OCR is disabled, skipping test');\n          return;\n        }\n\n        // If screen mode is enabled, OCR should work\n        const mode = visionService.getVisionMode();\n        if (mode === 'SCREEN' || mode === 'BOTH') {\n          const enhancedScene = await visionService.getEnhancedSceneDescription();\n          if (enhancedScene && enhancedScene.screenAnalysis) {\n            const ocrText = enhancedScene.screenAnalysis.fullScreenOCR;\n            if (ocrText) {\n              logger.info(`[Test] OCR extracted ${ocrText.length} characters`);\n              logger.info(`[Test] OCR sample: ${ocrText.substring(0, 100)}...`);\n            }\n          }\n        }\n\n        logger.info('[Test] ✅ OCR service test passed');\n      },\n    },\n\n    {\n      name: 'Entity tracking system',\n      fn: async (runtime: any) => {\n        logger.info('[Test] Testing entity tracking...');\n\n        const visionService = runtime.getService('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not found');\n        }\n\n        // Get entity tracker\n        const entityTracker = visionService.getEntityTracker();\n        if (!entityTracker) {\n          throw new Error('Entity tracker not found');\n        }\n\n        // Get current entities\n        const entities = entityTracker.getActiveEntities();\n        logger.info(`[Test] Active entities: ${entities.length}`);\n\n        // Check entity structure if any exist\n        for (const entity of entities) {\n          if (!entity.id || !entity.type || !entity.lastSeen) {\n            throw new Error('Entity missing required fields');\n          }\n\n          logger.info(\n            `[Test] Entity ${entity.id}: type=${entity.type}, tracked=${entity.trackingDuration}ms`\n          );\n        }\n\n        logger.info('[Test] ✅ Entity tracking test passed');\n      },\n    },\n  ];\n}\n\n// Export default instance for test runner\nexport default new VisionRuntimeTestSuite();\n"]}

package/dist/tests/e2e/vision-worker-tests.d.ts

@@ -1,11 +0,0 @@
-import type { TestSuite, IAgentRuntime } from '@elizaos/core';
-export declare class VisionWorkerE2ETestSuite implements TestSuite {
-    name: string;
-    description: string;
-    tests: {
-        name: string;
-        fn: (runtime: IAgentRuntime) => Promise<void>;
-    }[];
-}
-declare const _default: VisionWorkerE2ETestSuite;
-export default _default;