@elizaos/plugin-vision 1.2.1

package/dist/tests/e2e/vision-capture-log.js

@@ -0,0 +1,302 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VisionCaptureLogTestSuite = void 0;
+var fs = require("fs/promises");
+var path = require("path");
+var VisionCaptureLogTestSuite = /** @class */ (function () {
+    function VisionCaptureLogTestSuite() {
+        var _this = this;
+        this.name = 'plugin-vision-capture-log';
+        this.description = 'Captures 30 seconds of vision data and saves to logs for analysis';
+        this.tests = [
+            {
+                name: 'Should capture and log 30 seconds of vision data',
+                fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
+                    var visionService, logsDir, sessionDir, captureData, captureInterval, totalDuration, expectedCaptures, captureCount, totalChangePercentage, lastLoggedProgress, startTime, _loop_1, mainLogPath, reportPath, report;
+                    var _a, _b;
+                    return __generator(this, function (_c) {
+                        switch (_c.label) {
+                            case 0:
+                                console.log('Starting 30-second vision capture test...');
+                                visionService = runtime.getService('VISION');
+                                if (!visionService) {
+                                    throw new Error('Vision service not available');
+                                }
+                                if (!visionService.isActive()) {
+                                    console.warn('⚠️  Vision service not active - no camera available');
+                                    console.log('   Cannot capture vision data without a camera');
+                                    return [2 /*return*/];
+                                }
+                                logsDir = path.join(process.cwd(), 'logs');
+                                sessionDir = path.join(logsDir, "vision-capture-".concat(new Date().toISOString().replace(/[:.]/g, '-')));
+                                return [4 /*yield*/, fs.mkdir(sessionDir, { recursive: true })];
+                            case 1:
+                                _c.sent();
+                                console.log("\u2713 Created log directory: ".concat(sessionDir));
+                                captureData = {
+                                    sessionId: "vision-capture-".concat(Date.now()),
+                                    startTime: new Date().toISOString(),
+                                    endTime: '',
+                                    runtime: {
+                                        agentId: runtime.agentId,
+                                        characterName: runtime.character.name,
+                                    },
+                                    camera: visionService.getCameraInfo(),
+                                    captures: [],
+                                    statistics: {
+                                        totalFrames: 0,
+                                        totalSceneChanges: 0,
+                                        totalObjectsDetected: 0,
+                                        totalPeopleDetected: 0,
+                                        averageChangePercentage: 0,
+                                        objectTypeCounts: {},
+                                        poseCounts: {},
+                                    },
+                                };
+                                captureInterval = 500;
+                                totalDuration = 30000;
+                                expectedCaptures = totalDuration / captureInterval;
+                                console.log("\uD83D\uDCF8 Capturing vision data every ".concat(captureInterval, "ms for ").concat(totalDuration / 1000, " seconds..."));
+                                console.log("   Expected captures: ".concat(expectedCaptures));
+                                captureCount = 0;
+                                totalChangePercentage = 0;
+                                lastLoggedProgress = 0;
+                                startTime = Date.now();
+                                _loop_1 = function () {
+                                    var captureStartTime, scene, frame, imageBuffer, imageBase64, capture, imagePath, _d, _e, obj, _f, _g, person, sceneLogPath, elapsedSeconds, progress, error_1, captureEndTime, captureDuration, waitTime;
+                                    return __generator(this, function (_h) {
+                                        switch (_h.label) {
+                                            case 0:
+                                                captureStartTime = Date.now();
+                                                _h.label = 1;
+                                            case 1:
+                                                _h.trys.push([1, 9, , 10]);
+                                                return [4 /*yield*/, visionService.getSceneDescription()];
+                                            case 2:
+                                                scene = _h.sent();
+                                                return [4 /*yield*/, visionService.getCurrentFrame()];
+                                            case 3:
+                                                frame = _h.sent();
+                                                return [4 /*yield*/, visionService.captureImage()];
+                                            case 4:
+                                                imageBuffer = _h.sent();
+                                                imageBase64 = null;
+                                                if (imageBuffer) {
+                                                    imageBase64 = imageBuffer.toString('base64');
+                                                }
+                                                capture = {
+                                                    index: captureCount,
+                                                    timestamp: new Date().toISOString(),
+                                                    timestampMs: Date.now(),
+                                                    elapsedMs: Date.now() - startTime,
+                                                    scene: scene
+                                                        ? {
+                                                            description: scene.description,
+                                                            changePercentage: scene.changePercentage,
+                                                            sceneChanged: scene.sceneChanged,
+                                                            objectCount: scene.objects.length,
+                                                            peopleCount: scene.people.length,
+                                                            objects: scene.objects,
+                                                            people: scene.people,
+                                                        }
+                                                        : null,
+                                                    frame: frame
+                                                        ? {
+                                                            width: frame.width,
+                                                            height: frame.height,
+                                                            format: frame.format,
+                                                            timestamp: frame.timestamp,
+                                                        }
+                                                        : null,
+                                                    imageBase64: imageBase64 !== null && imageBase64 !== void 0 ? imageBase64 : null,
+                                                };
+                                                captureData.captures.push(capture);
+                                                if (!imageBase64) return [3 /*break*/, 6];
+                                                imagePath = path.join(sessionDir, "capture-".concat(String(captureCount).padStart(3, '0'), ".jpg"));
+                                                return [4 /*yield*/, fs.writeFile(imagePath, Buffer.from(imageBase64, 'base64'))];
+                                            case 5:
+                                                _h.sent();
+                                                _h.label = 6;
+                                            case 6:
+                                                if (!scene) return [3 /*break*/, 8];
+                                                captureData.statistics.totalFrames++;
+                                                if (scene.sceneChanged) {
+                                                    captureData.statistics.totalSceneChanges++;
+                                                }
+                                                if (scene.changePercentage) {
+                                                    totalChangePercentage += scene.changePercentage;
+                                                }
+                                                captureData.statistics.totalObjectsDetected += scene.objects.length;
+                                                captureData.statistics.totalPeopleDetected += scene.people.length;
+                                                // Count object types
+                                                for (_d = 0, _e = scene.objects; _d < _e.length; _d++) {
+                                                    obj = _e[_d];
+                                                    captureData.statistics.objectTypeCounts[obj.type] =
+                                                        (captureData.statistics.objectTypeCounts[obj.type] || 0) + 1;
+                                                }
+                                                // Count poses
+                                                for (_f = 0, _g = scene.people; _f < _g.length; _f++) {
+                                                    person = _g[_f];
+                                                    captureData.statistics.poseCounts[person.pose] =
+                                                        (captureData.statistics.poseCounts[person.pose] || 0) + 1;
+                                                }
+                                                sceneLogPath = path.join(sessionDir, "scene-".concat(String(captureCount).padStart(3, '0'), ".json"));
+                                                return [4 /*yield*/, fs.writeFile(sceneLogPath, JSON.stringify({
+                                                        capture: captureCount,
+                                                        timestamp: capture.timestamp,
+                                                        scene: scene,
+                                                        frame: frame
+                                                            ? {
+                                                                width: frame.width,
+                                                                height: frame.height,
+                                                                format: frame.format,
+                                                            }
+                                                            : null,
+                                                    }, null, 2))];
+                                            case 7:
+                                                _h.sent();
+                                                _h.label = 8;
+                                            case 8:
+                                                captureCount++;
+                                                elapsedSeconds = Math.floor((Date.now() - startTime) / 1000);
+                                                if (elapsedSeconds > lastLoggedProgress && elapsedSeconds % 5 === 0) {
+                                                    lastLoggedProgress = elapsedSeconds;
+                                                    progress = (elapsedSeconds / 30) * 100;
+                                                    console.log("  Progress: ".concat(progress.toFixed(0), "% (").concat(elapsedSeconds, "/30s) - Captured ").concat(captureCount, " frames"));
+                                                    if (scene) {
+                                                        console.log("    Last scene: \"".concat(scene.description.substring(0, 60), "...\""));
+                                                        if (scene.objects.length > 0 || scene.people.length > 0) {
+                                                            console.log("    Detected: ".concat(scene.objects.length, " objects, ").concat(scene.people.length, " people"));
+                                                        }
+                                                    }
+                                                }
+                                                return [3 /*break*/, 10];
+                                            case 9:
+                                                error_1 = _h.sent();
+                                                console.error("  Error in capture ".concat(captureCount, ":"), error_1);
+                                                captureData.captures.push({
+                                                    index: captureCount,
+                                                    timestamp: new Date().toISOString(),
+                                                    error: error_1 instanceof Error ? error_1.message : String(error_1),
+                                                });
+                                                captureCount++;
+                                                return [3 /*break*/, 10];
+                                            case 10:
+                                                captureEndTime = Date.now();
+                                                captureDuration = captureEndTime - captureStartTime;
+                                                waitTime = Math.max(0, captureInterval - captureDuration);
+                                                if (!(waitTime > 0)) return [3 /*break*/, 12];
+                                                return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, waitTime); })];
+                                            case 11:
+                                                _h.sent();
+                                                _h.label = 12;
+                                            case 12: return [2 /*return*/];
+                                        }
+                                    });
+                                };
+                                _c.label = 2;
+                            case 2:
+                                if (!(Date.now() - startTime < totalDuration)) return [3 /*break*/, 4];
+                                return [5 /*yield**/, _loop_1()];
+                            case 3:
+                                _c.sent();
+                                return [3 /*break*/, 2];
+                            case 4:
+                                // Finalize statistics
+                                captureData.endTime = new Date().toISOString();
+                                captureData.statistics.averageChangePercentage =
+                                    captureData.statistics.totalFrames > 0
+                                        ? totalChangePercentage / captureData.statistics.totalFrames
+                                        : 0;
+                                mainLogPath = path.join(sessionDir, 'vision-capture-summary.json');
+                                return [4 /*yield*/, fs.writeFile(mainLogPath, JSON.stringify(captureData, null, 2))];
+                            case 5:
+                                _c.sent();
+                                reportPath = path.join(sessionDir, 'vision-capture-report.md');
+                                report = "# Vision Capture Report\n\n## Session Information\n- **Session ID**: ".concat(captureData.sessionId, "\n- **Start Time**: ").concat(captureData.startTime, "\n- **End Time**: ").concat(captureData.endTime, "\n- **Duration**: 30 seconds\n- **Agent**: ").concat(captureData.runtime.characterName, " (").concat(captureData.runtime.agentId, ")\n\n## Camera Information\n- **Name**: ").concat(((_a = captureData.camera) === null || _a === void 0 ? void 0 : _a.name) || 'Unknown', "\n- **ID**: ").concat(((_b = captureData.camera) === null || _b === void 0 ? void 0 : _b.id) || 'Unknown', "\n\n## Capture Statistics\n- **Total Frames Captured**: ").concat(captureCount, "\n- **Total Scene Changes**: ").concat(captureData.statistics.totalSceneChanges, "\n- **Average Change Percentage**: ").concat(captureData.statistics.averageChangePercentage.toFixed(2), "%\n- **Total Objects Detected**: ").concat(captureData.statistics.totalObjectsDetected, "\n- **Total People Detected**: ").concat(captureData.statistics.totalPeopleDetected, "\n\n## Object Type Distribution\n").concat(Object.entries(captureData.statistics.objectTypeCounts)
+                                    .sort(function (_a, _b) {
+                                    var a = _a[1];
+                                    var b = _b[1];
+                                    return b - a;
+                                })
+                                    .map(function (_a) {
+                                    var type = _a[0], count = _a[1];
+                                    return "- **".concat(type, "**: ").concat(count, " detections");
+                                })
+                                    .join('\n') || '- No objects detected', "\n\n## Pose Distribution\n").concat(Object.entries(captureData.statistics.poseCounts)
+                                    .map(function (_a) {
+                                    var pose = _a[0], count = _a[1];
+                                    return "- **".concat(pose, "**: ").concat(count, " detections");
+                                })
+                                    .join('\n') || '- No people detected', "\n\n## Sample Scene Descriptions\n").concat(captureData.captures
+                                    .filter(function (c) { var _a; return (_a = c.scene) === null || _a === void 0 ? void 0 : _a.description; })
+                                    .slice(0, 5)
+                                    .map(function (c, _i) {
+                                    var _a;
+                                    return "### Capture ".concat(c.index, " (").concat(c.elapsedMs, "ms)\n\"").concat(c.scene.description, "\"\n- Change: ").concat((_a = c.scene.changePercentage) === null || _a === void 0 ? void 0 : _a.toFixed(1), "%\n- Objects: ").concat(c.scene.objectCount, "\n- People: ").concat(c.scene.peopleCount);
+                                })
+                                    .join('\n\n'), "\n\n## Files Generated\n- `vision-capture-summary.json` - Complete capture data\n- `vision-capture-report.md` - This report\n- `capture-XXX.jpg` - Individual captured images\n- `scene-XXX.json` - Detailed scene data for each capture\n");
+                                return [4 /*yield*/, fs.writeFile(reportPath, report)];
+                            case 6:
+                                _c.sent();
+                                console.log('\n✅ Vision capture test completed!');
+                                console.log('📊 Capture Summary:');
+                                console.log("   - Total frames: ".concat(captureCount));
+                                console.log("   - Scene changes: ".concat(captureData.statistics.totalSceneChanges));
+                                console.log("   - Average change: ".concat(captureData.statistics.averageChangePercentage.toFixed(2), "%"));
+                                console.log("   - Objects detected: ".concat(captureData.statistics.totalObjectsDetected));
+                                console.log("   - People detected: ".concat(captureData.statistics.totalPeopleDetected));
+                                console.log("\n\uD83D\uDCC1 Results saved to: ".concat(sessionDir));
+                                console.log('   - Summary: vision-capture-summary.json');
+                                console.log('   - Report: vision-capture-report.md');
+                                console.log('   - Images: capture-XXX.jpg');
+                                console.log('   - Scene data: scene-XXX.json');
+                                console.log('✅ Vision capture log test PASSED');
+                                return [2 /*return*/];
+                        }
+                    });
+                }); },
+            },
+        ];
+    }
+    return VisionCaptureLogTestSuite;
+}());
+exports.VisionCaptureLogTestSuite = VisionCaptureLogTestSuite;
+exports.default = new VisionCaptureLogTestSuite();
+//# sourceMappingURL=vision-capture-log.js.map

package/dist/tests/e2e/vision-capture-log.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision-capture-log.js","sourceRoot":"","sources":["../../../src/tests/e2e/vision-capture-log.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,gCAAkC;AAClC,2BAA6B;AAE7B;IAAA;QAAA,iBA6SC;QA5SC,SAAI,GAAG,2BAA2B,CAAC;QACnC,gBAAW,GAAG,mEAAmE,CAAC;QAElF,UAAK,GAAG;YACN;gBACE,IAAI,EAAE,kDAAkD;gBACxD,EAAE,EAAE,UAAO,OAAsB;;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;gCAEnD,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAClE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;gCAClD,CAAC;gCAED,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC;oCAC9B,OAAO,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;oCACpE,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;oCAC9D,sBAAO;gCACT,CAAC;gCAGK,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;gCAC3C,UAAU,GAAG,IAAI,CAAC,IAAI,CAC1B,OAAO,EACP,yBAAkB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAE,CACnE,CAAC;gCACF,qBAAM,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAA;;gCAA/C,SAA+C,CAAC;gCAEhD,OAAO,CAAC,GAAG,CAAC,wCAA4B,UAAU,CAAE,CAAC,CAAC;gCAGhD,WAAW,GAAG;oCAClB,SAAS,EAAE,yBAAkB,IAAI,CAAC,GAAG,EAAE,CAAE;oCACzC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oCACnC,OAAO,EAAE,EAAE;oCACX,OAAO,EAAE;wCACP,OAAO,EAAE,OAAO,CAAC,OAAO;wCACxB,aAAa,EAAE,OAAO,CAAC,SAAS,CAAC,IAAI;qCACtC;oCACD,MAAM,EAAE,aAAa,CAAC,aAAa,EAAE;oCACrC,QAAQ,EAAE,EAAW;oCACrB,UAAU,EAAE;wCACV,WAAW,EAAE,CAAC;wCACd,iBAAiB,EAAE,CAAC;wCACpB,oBAAoB,EAAE,CAAC;wCACvB,mBAAmB,EAAE,CAAC;wCACtB,uBAAuB,EAAE,CAAC;wCAC1B,gBAAgB,EAAE,EAA4B;wCAC9C,UAAU,EAAE,EAA4B;qCACzC;iCACF,CAAC;gCAGI,eAAe,GAAG,GAAG,CAAC;gCACtB,aAAa,GAAG,KAAK,CAAC;gCACtB,gBAAgB,GAAG,aAAa,GAAG,eAAe,CAAC;gCAEzD,OAAO,CAAC,GAAG,CACT,mDAAkC,eAAe,oBAAU,aAAa,GAAG,IAAI,gBAAa,CAC7F,CAAC;gCACF,OAAO,CAAC,GAAG,CAAC,gCAAyB,gBAAgB,CAAE,CAAC,CAAC;gCAErD,YAAY,GAAG,CAAC,CAAC;gCACjB,qBAAqB,GAAG,CAAC,CAAC;gCAC1B,kBAAkB,GAAG,CAAC,CAAC;gCAErB,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;;;;;;gDAGrB,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;;;;gDAIpB,qBAAM,aAAa,CAAC,mBAAmB,EAAE,EAAA;;gDAAjD,KAAK,GAAG,SAAyC;gDACzC,qBAAM,aAAa,CAAC,eAAe,EAAE,EAAA;;gDAA7C,KAAK,GAAG,SAAqC;gDAG/B,qBAAM,aAAa,CAAC,YAAY,EAAE,EAAA;;gDAAhD,WAAW,GAAG,SAAkC;gDAClD,WAAW,GAAkB,IAAI,CAAC;gDACtC,IAAI,WAAW,EAAE,CAAC;oDAChB,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gDAC/C,CAAC;gDAEK,OAAO,GAAG;oDACd,KAAK,EAAE,YAAY;oDACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oDACnC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;oDACvB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;oDACjC,KAAK,EAAE,KAAK;wDACV,CAAC,CAAC;4DACE,WAAW,EAAE,KAAK,CAAC,WAAW;4DAC9B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;4DACxC,YAAY,EAAE,KAAK,CAAC,YAAY;4DAChC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;4DACjC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;4DAChC,OAAO,EAAE,KAAK,CAAC,OAAO;4DACtB,MAAM,EAAE,KAAK,CAAC,MAAM;yDACrB;wDACH,CAAC,CAAC,IAAI;oDACR,KAAK,EAAE,KAAK;wDACV,CAAC,CAAC;4DACE,KAAK,EAAE,KAAK,CAAC,KAAK;4DAClB,MAAM,EAAE,KAAK,CAAC,MAAM;4DACpB,MAAM,EAAE,KAAK,CAAC,MAAM;4DACpB,SAAS,EAAE,KAAK,CAAC,SAAS;yDAC3B;wDACH,CAAC,CAAC,IAAI;oDACR,WAAW,EAAE,WAAW,aAAX,WAAW,cAAX,WAAW,GAAI,IAAI;iDACjC,CAAC;gDAEF,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;qDAG/B,WAAW,EAAX,wBAAW;gDACP,SAAS,GAAG,IAAI,CAAC,IAAI,CACzB,UAAU,EACV,kBAAW,MAAM,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,SAAM,CACvD,CAAC;gDACF,qBAAM,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,EAAA;;gDAAjE,SAAiE,CAAC;;;qDAIhE,KAAK,EAAL,wBAAK;gDACP,WAAW,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC;gDACrC,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;oDACvB,WAAW,CAAC,UAAU,CAAC,iBAAiB,EAAE,CAAC;gDAC7C,CAAC;gDACD,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;oDAC3B,qBAAqB,IAAI,KAAK,CAAC,gBAAgB,CAAC;gDAClD,CAAC;gDACD,WAAW,CAAC,UAAU,CAAC,oBAAoB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gDACpE,WAAW,CAAC,UAAU,CAAC,mBAAmB,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gDAElE,qBAAqB;gDACrB,WAA+B,EAAb,KAAA,KAAK,CAAC,OAAO,EAAb,cAAa,EAAb,IAAa,EAAE,CAAC;oDAAvB,GAAG;oDACZ,WAAW,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC;wDAC/C,CAAC,WAAW,CAAC,UAAU,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gDACjE,CAAC;gDAED,cAAc;gDACd,WAAiC,EAAZ,KAAA,KAAK,CAAC,MAAM,EAAZ,cAAY,EAAZ,IAAY,EAAE,CAAC;oDAAzB,MAAM;oDACf,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;wDAC5C,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gDAC9D,CAAC;gDAGK,YAAY,GAAG,IAAI,CAAC,IAAI,CAC5B,UAAU,EACV,gBAAS,MAAM,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,UAAO,CACtD,CAAC;gDACF,qBAAM,EAAE,CAAC,SAAS,CAChB,YAAY,EACZ,IAAI,CAAC,SAAS,CACZ;wDACE,OAAO,EAAE,YAAY;wDACrB,SAAS,EAAE,OAAO,CAAC,SAAS;wDAC5B,KAAK,OAAA;wDACL,KAAK,EAAE,KAAK;4DACV,CAAC,CAAC;gEACE,KAAK,EAAE,KAAK,CAAC,KAAK;gEAClB,MAAM,EAAE,KAAK,CAAC,MAAM;gEACpB,MAAM,EAAE,KAAK,CAAC,MAAM;6DACrB;4DACH,CAAC,CAAC,IAAI;qDACT,EACD,IAAI,EACJ,CAAC,CACF,CACF,EAAA;;gDAlBD,SAkBC,CAAC;;;gDAGJ,YAAY,EAAE,CAAC;gDAGT,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC;gDACnE,IAAI,cAAc,GAAG,kBAAkB,IAAI,cAAc,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oDACpE,kBAAkB,GAAG,cAAc,CAAC;oDAC9B,QAAQ,GAAG,CAAC,cAAc,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC;oDAC7C,OAAO,CAAC,GAAG,CACT,sBAAe,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAM,cAAc,8BAAoB,YAAY,YAAS,CAChG,CAAC;oDAEF,IAAI,KAAK,EAAE,CAAC;wDACV,OAAO,CAAC,GAAG,CAAC,4BAAoB,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,UAAM,CAAC,CAAC;wDAC1E,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4DACxD,OAAO,CAAC,GAAG,CACT,wBAAiB,KAAK,CAAC,OAAO,CAAC,MAAM,uBAAa,KAAK,CAAC,MAAM,CAAC,MAAM,YAAS,CAC/E,CAAC;wDACJ,CAAC;oDACH,CAAC;gDACH,CAAC;;;;gDAED,OAAO,CAAC,KAAK,CAAC,6BAAsB,YAAY,MAAG,EAAE,OAAK,CAAC,CAAC;gDAC5D,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC;oDACxB,KAAK,EAAE,YAAY;oDACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oDACnC,KAAK,EAAE,OAAK,YAAY,KAAK,CAAC,CAAC,CAAC,OAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAK,CAAC;iDAC9D,CAAC,CAAC;gDACH,YAAY,EAAE,CAAC;;;gDAIX,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gDAC5B,eAAe,GAAG,cAAc,GAAG,gBAAgB,CAAC;gDACpD,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,GAAG,eAAe,CAAC,CAAC;qDAE5D,CAAA,QAAQ,GAAG,CAAC,CAAA,EAAZ,yBAAY;gDACd,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,EAA7B,CAA6B,CAAC,EAAA;;gDAA7D,SAA6D,CAAC;;;;;;;;qCA3I3D,CAAA,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,aAAa,CAAA;;;;;;gCA+I7C,sBAAsB;gCACtB,WAAW,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gCAC/C,WAAW,CAAC,UAAU,CAAC,uBAAuB;oCAC5C,WAAW,CAAC,UAAU,CAAC,WAAW,GAAG,CAAC;wCACpC,CAAC,CAAC,qBAAqB,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW;wCAC5D,CAAC,CAAC,CAAC,CAAC;gCAGF,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAAC;gCACzE,qBAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAA;;gCAArE,SAAqE,CAAC;gCAGhE,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAC;gCAC/D,MAAM,GAAG,+EAGH,WAAW,CAAC,SAAS,iCACrB,WAAW,CAAC,SAAS,+BACvB,WAAW,CAAC,OAAO,wDAEtB,WAAW,CAAC,OAAO,CAAC,aAAa,eAAK,WAAW,CAAC,OAAO,CAAC,OAAO,qDAGlE,CAAA,MAAA,WAAW,CAAC,MAAM,0CAAE,IAAI,KAAI,SAAS,yBACvC,CAAA,MAAA,WAAW,CAAC,MAAM,0CAAE,EAAE,KAAI,SAAS,qEAGhB,YAAY,0CACd,WAAW,CAAC,UAAU,CAAC,iBAAiB,gDAClC,WAAW,CAAC,UAAU,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,8CAC5D,WAAW,CAAC,UAAU,CAAC,oBAAoB,4CAC5C,WAAW,CAAC,UAAU,CAAC,mBAAmB,8CAIvE,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,gBAAgB,CAAC;qCACpD,IAAI,CAAC,UAAC,EAAK,EAAE,EAAK;wCAAT,CAAC,QAAA;wCAAM,CAAC,QAAA;oCAAM,OAAA,CAAC,GAAG,CAAC;gCAAL,CAAK,CAAC;qCAC7B,GAAG,CAAC,UAAC,EAAa;wCAAZ,IAAI,QAAA,EAAE,KAAK,QAAA;oCAAM,OAAA,cAAO,IAAI,iBAAO,KAAK,gBAAa;gCAApC,CAAoC,CAAC;qCAC5D,IAAI,CAAC,IAAI,CAAC,IAAI,uBAAuB,uCAKxC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC;qCAC9C,GAAG,CAAC,UAAC,EAAa;wCAAZ,IAAI,QAAA,EAAE,KAAK,QAAA;oCAAM,OAAA,cAAO,IAAI,iBAAO,KAAK,gBAAa;gCAApC,CAAoC,CAAC;qCAC5D,IAAI,CAAC,IAAI,CAAC,IAAI,sBAAsB,+CAIvC,WAAW,CAAC,QAAQ;qCACnB,MAAM,CAAC,UAAC,CAAC,YAAK,OAAA,MAAA,CAAC,CAAC,KAAK,0CAAE,WAAW,CAAA,EAAA,CAAC;qCACnC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;qCACX,GAAG,CACF,UAAC,CAAC,EAAE,EAAE;;oCAAK,OAAA,sBAAe,CAAC,CAAC,KAAK,eAAK,CAAC,CAAC,SAAS,oBAClD,CAAC,CAAC,KAAK,CAAC,WAAW,2BACV,MAAA,CAAC,CAAC,KAAK,CAAC,gBAAgB,0CAAE,OAAO,CAAC,CAAC,CAAC,2BACnC,CAAC,CAAC,KAAK,CAAC,WAAW,yBACpB,CAAC,CAAC,KAAK,CAAC,WAAW,CAAE,CAAA;iCAAA,CAC9B;qCACA,IAAI,CAAC,MAAM,CAAC,+OAOd,CAAC;gCAEM,qBAAM,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,EAAA;;gCAAtC,SAAsC,CAAC;gCAEvC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;gCAClD,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gCACnC,OAAO,CAAC,GAAG,CAAC,6BAAsB,YAAY,CAAE,CAAC,CAAC;gCAClD,OAAO,CAAC,GAAG,CAAC,8BAAuB,WAAW,CAAC,UAAU,CAAC,iBAAiB,CAAE,CAAC,CAAC;gCAC/E,OAAO,CAAC,GAAG,CACT,+BAAwB,WAAW,CAAC,UAAU,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAG,CACrF,CAAC;gCACF,OAAO,CAAC,GAAG,CAAC,iCAA0B,WAAW,CAAC,UAAU,CAAC,oBAAoB,CAAE,CAAC,CAAC;gCACrF,OAAO,CAAC,GAAG,CAAC,gCAAyB,WAAW,CAAC,UAAU,CAAC,mBAAmB,CAAE,CAAC,CAAC;gCACnF,OAAO,CAAC,GAAG,CAAC,2CAA0B,UAAU,CAAE,CAAC,CAAC;gCACpD,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;gCACzD,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;gCACrD,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;gCAC5C,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;gCAE/C,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;;;;qBACjD;aACF;SACF,CAAC;IACJ,CAAC;IAAD,gCAAC;AAAD,CAAC,AA7SD,IA6SC;AA7SY,8DAAyB;AA+StC,kBAAe,IAAI,yBAAyB,EAAE,CAAC","sourcesContent":["import type { TestSuite, IAgentRuntime } from '@elizaos/core';\nimport { VisionService } from '../../service';\nimport * as fs from 'fs/promises';\nimport * as path from 'path';\n\nexport class VisionCaptureLogTestSuite implements TestSuite {\n  name = 'plugin-vision-capture-log';\n  description = 'Captures 30 seconds of vision data and saves to logs for analysis';\n\n  tests = [\n    {\n      name: 'Should capture and log 30 seconds of vision data',\n      fn: async (runtime: IAgentRuntime) => {\n        console.log('Starting 30-second vision capture test...');\n\n        const visionService = runtime.getService<VisionService>('VISION');\n        if (!visionService) {\n          throw new Error('Vision service not available');\n        }\n\n        if (!visionService.isActive()) {\n          console.warn('⚠️  Vision service not active - no camera available');\n          console.log('   Cannot capture vision data without a camera');\n          return;\n        }\n\n        // Create logs directory if it doesn't exist\n        const logsDir = path.join(process.cwd(), 'logs');\n        const sessionDir = path.join(\n          logsDir,\n          `vision-capture-${new Date().toISOString().replace(/[:.]/g, '-')}`\n        );\n        await fs.mkdir(sessionDir, { recursive: true });\n\n        console.log(`✓ Created log directory: ${sessionDir}`);\n\n        // Initialize capture data\n        const captureData = {\n          sessionId: `vision-capture-${Date.now()}`,\n          startTime: new Date().toISOString(),\n          endTime: '',\n          runtime: {\n            agentId: runtime.agentId,\n            characterName: runtime.character.name,\n          },\n          camera: visionService.getCameraInfo(),\n          captures: [] as any[],\n          statistics: {\n            totalFrames: 0,\n            totalSceneChanges: 0,\n            totalObjectsDetected: 0,\n            totalPeopleDetected: 0,\n            averageChangePercentage: 0,\n            objectTypeCounts: {} as Record<string, number>,\n            poseCounts: {} as Record<string, number>,\n          },\n        };\n\n        // Capture interval - every 500ms for 30 seconds = 60 captures\n        const captureInterval = 500;\n        const totalDuration = 30000;\n        const expectedCaptures = totalDuration / captureInterval;\n\n        console.log(\n          `📸 Capturing vision data every ${captureInterval}ms for ${totalDuration / 1000} seconds...`\n        );\n        console.log(`   Expected captures: ${expectedCaptures}`);\n\n        let captureCount = 0;\n        let totalChangePercentage = 0;\n        let lastLoggedProgress = 0;\n\n        const startTime = Date.now();\n\n        while (Date.now() - startTime < totalDuration) {\n          const captureStartTime = Date.now();\n\n          try {\n            // Get current scene description\n            const scene = await visionService.getSceneDescription();\n            const frame = await visionService.getCurrentFrame();\n\n            // Capture image as base64\n            const imageBuffer = await visionService.captureImage();\n            let imageBase64: string | null = null;\n            if (imageBuffer) {\n              imageBase64 = imageBuffer.toString('base64');\n            }\n\n            const capture = {\n              index: captureCount,\n              timestamp: new Date().toISOString(),\n              timestampMs: Date.now(),\n              elapsedMs: Date.now() - startTime,\n              scene: scene\n                ? {\n                    description: scene.description,\n                    changePercentage: scene.changePercentage,\n                    sceneChanged: scene.sceneChanged,\n                    objectCount: scene.objects.length,\n                    peopleCount: scene.people.length,\n                    objects: scene.objects,\n                    people: scene.people,\n                  }\n                : null,\n              frame: frame\n                ? {\n                    width: frame.width,\n                    height: frame.height,\n                    format: frame.format,\n                    timestamp: frame.timestamp,\n                  }\n                : null,\n              imageBase64: imageBase64 ?? null,\n            };\n\n            captureData.captures.push(capture);\n\n            // Save full image separately\n            if (imageBase64) {\n              const imagePath = path.join(\n                sessionDir,\n                `capture-${String(captureCount).padStart(3, '0')}.jpg`\n              );\n              await fs.writeFile(imagePath, Buffer.from(imageBase64, 'base64'));\n            }\n\n            // Update statistics\n            if (scene) {\n              captureData.statistics.totalFrames++;\n              if (scene.sceneChanged) {\n                captureData.statistics.totalSceneChanges++;\n              }\n              if (scene.changePercentage) {\n                totalChangePercentage += scene.changePercentage;\n              }\n              captureData.statistics.totalObjectsDetected += scene.objects.length;\n              captureData.statistics.totalPeopleDetected += scene.people.length;\n\n              // Count object types\n              for (const obj of scene.objects) {\n                captureData.statistics.objectTypeCounts[obj.type] =\n                  (captureData.statistics.objectTypeCounts[obj.type] || 0) + 1;\n              }\n\n              // Count poses\n              for (const person of scene.people) {\n                captureData.statistics.poseCounts[person.pose] =\n                  (captureData.statistics.poseCounts[person.pose] || 0) + 1;\n              }\n\n              // Save detailed scene log\n              const sceneLogPath = path.join(\n                sessionDir,\n                `scene-${String(captureCount).padStart(3, '0')}.json`\n              );\n              await fs.writeFile(\n                sceneLogPath,\n                JSON.stringify(\n                  {\n                    capture: captureCount,\n                    timestamp: capture.timestamp,\n                    scene,\n                    frame: frame\n                      ? {\n                          width: frame.width,\n                          height: frame.height,\n                          format: frame.format,\n                        }\n                      : null,\n                  },\n                  null,\n                  2\n                )\n              );\n            }\n\n            captureCount++;\n\n            // Progress logging every 5 seconds\n            const elapsedSeconds = Math.floor((Date.now() - startTime) / 1000);\n            if (elapsedSeconds > lastLoggedProgress && elapsedSeconds % 5 === 0) {\n              lastLoggedProgress = elapsedSeconds;\n              const progress = (elapsedSeconds / 30) * 100;\n              console.log(\n                `  Progress: ${progress.toFixed(0)}% (${elapsedSeconds}/30s) - Captured ${captureCount} frames`\n              );\n\n              if (scene) {\n                console.log(`    Last scene: \"${scene.description.substring(0, 60)}...\"`);\n                if (scene.objects.length > 0 || scene.people.length > 0) {\n                  console.log(\n                    `    Detected: ${scene.objects.length} objects, ${scene.people.length} people`\n                  );\n                }\n              }\n            }\n          } catch (error) {\n            console.error(`  Error in capture ${captureCount}:`, error);\n            captureData.captures.push({\n              index: captureCount,\n              timestamp: new Date().toISOString(),\n              error: error instanceof Error ? error.message : String(error),\n            });\n            captureCount++;\n          }\n\n          // Wait for next capture interval\n          const captureEndTime = Date.now();\n          const captureDuration = captureEndTime - captureStartTime;\n          const waitTime = Math.max(0, captureInterval - captureDuration);\n\n          if (waitTime > 0) {\n            await new Promise((resolve) => setTimeout(resolve, waitTime));\n          }\n        }\n\n        // Finalize statistics\n        captureData.endTime = new Date().toISOString();\n        captureData.statistics.averageChangePercentage =\n          captureData.statistics.totalFrames > 0\n            ? totalChangePercentage / captureData.statistics.totalFrames\n            : 0;\n\n        // Save main capture log\n        const mainLogPath = path.join(sessionDir, 'vision-capture-summary.json');\n        await fs.writeFile(mainLogPath, JSON.stringify(captureData, null, 2));\n\n        // Generate markdown report\n        const reportPath = path.join(sessionDir, 'vision-capture-report.md');\n        const report = `# Vision Capture Report\n\n## Session Information\n- **Session ID**: ${captureData.sessionId}\n- **Start Time**: ${captureData.startTime}\n- **End Time**: ${captureData.endTime}\n- **Duration**: 30 seconds\n- **Agent**: ${captureData.runtime.characterName} (${captureData.runtime.agentId})\n\n## Camera Information\n- **Name**: ${captureData.camera?.name || 'Unknown'}\n- **ID**: ${captureData.camera?.id || 'Unknown'}\n\n## Capture Statistics\n- **Total Frames Captured**: ${captureCount}\n- **Total Scene Changes**: ${captureData.statistics.totalSceneChanges}\n- **Average Change Percentage**: ${captureData.statistics.averageChangePercentage.toFixed(2)}%\n- **Total Objects Detected**: ${captureData.statistics.totalObjectsDetected}\n- **Total People Detected**: ${captureData.statistics.totalPeopleDetected}\n\n## Object Type Distribution\n${\n  Object.entries(captureData.statistics.objectTypeCounts)\n    .sort(([, a], [, b]) => b - a)\n    .map(([type, count]) => `- **${type}**: ${count} detections`)\n    .join('\\n') || '- No objects detected'\n}\n\n## Pose Distribution\n${\n  Object.entries(captureData.statistics.poseCounts)\n    .map(([pose, count]) => `- **${pose}**: ${count} detections`)\n    .join('\\n') || '- No people detected'\n}\n\n## Sample Scene Descriptions\n${captureData.captures\n  .filter((c) => c.scene?.description)\n  .slice(0, 5)\n  .map(\n    (c, _i) => `### Capture ${c.index} (${c.elapsedMs}ms)\n\"${c.scene.description}\"\n- Change: ${c.scene.changePercentage?.toFixed(1)}%\n- Objects: ${c.scene.objectCount}\n- People: ${c.scene.peopleCount}`\n  )\n  .join('\\n\\n')}\n\n## Files Generated\n- \\`vision-capture-summary.json\\` - Complete capture data\n- \\`vision-capture-report.md\\` - This report\n- \\`capture-XXX.jpg\\` - Individual captured images\n- \\`scene-XXX.json\\` - Detailed scene data for each capture\n`;\n\n        await fs.writeFile(reportPath, report);\n\n        console.log('\\n✅ Vision capture test completed!');\n        console.log('📊 Capture Summary:');\n        console.log(`   - Total frames: ${captureCount}`);\n        console.log(`   - Scene changes: ${captureData.statistics.totalSceneChanges}`);\n        console.log(\n          `   - Average change: ${captureData.statistics.averageChangePercentage.toFixed(2)}%`\n        );\n        console.log(`   - Objects detected: ${captureData.statistics.totalObjectsDetected}`);\n        console.log(`   - People detected: ${captureData.statistics.totalPeopleDetected}`);\n        console.log(`\\n📁 Results saved to: ${sessionDir}`);\n        console.log('   - Summary: vision-capture-summary.json');\n        console.log('   - Report: vision-capture-report.md');\n        console.log('   - Images: capture-XXX.jpg');\n        console.log('   - Scene data: scene-XXX.json');\n\n        console.log('✅ Vision capture log test PASSED');\n      },\n    },\n  ];\n}\n\nexport default new VisionCaptureLogTestSuite();\n"]}

package/dist/tests/e2e/vision-runtime.d.ts

@@ -0,0 +1,11 @@
+import type { TestSuite } from '@elizaos/core';
+export declare class VisionRuntimeTestSuite implements TestSuite {
+    name: string;
+    description: string;
+    tests: {
+        name: string;
+        fn: (runtime: any) => Promise<void>;
+    }[];
+}
+declare const _default: VisionRuntimeTestSuite;
+export default _default;