@elizaos/plugin-vision 1.2.1

package/dist/tests/test-pattern-generator.js

@@ -0,0 +1,191 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestPatternGenerator = void 0;
+var sharp_1 = require("sharp");
+var path = require("path");
+var fs = require("fs/promises");
+var core_1 = require("@elizaos/core");
+var TestPatternGenerator = /** @class */ (function () {
+    function TestPatternGenerator() {
+    }
+    /**
+     * Generate a test pattern with numbers in each quadrant and center
+     */
+    TestPatternGenerator.generateQuadrantPattern = function (config) {
+        return __awaiter(this, void 0, void 0, function () {
+            var width, height, _a, backgroundColor, _b, textColor, _c, fontSize, _d, includeGrid, _e, includeTimestamp, _f, displayIndex, svg, buffer;
+            return __generator(this, function (_g) {
+                switch (_g.label) {
+                    case 0:
+                        width = config.width, height = config.height, _a = config.backgroundColor, backgroundColor = _a === void 0 ? '#ffffff' : _a, _b = config.textColor, textColor = _b === void 0 ? '#000000' : _b, _c = config.fontSize, fontSize = _c === void 0 ? 48 : _c, _d = config.includeGrid, includeGrid = _d === void 0 ? true : _d, _e = config.includeTimestamp, includeTimestamp = _e === void 0 ? true : _e, _f = config.displayIndex, displayIndex = _f === void 0 ? 0 : _f;
+                        svg = "\n      <svg width=\"".concat(width, "\" height=\"").concat(height, "\" xmlns=\"http://www.w3.org/2000/svg\">\n        <!-- Background -->\n        <rect width=\"").concat(width, "\" height=\"").concat(height, "\" fill=\"").concat(backgroundColor, "\"/>\n        \n        ").concat(includeGrid ? this.generateGrid(width, height) : '', "\n        \n        <!-- Quadrant dividers -->\n        <line x1=\"").concat(width / 2, "\" y1=\"0\" x2=\"").concat(width / 2, "\" y2=\"").concat(height, "\" stroke=\"#cccccc\" stroke-width=\"2\"/>\n        <line x1=\"0\" y1=\"").concat(height / 2, "\" x2=\"").concat(width, "\" y2=\"").concat(height / 2, "\" stroke=\"#cccccc\" stroke-width=\"2\"/>\n        \n        <!-- Quadrant numbers -->\n        <text x=\"").concat(width / 4, "\" y=\"").concat(height / 4, "\" font-family=\"Arial\" font-size=\"").concat(fontSize, "\" fill=\"").concat(textColor, "\" text-anchor=\"middle\" dominant-baseline=\"middle\">1</text>\n        <text x=\"").concat((3 * width) / 4, "\" y=\"").concat(height / 4, "\" font-family=\"Arial\" font-size=\"").concat(fontSize, "\" fill=\"").concat(textColor, "\" text-anchor=\"middle\" dominant-baseline=\"middle\">2</text>\n        <text x=\"").concat(width / 4, "\" y=\"").concat((3 * height) / 4, "\" font-family=\"Arial\" font-size=\"").concat(fontSize, "\" fill=\"").concat(textColor, "\" text-anchor=\"middle\" dominant-baseline=\"middle\">3</text>\n        <text x=\"").concat((3 * width) / 4, "\" y=\"").concat((3 * height) / 4, "\" font-family=\"Arial\" font-size=\"").concat(fontSize, "\" fill=\"").concat(textColor, "\" text-anchor=\"middle\" dominant-baseline=\"middle\">4</text>\n        \n        <!-- Center number -->\n        <circle cx=\"").concat(width / 2, "\" cy=\"").concat(height / 2, "\" r=\"").concat(fontSize, "\" fill=\"#ff0000\" opacity=\"0.3\"/>\n        <text x=\"").concat(width / 2, "\" y=\"").concat(height / 2, "\" font-family=\"Arial\" font-size=\"").concat(fontSize, "\" fill=\"").concat(textColor, "\" text-anchor=\"middle\" dominant-baseline=\"middle\">5</text>\n        \n        <!-- Display info -->\n        <text x=\"20\" y=\"30\" font-family=\"Arial\" font-size=\"16\" fill=\"").concat(textColor, "\">Display ").concat(displayIndex, "</text>\n        <text x=\"20\" y=\"50\" font-family=\"Arial\" font-size=\"16\" fill=\"").concat(textColor, "\">").concat(width, "x").concat(height, "</text>\n        \n        ").concat(includeTimestamp ? "<text x=\"20\" y=\"70\" font-family=\"Arial\" font-size=\"16\" fill=\"".concat(textColor, "\">Time: ").concat(new Date().toISOString(), "</text>") : '', "\n        \n        <!-- Corner markers -->\n        <circle cx=\"10\" cy=\"10\" r=\"5\" fill=\"#ff0000\"/>\n        <circle cx=\"").concat(width - 10, "\" cy=\"10\" r=\"5\" fill=\"#00ff00\"/>\n        <circle cx=\"10\" cy=\"").concat(height - 10, "\" r=\"5\" fill=\"#0000ff\"/>\n        <circle cx=\"").concat(width - 10, "\" cy=\"").concat(height - 10, "\" r=\"5\" fill=\"#ffff00\"/>\n      </svg>\n    ");
+                        return [4 /*yield*/, (0, sharp_1.default)(Buffer.from(svg)).png().toBuffer()];
+                    case 1:
+                        buffer = _g.sent();
+                        return [2 /*return*/, buffer];
+                }
+            });
+        });
+    };
+    /**
+     * Generate a complex test pattern with multiple text regions
+     */
+    TestPatternGenerator.generateComplexPattern = function (config) {
+        return __awaiter(this, void 0, void 0, function () {
+            var width, height, _a, backgroundColor, _b, textColor, _c, fontSize, _d, displayIndex, sampleTexts, svg, buffer;
+            return __generator(this, function (_e) {
+                switch (_e.label) {
+                    case 0:
+                        width = config.width, height = config.height, _a = config.backgroundColor, backgroundColor = _a === void 0 ? '#f0f0f0' : _a, _b = config.textColor, textColor = _b === void 0 ? '#000000' : _b, _c = config.fontSize, fontSize = _c === void 0 ? 24 : _c, _d = config.displayIndex, displayIndex = _d === void 0 ? 0 : _d;
+                        sampleTexts = [
+                            'The quick brown fox jumps over the lazy dog',
+                            'ABCDEFGHIJKLMNOPQRSTUVWXYZ',
+                            'abcdefghijklmnopqrstuvwxyz',
+                            '0123456789',
+                            '!@#$%^&*()_+-=[]{}|;:,.<>?',
+                        ];
+                        svg = "\n      <svg width=\"".concat(width, "\" height=\"").concat(height, "\" xmlns=\"http://www.w3.org/2000/svg\">\n        <!-- Background -->\n        <rect width=\"").concat(width, "\" height=\"").concat(height, "\" fill=\"").concat(backgroundColor, "\"/>\n        \n        <!-- Title -->\n        <text x=\"").concat(width / 2, "\" y=\"50\" font-family=\"Arial\" font-size=\"32\" fill=\"").concat(textColor, "\" text-anchor=\"middle\">Vision Test Pattern - Display ").concat(displayIndex, "</text>\n        \n        <!-- Text regions for OCR testing -->\n        ").concat(sampleTexts
+                            .map(function (text, i) { return "\n          <rect x=\"50\" y=\"".concat(150 + i * 80, "\" width=\"").concat(width - 100, "\" height=\"60\" fill=\"white\" stroke=\"#333\" stroke-width=\"1\"/>\n          <text x=\"70\" y=\"").concat(185 + i * 80, "\" font-family=\"Arial\" font-size=\"").concat(fontSize, "\" fill=\"").concat(textColor, "\">").concat(text, "</text>\n        "); })
+                            .join(''), "\n        \n        <!-- UI Elements -->\n        <rect x=\"50\" y=\"").concat(height - 200, "\" width=\"150\" height=\"40\" fill=\"#007bff\" rx=\"5\"/>\n        <text x=\"125\" y=\"").concat(height - 175, "\" font-family=\"Arial\" font-size=\"16\" fill=\"white\" text-anchor=\"middle\">Button</text>\n        \n        <rect x=\"220\" y=\"").concat(height - 200, "\" width=\"200\" height=\"30\" fill=\"white\" stroke=\"#333\" stroke-width=\"1\"/>\n        <text x=\"230\" y=\"").concat(height - 180, "\" font-family=\"Arial\" font-size=\"14\" fill=\"#666\">Input Field</text>\n        \n        <!-- Timestamp -->\n        <text x=\"").concat(width - 20, "\" y=\"").concat(height - 20, "\" font-family=\"Arial\" font-size=\"12\" fill=\"#666\" text-anchor=\"end\">").concat(new Date().toISOString(), "</text>\n      </svg>\n    ");
+                        return [4 /*yield*/, (0, sharp_1.default)(Buffer.from(svg)).png().toBuffer()];
+                    case 1:
+                        buffer = _e.sent();
+                        return [2 /*return*/, buffer];
+                }
+            });
+        });
+    };
+    /**
+     * Generate grid lines for the pattern
+     */
+    TestPatternGenerator.generateGrid = function (width, height, spacing) {
+        if (spacing === void 0) { spacing = 100; }
+        var lines = [];
+        // Vertical lines
+        for (var x = spacing; x < width; x += spacing) {
+            lines.push("<line x1=\"".concat(x, "\" y1=\"0\" x2=\"").concat(x, "\" y2=\"").concat(height, "\" stroke=\"#eeeeee\" stroke-width=\"1\"/>"));
+        }
+        // Horizontal lines
+        for (var y = spacing; y < height; y += spacing) {
+            lines.push("<line x1=\"0\" y1=\"".concat(y, "\" x2=\"").concat(width, "\" y2=\"").concat(y, "\" stroke=\"#eeeeee\" stroke-width=\"1\"/>"));
+        }
+        return lines.join('\n');
+    };
+    /**
+     * Save test pattern to file
+     */
+    TestPatternGenerator.savePattern = function (buffer, filename) {
+        return __awaiter(this, void 0, void 0, function () {
+            var outputDir, filepath;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        outputDir = path.join(process.cwd(), 'test-patterns');
+                        return [4 /*yield*/, fs.mkdir(outputDir, { recursive: true })];
+                    case 1:
+                        _a.sent();
+                        filepath = path.join(outputDir, filename);
+                        return [4 /*yield*/, fs.writeFile(filepath, buffer)];
+                    case 2:
+                        _a.sent();
+                        core_1.logger.info("[TestPatternGenerator] Saved test pattern to ".concat(filepath));
+                        return [2 /*return*/, filepath];
+                }
+            });
+        });
+    };
+    /**
+     * Generate patterns for all displays
+     */
+    TestPatternGenerator.generatePatternsForAllDisplays = function (displayCount) {
+        return __awaiter(this, void 0, void 0, function () {
+            var patterns, i, pattern;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        patterns = new Map();
+                        i = 0;
+                        _a.label = 1;
+                    case 1:
+                        if (!(i < displayCount)) return [3 /*break*/, 4];
+                        return [4 /*yield*/, this.generateQuadrantPattern({
+                                width: 1920,
+                                height: 1080,
+                                displayIndex: i,
+                                includeTimestamp: true,
+                            })];
+                    case 2:
+                        pattern = _a.sent();
+                        patterns.set(i, pattern);
+                        _a.label = 3;
+                    case 3:
+                        i++;
+                        return [3 /*break*/, 1];
+                    case 4: return [2 /*return*/, patterns];
+                }
+            });
+        });
+    };
+    /**
+     * Verify OCR results match expected quadrant numbers
+     */
+    TestPatternGenerator.verifyQuadrantNumbers = function (ocrText) {
+        var expectedNumbers = [1, 2, 3, 4, 5];
+        var foundNumbers = [];
+        // Extract all numbers from OCR text
+        var matches = ocrText.match(/\d+/g);
+        if (matches) {
+            matches.forEach(function (match) {
+                var num = parseInt(match, 10);
+                if (expectedNumbers.includes(num) && !foundNumbers.includes(num)) {
+                    foundNumbers.push(num);
+                }
+            });
+        }
+        var missingNumbers = expectedNumbers.filter(function (n) { return !foundNumbers.includes(n); });
+        return {
+            success: missingNumbers.length === 0,
+            foundNumbers: foundNumbers,
+            missingNumbers: missingNumbers,
+        };
+    };
+    return TestPatternGenerator;
+}());
+exports.TestPatternGenerator = TestPatternGenerator;
+//# sourceMappingURL=test-pattern-generator.js.map

package/dist/tests/test-pattern-generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-pattern-generator.js","sourceRoot":"","sources":["../../src/tests/test-pattern-generator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+BAA0B;AAC1B,2BAA6B;AAC7B,gCAAkC;AAClC,sCAAuC;AAavC;IAAA;IA0MA,CAAC;IAzMC;;OAEG;IACU,4CAAuB,GAApC,UAAqC,MAAyB;;;;;;wBAE1D,KAAK,GAQH,MAAM,MARH,EACL,MAAM,GAOJ,MAAM,OAPF,EACN,KAME,MAAM,gBANmB,EAA3B,eAAe,mBAAG,SAAS,KAAA,EAC3B,KAKE,MAAM,UALa,EAArB,SAAS,mBAAG,SAAS,KAAA,EACrB,KAIE,MAAM,SAJK,EAAb,QAAQ,mBAAG,EAAE,KAAA,EACb,KAGE,MAAM,YAHU,EAAlB,WAAW,mBAAG,IAAI,KAAA,EAClB,KAEE,MAAM,iBAFe,EAAvB,gBAAgB,mBAAG,IAAI,KAAA,EACvB,KACE,MAAM,aADQ,EAAhB,YAAY,mBAAG,CAAC,KAAA,CACP;wBAGL,GAAG,GAAG,+BACI,KAAK,yBAAa,MAAM,0GAErB,KAAK,yBAAa,MAAM,uBAAW,eAAe,qCAE/D,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,gFAGzC,KAAK,GAAG,CAAC,8BAAgB,KAAK,GAAG,CAAC,qBAAS,MAAM,qFAC1C,MAAM,GAAG,CAAC,qBAAS,KAAK,qBAAS,MAAM,GAAG,CAAC,wHAGnD,KAAK,GAAG,CAAC,oBAAQ,MAAM,GAAG,CAAC,kDAAoC,QAAQ,uBAAW,SAAS,gGAC3F,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,oBAAQ,MAAM,GAAG,CAAC,kDAAoC,QAAQ,uBAAW,SAAS,gGACjG,KAAK,GAAG,CAAC,oBAAQ,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,kDAAoC,QAAQ,uBAAW,SAAS,gGACjG,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,oBAAQ,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,kDAAoC,QAAQ,uBAAW,SAAS,6IAGpG,KAAK,GAAG,CAAC,qBAAS,MAAM,GAAG,CAAC,oBAAQ,QAAQ,sEAC/C,KAAK,GAAG,CAAC,oBAAQ,MAAM,GAAG,CAAC,kDAAoC,QAAQ,uBAAW,SAAS,qMAGvC,SAAS,wBAAa,YAAY,oGAClC,SAAS,gBAAK,KAAK,cAAI,MAAM,wCAE1F,gBAAgB,CAAC,CAAC,CAAC,gFAAgE,SAAS,sBAAW,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,YAAS,CAAC,CAAC,CAAC,EAAE,+IAIjI,KAAK,GAAG,EAAE,qFACF,MAAM,GAAG,EAAE,iEACnB,KAAK,GAAG,EAAE,qBAAS,MAAM,GAAG,EAAE,sDAE/C,CAAC;wBAGa,qBAAM,IAAA,eAAK,EAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAA;;wBAAvD,MAAM,GAAG,SAA8C;wBAE7D,sBAAO,MAAM,EAAC;;;;KACf;IAED;;OAEG;IACU,2CAAsB,GAAnC,UAAoC,MAAyB;;;;;;wBAEzD,KAAK,GAMH,MAAM,MANH,EACL,MAAM,GAKJ,MAAM,OALF,EACN,KAIE,MAAM,gBAJmB,EAA3B,eAAe,mBAAG,SAAS,KAAA,EAC3B,KAGE,MAAM,UAHa,EAArB,SAAS,mBAAG,SAAS,KAAA,EACrB,KAEE,MAAM,SAFK,EAAb,QAAQ,mBAAG,EAAE,KAAA,EACb,KACE,MAAM,aADQ,EAAhB,YAAY,mBAAG,CAAC,KAAA,CACP;wBAGL,WAAW,GAAG;4BAClB,6CAA6C;4BAC7C,4BAA4B;4BAC5B,4BAA4B;4BAC5B,YAAY;4BACZ,4BAA4B;yBAC7B,CAAC;wBAEI,GAAG,GAAG,+BACI,KAAK,yBAAa,MAAM,0GAErB,KAAK,yBAAa,MAAM,uBAAW,eAAe,uEAGtD,KAAK,GAAG,CAAC,uEAAqD,SAAS,qEAAwD,YAAY,uFAGpJ,WAAW;6BACV,GAAG,CACF,UAAC,IAAI,EAAE,CAAC,IAAK,OAAA,yCACG,GAAG,GAAG,CAAC,GAAG,EAAE,wBAAY,KAAK,GAAG,GAAG,gHACnC,GAAG,GAAG,CAAC,GAAG,EAAE,kDAAoC,QAAQ,uBAAW,SAAS,gBAAK,IAAI,sBACxG,EAHgB,CAGhB,CACE;6BACA,IAAI,CAAC,EAAE,CAAC,kFAGO,MAAM,GAAG,GAAG,qGACX,MAAM,GAAG,GAAG,kJAEZ,MAAM,GAAG,GAAG,6HACZ,MAAM,GAAG,GAAG,iJAGpB,KAAK,GAAG,EAAE,oBAAQ,MAAM,GAAG,EAAE,yFAAsE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,gCAEzI,CAAC;wBAEa,qBAAM,IAAA,eAAK,EAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAA;;wBAAvD,MAAM,GAAG,SAA8C;wBAE7D,sBAAO,MAAM,EAAC;;;;KACf;IAED;;OAEG;IACY,iCAAY,GAA3B,UAA4B,KAAa,EAAE,MAAc,EAAE,OAAqB;QAArB,wBAAA,EAAA,aAAqB;QAC9E,IAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,iBAAiB;QACjB,KAAK,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC;YAC9C,KAAK,CAAC,IAAI,CACR,qBAAa,CAAC,8BAAgB,CAAC,qBAAS,MAAM,+CAAuC,CACtF,CAAC;QACJ,CAAC;QAED,mBAAmB;QACnB,KAAK,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC;YAC/C,KAAK,CAAC,IAAI,CACR,8BAAoB,CAAC,qBAAS,KAAK,qBAAS,CAAC,+CAAuC,CACrF,CAAC;QACJ,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;OAEG;IACU,gCAAW,GAAxB,UAAyB,MAAc,EAAE,QAAgB;;;;;;wBACjD,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC,CAAC;wBAC5D,qBAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAA;;wBAA9C,SAA8C,CAAC;wBAEzC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;wBAChD,qBAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAA;;wBAApC,SAAoC,CAAC;wBAErC,aAAM,CAAC,IAAI,CAAC,uDAAgD,QAAQ,CAAE,CAAC,CAAC;wBACxE,sBAAO,QAAQ,EAAC;;;;KACjB;IAED;;OAEG;IACU,mDAA8B,GAA3C,UAA4C,YAAoB;;;;;;wBACxD,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;wBAElC,CAAC,GAAG,CAAC;;;6BAAE,CAAA,CAAC,GAAG,YAAY,CAAA;wBACd,qBAAM,IAAI,CAAC,uBAAuB,CAAC;gCACjD,KAAK,EAAE,IAAI;gCACX,MAAM,EAAE,IAAI;gCACZ,YAAY,EAAE,CAAC;gCACf,gBAAgB,EAAE,IAAI;6BACvB,CAAC,EAAA;;wBALI,OAAO,GAAG,SAKd;wBAEF,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;;;wBARO,CAAC,EAAE,CAAA;;4BAWrC,sBAAO,QAAQ,EAAC;;;;KACjB;IAED;;OAEG;IACI,0CAAqB,GAA5B,UAA6B,OAAe;QAK1C,IAAM,eAAe,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACxC,IAAM,YAAY,GAAa,EAAE,CAAC;QAElC,oCAAoC;QACpC,IAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,OAAO,CAAC,UAAC,KAAK;gBACpB,IAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAChC,IAAI,eAAe,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAM,cAAc,GAAG,eAAe,CAAC,MAAM,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAzB,CAAyB,CAAC,CAAC;QAEhF,OAAO;YACL,OAAO,EAAE,cAAc,CAAC,MAAM,KAAK,CAAC;YACpC,YAAY,cAAA;YACZ,cAAc,gBAAA;SACf,CAAC;IACJ,CAAC;IACH,2BAAC;AAAD,CAAC,AA1MD,IA0MC;AA1MY,oDAAoB","sourcesContent":["import sharp from 'sharp';\nimport * as path from 'path';\nimport * as fs from 'fs/promises';\nimport { logger } from '@elizaos/core';\n\nexport interface TestPatternConfig {\n  width: number;\n  height: number;\n  backgroundColor?: string;\n  textColor?: string;\n  fontSize?: number;\n  includeGrid?: boolean;\n  includeTimestamp?: boolean;\n  displayIndex?: number;\n}\n\nexport class TestPatternGenerator {\n  /**\n   * Generate a test pattern with numbers in each quadrant and center\n   */\n  static async generateQuadrantPattern(config: TestPatternConfig): Promise<Buffer> {\n    const {\n      width,\n      height,\n      backgroundColor = '#ffffff',\n      textColor = '#000000',\n      fontSize = 48,\n      includeGrid = true,\n      includeTimestamp = true,\n      displayIndex = 0,\n    } = config;\n\n    // Create SVG with test pattern\n    const svg = `\n      <svg width=\"${width}\" height=\"${height}\" xmlns=\"http://www.w3.org/2000/svg\">\n        <!-- Background -->\n        <rect width=\"${width}\" height=\"${height}\" fill=\"${backgroundColor}\"/>\n        \n        ${includeGrid ? this.generateGrid(width, height) : ''}\n        \n        <!-- Quadrant dividers -->\n        <line x1=\"${width / 2}\" y1=\"0\" x2=\"${width / 2}\" y2=\"${height}\" stroke=\"#cccccc\" stroke-width=\"2\"/>\n        <line x1=\"0\" y1=\"${height / 2}\" x2=\"${width}\" y2=\"${height / 2}\" stroke=\"#cccccc\" stroke-width=\"2\"/>\n        \n        <!-- Quadrant numbers -->\n        <text x=\"${width / 4}\" y=\"${height / 4}\" font-family=\"Arial\" font-size=\"${fontSize}\" fill=\"${textColor}\" text-anchor=\"middle\" dominant-baseline=\"middle\">1</text>\n        <text x=\"${(3 * width) / 4}\" y=\"${height / 4}\" font-family=\"Arial\" font-size=\"${fontSize}\" fill=\"${textColor}\" text-anchor=\"middle\" dominant-baseline=\"middle\">2</text>\n        <text x=\"${width / 4}\" y=\"${(3 * height) / 4}\" font-family=\"Arial\" font-size=\"${fontSize}\" fill=\"${textColor}\" text-anchor=\"middle\" dominant-baseline=\"middle\">3</text>\n        <text x=\"${(3 * width) / 4}\" y=\"${(3 * height) / 4}\" font-family=\"Arial\" font-size=\"${fontSize}\" fill=\"${textColor}\" text-anchor=\"middle\" dominant-baseline=\"middle\">4</text>\n        \n        <!-- Center number -->\n        <circle cx=\"${width / 2}\" cy=\"${height / 2}\" r=\"${fontSize}\" fill=\"#ff0000\" opacity=\"0.3\"/>\n        <text x=\"${width / 2}\" y=\"${height / 2}\" font-family=\"Arial\" font-size=\"${fontSize}\" fill=\"${textColor}\" text-anchor=\"middle\" dominant-baseline=\"middle\">5</text>\n        \n        <!-- Display info -->\n        <text x=\"20\" y=\"30\" font-family=\"Arial\" font-size=\"16\" fill=\"${textColor}\">Display ${displayIndex}</text>\n        <text x=\"20\" y=\"50\" font-family=\"Arial\" font-size=\"16\" fill=\"${textColor}\">${width}x${height}</text>\n        \n        ${includeTimestamp ? `<text x=\"20\" y=\"70\" font-family=\"Arial\" font-size=\"16\" fill=\"${textColor}\">Time: ${new Date().toISOString()}</text>` : ''}\n        \n        <!-- Corner markers -->\n        <circle cx=\"10\" cy=\"10\" r=\"5\" fill=\"#ff0000\"/>\n        <circle cx=\"${width - 10}\" cy=\"10\" r=\"5\" fill=\"#00ff00\"/>\n        <circle cx=\"10\" cy=\"${height - 10}\" r=\"5\" fill=\"#0000ff\"/>\n        <circle cx=\"${width - 10}\" cy=\"${height - 10}\" r=\"5\" fill=\"#ffff00\"/>\n      </svg>\n    `;\n\n    // Convert SVG to PNG\n    const buffer = await sharp(Buffer.from(svg)).png().toBuffer();\n\n    return buffer;\n  }\n\n  /**\n   * Generate a complex test pattern with multiple text regions\n   */\n  static async generateComplexPattern(config: TestPatternConfig): Promise<Buffer> {\n    const {\n      width,\n      height,\n      backgroundColor = '#f0f0f0',\n      textColor = '#000000',\n      fontSize = 24,\n      displayIndex = 0,\n    } = config;\n\n    // Sample text for OCR testing\n    const sampleTexts = [\n      'The quick brown fox jumps over the lazy dog',\n      'ABCDEFGHIJKLMNOPQRSTUVWXYZ',\n      'abcdefghijklmnopqrstuvwxyz',\n      '0123456789',\n      '!@#$%^&*()_+-=[]{}|;:,.<>?',\n    ];\n\n    const svg = `\n      <svg width=\"${width}\" height=\"${height}\" xmlns=\"http://www.w3.org/2000/svg\">\n        <!-- Background -->\n        <rect width=\"${width}\" height=\"${height}\" fill=\"${backgroundColor}\"/>\n        \n        <!-- Title -->\n        <text x=\"${width / 2}\" y=\"50\" font-family=\"Arial\" font-size=\"32\" fill=\"${textColor}\" text-anchor=\"middle\">Vision Test Pattern - Display ${displayIndex}</text>\n        \n        <!-- Text regions for OCR testing -->\n        ${sampleTexts\n          .map(\n            (text, i) => `\n          <rect x=\"50\" y=\"${150 + i * 80}\" width=\"${width - 100}\" height=\"60\" fill=\"white\" stroke=\"#333\" stroke-width=\"1\"/>\n          <text x=\"70\" y=\"${185 + i * 80}\" font-family=\"Arial\" font-size=\"${fontSize}\" fill=\"${textColor}\">${text}</text>\n        `\n          )\n          .join('')}\n        \n        <!-- UI Elements -->\n        <rect x=\"50\" y=\"${height - 200}\" width=\"150\" height=\"40\" fill=\"#007bff\" rx=\"5\"/>\n        <text x=\"125\" y=\"${height - 175}\" font-family=\"Arial\" font-size=\"16\" fill=\"white\" text-anchor=\"middle\">Button</text>\n        \n        <rect x=\"220\" y=\"${height - 200}\" width=\"200\" height=\"30\" fill=\"white\" stroke=\"#333\" stroke-width=\"1\"/>\n        <text x=\"230\" y=\"${height - 180}\" font-family=\"Arial\" font-size=\"14\" fill=\"#666\">Input Field</text>\n        \n        <!-- Timestamp -->\n        <text x=\"${width - 20}\" y=\"${height - 20}\" font-family=\"Arial\" font-size=\"12\" fill=\"#666\" text-anchor=\"end\">${new Date().toISOString()}</text>\n      </svg>\n    `;\n\n    const buffer = await sharp(Buffer.from(svg)).png().toBuffer();\n\n    return buffer;\n  }\n\n  /**\n   * Generate grid lines for the pattern\n   */\n  private static generateGrid(width: number, height: number, spacing: number = 100): string {\n    const lines: string[] = [];\n\n    // Vertical lines\n    for (let x = spacing; x < width; x += spacing) {\n      lines.push(\n        `<line x1=\"${x}\" y1=\"0\" x2=\"${x}\" y2=\"${height}\" stroke=\"#eeeeee\" stroke-width=\"1\"/>`\n      );\n    }\n\n    // Horizontal lines\n    for (let y = spacing; y < height; y += spacing) {\n      lines.push(\n        `<line x1=\"0\" y1=\"${y}\" x2=\"${width}\" y2=\"${y}\" stroke=\"#eeeeee\" stroke-width=\"1\"/>`\n      );\n    }\n\n    return lines.join('\\n');\n  }\n\n  /**\n   * Save test pattern to file\n   */\n  static async savePattern(buffer: Buffer, filename: string): Promise<string> {\n    const outputDir = path.join(process.cwd(), 'test-patterns');\n    await fs.mkdir(outputDir, { recursive: true });\n\n    const filepath = path.join(outputDir, filename);\n    await fs.writeFile(filepath, buffer);\n\n    logger.info(`[TestPatternGenerator] Saved test pattern to ${filepath}`);\n    return filepath;\n  }\n\n  /**\n   * Generate patterns for all displays\n   */\n  static async generatePatternsForAllDisplays(displayCount: number): Promise<Map<number, Buffer>> {\n    const patterns = new Map<number, Buffer>();\n\n    for (let i = 0; i < displayCount; i++) {\n      const pattern = await this.generateQuadrantPattern({\n        width: 1920,\n        height: 1080,\n        displayIndex: i,\n        includeTimestamp: true,\n      });\n\n      patterns.set(i, pattern);\n    }\n\n    return patterns;\n  }\n\n  /**\n   * Verify OCR results match expected quadrant numbers\n   */\n  static verifyQuadrantNumbers(ocrText: string): {\n    success: boolean;\n    foundNumbers: number[];\n    missingNumbers: number[];\n  } {\n    const expectedNumbers = [1, 2, 3, 4, 5];\n    const foundNumbers: number[] = [];\n\n    // Extract all numbers from OCR text\n    const matches = ocrText.match(/\\d+/g);\n    if (matches) {\n      matches.forEach((match) => {\n        const num = parseInt(match, 10);\n        if (expectedNumbers.includes(num) && !foundNumbers.includes(num)) {\n          foundNumbers.push(num);\n        }\n      });\n    }\n\n    const missingNumbers = expectedNumbers.filter((n) => !foundNumbers.includes(n));\n\n    return {\n      success: missingNumbers.length === 0,\n      foundNumbers,\n      missingNumbers,\n    };\n  }\n}\n"]}

package/dist/tests.d.ts

@@ -0,0 +1,3 @@
+export { default as VisionBasicE2ETestSuite } from './tests/e2e/vision-basic';
+export { default as VisionAutonomyE2ETestSuite } from './tests/e2e/vision-autonomy';
+export { testSuites as default } from './tests/e2e/index';

package/dist/tests.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = exports.VisionAutonomyE2ETestSuite = exports.VisionBasicE2ETestSuite = void 0;
+// Export all test suites for the vision plugin
+var vision_basic_1 = require("./tests/e2e/vision-basic");
+Object.defineProperty(exports, "VisionBasicE2ETestSuite", { enumerable: true, get: function () { return vision_basic_1.default; } });
+var vision_autonomy_1 = require("./tests/e2e/vision-autonomy");
+Object.defineProperty(exports, "VisionAutonomyE2ETestSuite", { enumerable: true, get: function () { return vision_autonomy_1.default; } });
+var index_1 = require("./tests/e2e/index");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return index_1.testSuites; } });
+//# sourceMappingURL=tests.js.map

package/dist/tests.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tests.js","sourceRoot":"","sources":["../src/tests.ts"],"names":[],"mappings":";;;AAAA,+CAA+C;AAC/C,yDAA8E;AAArE,uHAAA,OAAO,OAA2B;AAC3C,+DAAoF;AAA3E,6HAAA,OAAO,OAA8B;AAC9C,2CAA0D;AAAjD,gGAAA,UAAU,OAAW","sourcesContent":["// Export all test suites for the vision plugin\nexport { default as VisionBasicE2ETestSuite } from './tests/e2e/vision-basic';\nexport { default as VisionAutonomyE2ETestSuite } from './tests/e2e/vision-autonomy';\nexport { testSuites as default } from './tests/e2e/index';\n"]}

package/dist/types.d.ts

@@ -0,0 +1,222 @@
+export declare const VisionServiceType: {
+    VISION: "VISION";
+};
+export interface CameraInfo {
+    id: string;
+    name: string;
+    connected: boolean;
+}
+export interface SceneDescription {
+    timestamp: number;
+    description: string;
+    objects: DetectedObject[];
+    people: PersonInfo[];
+    sceneChanged: boolean;
+    changePercentage: number;
+    audioTranscription?: string;
+}
+export interface DetectedObject {
+    id: string;
+    type: string;
+    confidence: number;
+    boundingBox: BoundingBox;
+}
+export interface PersonInfo {
+    id: string;
+    pose: 'sitting' | 'standing' | 'lying' | 'unknown';
+    facing: 'camera' | 'away' | 'left' | 'right' | 'unknown';
+    confidence: number;
+    boundingBox: BoundingBox;
+    keypoints?: Array<{
+        part: string;
+        position: {
+            x: number;
+            y: number;
+        };
+        score: number;
+    }>;
+}
+export interface BoundingBox {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface VisionFrame {
+    timestamp: number;
+    width: number;
+    height: number;
+    data: Buffer;
+    format: 'rgb' | 'rgba' | 'jpeg' | 'png';
+}
+export declare enum VisionMode {
+    OFF = "OFF",
+    CAMERA = "CAMERA",
+    SCREEN = "SCREEN",
+    BOTH = "BOTH"
+}
+export interface ScreenCapture {
+    timestamp: number;
+    width: number;
+    height: number;
+    data: Buffer;
+    tiles: ScreenTile[];
+}
+export interface ScreenTile {
+    id: string;
+    row: number;
+    col: number;
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    data?: Buffer;
+    analysis?: TileAnalysis;
+}
+export interface TileAnalysis {
+    timestamp: number;
+    florence2?: Florence2Result;
+    ocr?: OCRResult;
+    objects?: DetectedObject[];
+    text?: string;
+    summary?: string;
+}
+export interface Florence2Result {
+    caption?: string;
+    objects?: Array<{
+        label: string;
+        bbox: BoundingBox;
+        confidence: number;
+    }>;
+    regions?: Array<{
+        description: string;
+        bbox: BoundingBox;
+    }>;
+    tags?: string[];
+}
+export interface OCRResult {
+    text: string;
+    blocks: Array<{
+        text: string;
+        bbox: BoundingBox;
+        confidence: number;
+        words?: Array<{
+            text: string;
+            bbox: BoundingBox;
+            confidence: number;
+        }>;
+    }>;
+    fullText: string;
+}
+export interface EnhancedSceneDescription extends SceneDescription {
+    screenCapture?: ScreenCapture;
+    screenAnalysis?: {
+        fullScreenOCR?: string;
+        activeTile?: TileAnalysis;
+        gridSummary?: string;
+        focusedApp?: string;
+        uiElements?: Array<{
+            type: string;
+            text: string;
+            position: BoundingBox;
+        }>;
+    };
+}
+export interface VisionConfig {
+    cameraName?: string;
+    pixelChangeThreshold?: number;
+    updateInterval?: number;
+    enablePoseDetection?: boolean;
+    enableObjectDetection?: boolean;
+    tfUpdateInterval?: number;
+    vlmUpdateInterval?: number;
+    tfChangeThreshold?: number;
+    vlmChangeThreshold?: number;
+    visionMode?: VisionMode;
+    screenCaptureInterval?: number;
+    tileSize?: number;
+    tileProcessingOrder?: 'sequential' | 'priority' | 'random';
+    ocrEnabled?: boolean;
+    florence2Enabled?: boolean;
+    screenRegion?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    displayIndex?: number;
+    captureAllDisplays?: boolean;
+    targetScreenFPS?: number;
+    textRegions?: Array<{
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    }>;
+}
+export interface TrackedEntity {
+    id: string;
+    entityType: 'person' | 'object' | 'pet';
+    firstSeen: number;
+    lastSeen: number;
+    lastPosition: BoundingBox;
+    appearances: EntityAppearance[];
+    attributes: EntityAttributes;
+    worldId?: string;
+    roomId?: string;
+}
+export interface EntityAppearance {
+    timestamp: number;
+    boundingBox: BoundingBox;
+    confidence: number;
+    embedding?: number[];
+    keypoints?: Array<{
+        part: string;
+        position: {
+            x: number;
+            y: number;
+        };
+        score: number;
+    }>;
+}
+export interface EntityAttributes {
+    name?: string;
+    faceEmbedding?: number[];
+    faceId?: string;
+    clothing?: string[];
+    hairColor?: string;
+    accessories?: string[];
+    objectType?: string;
+    color?: string;
+    size?: 'small' | 'medium' | 'large';
+    description?: string;
+    tags?: string[];
+}
+export interface FaceLibrary {
+    faces: Map<string, FaceProfile>;
+    embeddings: Map<string, number[][]>;
+}
+export interface FaceProfile {
+    id: string;
+    name?: string;
+    embeddings: number[][];
+    firstSeen: number;
+    lastSeen: number;
+    seenCount: number;
+    attributes?: {
+        age?: string;
+        gender?: string;
+        emotion?: string;
+    };
+}
+export interface WorldState {
+    worldId: string;
+    entities: Map<string, TrackedEntity>;
+    lastUpdate: number;
+    activeEntities: string[];
+    recentlyLeft: Array<{
+        entityId: string;
+        leftAt: number;
+        lastPosition: BoundingBox;
+    }>;
+}

package/dist/types.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VisionMode = exports.VisionServiceType = void 0;
+// Vision service types and interfaces
+exports.VisionServiceType = {
+    VISION: 'VISION',
+};
+// Vision modes
+var VisionMode;
+(function (VisionMode) {
+    VisionMode["OFF"] = "OFF";
+    VisionMode["CAMERA"] = "CAMERA";
+    VisionMode["SCREEN"] = "SCREEN";
+    VisionMode["BOTH"] = "BOTH";
+})(VisionMode || (exports.VisionMode = VisionMode = {}));
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";;;AAAA,sCAAsC;AACzB,QAAA,iBAAiB,GAAG;IAC/B,MAAM,EAAE,QAAiB;CAC1B,CAAC;AAsDF,eAAe;AACf,IAAY,UAKX;AALD,WAAY,UAAU;IACpB,yBAAW,CAAA;IACX,+BAAiB,CAAA;IACjB,+BAAiB,CAAA;IACjB,2BAAa,CAAA;AACf,CAAC,EALW,UAAU,0BAAV,UAAU,QAKrB","sourcesContent":["// Vision service types and interfaces\nexport const VisionServiceType = {\n  VISION: 'VISION' as const,\n};\n\n// Vision-specific types\nexport interface CameraInfo {\n  id: string;\n  name: string;\n  connected: boolean;\n}\n\nexport interface SceneDescription {\n  timestamp: number;\n  description: string;\n  objects: DetectedObject[];\n  people: PersonInfo[];\n  sceneChanged: boolean;\n  changePercentage: number;\n  audioTranscription?: string; // Latest audio transcription\n}\n\nexport interface DetectedObject {\n  id: string;\n  type: string;\n  confidence: number;\n  boundingBox: BoundingBox;\n}\n\nexport interface PersonInfo {\n  id: string;\n  pose: 'sitting' | 'standing' | 'lying' | 'unknown';\n  facing: 'camera' | 'away' | 'left' | 'right' | 'unknown';\n  confidence: number;\n  boundingBox: BoundingBox;\n  keypoints?: Array<{\n    part: string;\n    position: { x: number; y: number };\n    score: number;\n  }>;\n}\n\nexport interface BoundingBox {\n  x: number;\n  y: number;\n  width: number;\n  height: number;\n}\n\nexport interface VisionFrame {\n  timestamp: number;\n  width: number;\n  height: number;\n  data: Buffer;\n  format: 'rgb' | 'rgba' | 'jpeg' | 'png';\n}\n\n// Vision modes\nexport enum VisionMode {\n  OFF = 'OFF',\n  CAMERA = 'CAMERA',\n  SCREEN = 'SCREEN',\n  BOTH = 'BOTH',\n}\n\n// Screen capture types\nexport interface ScreenCapture {\n  timestamp: number;\n  width: number;\n  height: number;\n  data: Buffer;\n  tiles: ScreenTile[];\n}\n\nexport interface ScreenTile {\n  id: string;\n  row: number;\n  col: number;\n  x: number;\n  y: number;\n  width: number;\n  height: number;\n  data?: Buffer;\n  analysis?: TileAnalysis;\n}\n\nexport interface TileAnalysis {\n  timestamp: number;\n  florence2?: Florence2Result;\n  ocr?: OCRResult;\n  objects?: DetectedObject[];\n  text?: string;\n  summary?: string;\n}\n\nexport interface Florence2Result {\n  caption?: string;\n  objects?: Array<{\n    label: string;\n    bbox: BoundingBox;\n    confidence: number;\n  }>;\n  regions?: Array<{\n    description: string;\n    bbox: BoundingBox;\n  }>;\n  tags?: string[];\n}\n\nexport interface OCRResult {\n  text: string;\n  blocks: Array<{\n    text: string;\n    bbox: BoundingBox;\n    confidence: number;\n    words?: Array<{\n      text: string;\n      bbox: BoundingBox;\n      confidence: number;\n    }>;\n  }>;\n  fullText: string;\n}\n\n// Enhanced scene description with screen data\nexport interface EnhancedSceneDescription extends SceneDescription {\n  screenCapture?: ScreenCapture;\n  screenAnalysis?: {\n    fullScreenOCR?: string;\n    activeTile?: TileAnalysis;\n    gridSummary?: string;\n    focusedApp?: string;\n    uiElements?: Array<{\n      type: string;\n      text: string;\n      position: BoundingBox;\n    }>;\n  };\n}\n\n// Update VisionConfig\nexport interface VisionConfig {\n  cameraName?: string;\n  pixelChangeThreshold?: number;\n  updateInterval?: number;\n  enablePoseDetection?: boolean;\n  enableObjectDetection?: boolean;\n  tfUpdateInterval?: number;\n  vlmUpdateInterval?: number;\n  tfChangeThreshold?: number;\n  vlmChangeThreshold?: number;\n\n  // Screen vision config\n  visionMode?: VisionMode;\n  screenCaptureInterval?: number; // ms between screen captures\n  tileSize?: number; // Size of tiles (e.g., 256 for 256x256)\n  tileProcessingOrder?: 'sequential' | 'priority' | 'random';\n  ocrEnabled?: boolean;\n  florence2Enabled?: boolean;\n  screenRegion?: {\n    // Optional: capture only part of screen\n    x: number;\n    y: number;\n    width: number;\n    height: number;\n  };\n\n  // Multi-display and worker config\n  displayIndex?: number; // Specific display to capture\n  captureAllDisplays?: boolean; // Cycle through all displays\n  targetScreenFPS?: number; // Target FPS for screen capture\n  textRegions?: Array<{\n    // Specific regions for OCR\n    x: number;\n    y: number;\n    width: number;\n    height: number;\n  }>;\n}\n\n// Entity tracking types\nexport interface TrackedEntity {\n  id: string;\n  entityType: 'person' | 'object' | 'pet';\n  firstSeen: number;\n  lastSeen: number;\n  lastPosition: BoundingBox;\n  appearances: EntityAppearance[];\n  attributes: EntityAttributes;\n  worldId?: string;\n  roomId?: string;\n}\n\nexport interface EntityAppearance {\n  timestamp: number;\n  boundingBox: BoundingBox;\n  confidence: number;\n  embedding?: number[]; // Face embedding for person recognition\n  keypoints?: Array<{\n    part: string;\n    position: { x: number; y: number };\n    score: number;\n  }>;\n}\n\nexport interface EntityAttributes {\n  // For people\n  name?: string;\n  faceEmbedding?: number[];\n  faceId?: string;\n  clothing?: string[];\n  hairColor?: string;\n  accessories?: string[];\n\n  // For objects\n  objectType?: string;\n  color?: string;\n  size?: 'small' | 'medium' | 'large';\n\n  // Common\n  description?: string;\n  tags?: string[];\n}\n\nexport interface FaceLibrary {\n  faces: Map<string, FaceProfile>;\n  embeddings: Map<string, number[][]>; // Multiple embeddings per profile\n}\n\nexport interface FaceProfile {\n  id: string;\n  name?: string;\n  embeddings: number[][]; // Multiple embeddings for better recognition\n  firstSeen: number;\n  lastSeen: number;\n  seenCount: number;\n  attributes?: {\n    age?: string;\n    gender?: string;\n    emotion?: string;\n  };\n}\n\nexport interface WorldState {\n  worldId: string;\n  entities: Map<string, TrackedEntity>;\n  lastUpdate: number;\n  activeEntities: string[]; // Currently visible\n  recentlyLeft: Array<{\n    entityId: string;\n    leftAt: number;\n    lastPosition: BoundingBox;\n  }>;\n}\n"]}

package/dist/vision-models.d.ts

@@ -0,0 +1,47 @@
+import * as poseDetection from '@tensorflow-models/pose-detection';
+import { IAgentRuntime } from '@elizaos/core';
+import { DetectedObject, PersonInfo } from './types';
+export interface VisionModelConfig {
+    enableObjectDetection?: boolean;
+    enablePoseDetection?: boolean;
+    florence2?: {
+        baseUrl: string;
+        apiKey?: string;
+    };
+    vlm?: {
+        model: string;
+        temperature: number;
+        maxTokens: number;
+    };
+}
+export type Pose = 'sitting' | 'standing' | 'lying' | 'walking' | 'unknown';
+export interface PoseLandmark {
+    name: string;
+    x: number;
+    y: number;
+    score: number;
+}
+export declare class VisionModels {
+    private runtime;
+    private config;
+    private objectDetectionModel;
+    private poseDetector;
+    private initialized;
+    private florence2Model;
+    private cocoSsdModel;
+    private posenetModel;
+    constructor(runtime: IAgentRuntime, config?: VisionModelConfig);
+    initialize(config: VisionModelConfig): Promise<void>;
+    hasObjectDetection(): boolean;
+    hasPoseDetection(): boolean;
+    detectObjects(imageData: Buffer, width: number, height: number, description?: string): Promise<DetectedObject[]>;
+    private enhancedObjectDetection;
+    private generatePlausibleBoundingBox;
+    detectPoses(imageData: Buffer, width: number, height: number, description?: string): Promise<PersonInfo[]>;
+    private enhancedPoseDetection;
+    private generatePlausibleKeypoints;
+    convertPosesToPersonInfo(poses: poseDetection.Pose[]): PersonInfo[];
+    private determinePoseFromKeypoints;
+    private determineFacingDirection;
+    dispose(): Promise<void>;
+}