@elizaos/plugin-vision 1.2.1 → 2.0.0-alpha.1

package/dist/screen-capture.js

@@ -1,302 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
-    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ScreenCaptureService = void 0;
-var core_1 = require("@elizaos/core");
-var child_process_1 = require("child_process");
-var util_1 = require("util");
-var fs = require("fs/promises");
-var path = require("path");
-var sharp_1 = require("sharp");
-var execAsync = (0, util_1.promisify)(child_process_1.exec);
-var ScreenCaptureService = /** @class */ (function () {
-    function ScreenCaptureService(config) {
-        this.activeTileIndex = 0;
-        this.lastCapture = null;
-        this.config = config;
-    }
-    ScreenCaptureService.prototype.getScreenInfo = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var platform, stdout, data, display, resolution, match, stdout, match, stdout, width, height, error_1;
-            var _a, _b, _c, _d;
-            return __generator(this, function (_e) {
-                switch (_e.label) {
-                    case 0:
-                        platform = process.platform;
-                        _e.label = 1;
-                    case 1:
-                        _e.trys.push([1, 8, , 9]);
-                        if (!(platform === 'darwin')) return [3 /*break*/, 3];
-                        return [4 /*yield*/, execAsync('system_profiler SPDisplaysDataType -json')];
-                    case 2:
-                        stdout = (_e.sent()).stdout;
-                        data = JSON.parse(stdout);
-                        if (data.SPDisplaysDataType && data.SPDisplaysDataType[0]) {
-                            display = data.SPDisplaysDataType[0];
-                            resolution = (_b = (_a = display._items) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.native_resolution;
-                            if (resolution) {
-                                match = resolution.match(/(\d+) x (\d+)/);
-                                if (match) {
-                                    return [2 /*return*/, {
-                                            width: parseInt(match[1], 10),
-                                            height: parseInt(match[2], 10),
-                                        }];
-                                }
-                            }
-                        }
-                        return [3 /*break*/, 7];
-                    case 3:
-                        if (!(platform === 'linux')) return [3 /*break*/, 5];
-                        return [4 /*yield*/, execAsync('xrandr | grep " connected primary"')];
-                    case 4:
-                        stdout = (_e.sent()).stdout;
-                        match = stdout.match(/(\d+)x(\d+)/);
-                        if (match) {
-                            return [2 /*return*/, {
-                                    width: parseInt(match[1], 10),
-                                    height: parseInt(match[2], 10),
-                                }];
-                        }
-                        return [3 /*break*/, 7];
-                    case 5:
-                        if (!(platform === 'win32')) return [3 /*break*/, 7];
-                        return [4 /*yield*/, execAsync('wmic path Win32_VideoController get CurrentHorizontalResolution,CurrentVerticalResolution /value')];
-                    case 6:
-                        stdout = (_e.sent()).stdout;
-                        width = (_c = stdout.match(/CurrentHorizontalResolution=(\d+)/)) === null || _c === void 0 ? void 0 : _c[1];
-                        height = (_d = stdout.match(/CurrentVerticalResolution=(\d+)/)) === null || _d === void 0 ? void 0 : _d[1];
-                        if (width && height) {
-                            return [2 /*return*/, {
-                                    width: parseInt(width, 10),
-                                    height: parseInt(height, 10),
-                                }];
-                        }
-                        _e.label = 7;
-                    case 7: return [3 /*break*/, 9];
-                    case 8:
-                        error_1 = _e.sent();
-                        core_1.logger.error('[ScreenCapture] Failed to get screen info:', error_1);
-                        return [3 /*break*/, 9];
-                    case 9: 
-                    // Default fallback
-                    return [2 /*return*/, { width: 1920, height: 1080 }];
-                }
-            });
-        });
-    };
-    ScreenCaptureService.prototype.captureScreen = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var tempFile, imageBuffer, image, metadata, width, height, tileSize, tiles, row, col, x, y, tileWidth, tileHeight, centerRow, centerCol, activeTile, tileBuffer, error_2, capture, error_3;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        tempFile = path.join(process.cwd(), "temp_screen_".concat(Date.now(), ".png"));
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 10, , 12]);
-                        // Capture the screen
-                        return [4 /*yield*/, this.captureScreenToFile(tempFile)];
-                    case 2:
-                        // Capture the screen
-                        _a.sent();
-                        return [4 /*yield*/, fs.readFile(tempFile)];
-                    case 3:
-                        imageBuffer = _a.sent();
-                        image = (0, sharp_1.default)(imageBuffer);
-                        return [4 /*yield*/, image.metadata()];
-                    case 4:
-                        metadata = _a.sent();
-                        width = metadata.width || 1920;
-                        height = metadata.height || 1080;
-                        tileSize = this.config.tileSize || 256;
-                        tiles = [];
-                        for (row = 0; row < Math.ceil(height / tileSize); row++) {
-                            for (col = 0; col < Math.ceil(width / tileSize); col++) {
-                                x = col * tileSize;
-                                y = row * tileSize;
-                                tileWidth = Math.min(tileSize, width - x);
-                                tileHeight = Math.min(tileSize, height - y);
-                                tiles.push({
-                                    id: "tile-".concat(row, "-").concat(col),
-                                    row: row,
-                                    col: col,
-                                    x: x,
-                                    y: y,
-                                    width: tileWidth,
-                                    height: tileHeight,
-                                });
-                            }
-                        }
-                        // Process active tile based on order
-                        if (this.config.tileProcessingOrder === 'priority') {
-                            centerRow = Math.floor(tiles.length / 2 / Math.ceil(width / tileSize));
-                            centerCol = Math.floor((tiles.length / 2) % Math.ceil(width / tileSize));
-                            this.activeTileIndex = centerRow * Math.ceil(width / tileSize) + centerCol;
-                        }
-                        else if (this.config.tileProcessingOrder === 'random') {
-                            this.activeTileIndex = Math.floor(Math.random() * tiles.length);
-                        }
-                        else {
-                            // Sequential
-                            this.activeTileIndex = (this.activeTileIndex + 1) % tiles.length;
-                        }
-                        activeTile = tiles[this.activeTileIndex];
-                        if (!activeTile) return [3 /*break*/, 8];
-                        _a.label = 5;
-                    case 5:
-                        _a.trys.push([5, 7, , 8]);
-                        return [4 /*yield*/, image
-                                .extract({
-                                left: activeTile.x,
-                                top: activeTile.y,
-                                width: activeTile.width,
-                                height: activeTile.height,
-                            })
-                                .png()
-                                .toBuffer()];
-                    case 6:
-                        tileBuffer = _a.sent();
-                        activeTile.data = tileBuffer;
-                        return [3 /*break*/, 8];
-                    case 7:
-                        error_2 = _a.sent();
-                        core_1.logger.error('[ScreenCapture] Failed to extract tile:', error_2);
-                        return [3 /*break*/, 8];
-                    case 8: 
-                    // Clean up temp file
-                    return [4 /*yield*/, fs.unlink(tempFile).catch(function () { })];
-                    case 9:
-                        // Clean up temp file
-                        _a.sent();
-                        capture = {
-                            timestamp: Date.now(),
-                            width: width,
-                            height: height,
-                            data: imageBuffer,
-                            tiles: tiles,
-                        };
-                        this.lastCapture = capture;
-                        return [2 /*return*/, capture];
-                    case 10:
-                        error_3 = _a.sent();
-                        // Clean up temp file on error
-                        return [4 /*yield*/, fs.unlink(tempFile).catch(function () { })];
-                    case 11:
-                        // Clean up temp file on error
-                        _a.sent();
-                        throw error_3;
-                    case 12: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    ScreenCaptureService.prototype.captureScreenToFile = function (outputPath) {
-        return __awaiter(this, void 0, void 0, function () {
-            var platform, _error_1, script, error_4;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        platform = process.platform;
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 13, , 14]);
-                        if (!(platform === 'darwin')) return [3 /*break*/, 3];
-                        // macOS: Use screencapture
-                        return [4 /*yield*/, execAsync("screencapture -x \"".concat(outputPath, "\""))];
-                    case 2:
-                        // macOS: Use screencapture
-                        _a.sent();
-                        return [3 /*break*/, 12];
-                    case 3:
-                        if (!(platform === 'linux')) return [3 /*break*/, 9];
-                        _a.label = 4;
-                    case 4:
-                        _a.trys.push([4, 6, , 8]);
-                        return [4 /*yield*/, execAsync("scrot \"".concat(outputPath, "\""))];
-                    case 5:
-                        _a.sent();
-                        return [3 /*break*/, 8];
-                    case 6:
-                        _error_1 = _a.sent();
-                        // Fallback to gnome-screenshot
-                        return [4 /*yield*/, execAsync("gnome-screenshot -f \"".concat(outputPath, "\""))];
-                    case 7:
-                        // Fallback to gnome-screenshot
-                        _a.sent();
-                        return [3 /*break*/, 8];
-                    case 8: return [3 /*break*/, 12];
-                    case 9:
-                        if (!(platform === 'win32')) return [3 /*break*/, 11];
-                        script = "\n          Add-Type -AssemblyName System.Windows.Forms;\n          Add-Type -AssemblyName System.Drawing;\n          $screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds;\n          $bitmap = New-Object System.Drawing.Bitmap $screen.Width, $screen.Height;\n          $graphics = [System.Drawing.Graphics]::FromImage($bitmap);\n          $graphics.CopyFromScreen($screen.Location, [System.Drawing.Point]::Empty, $screen.Size);\n          $bitmap.Save('".concat(outputPath.replace(/\\/g, '\\\\'), "');\n          $graphics.Dispose();\n          $bitmap.Dispose();\n        ");
-                        return [4 /*yield*/, execAsync("powershell -Command \"".concat(script.replace(/\n/g, ' '), "\""))];
-                    case 10:
-                        _a.sent();
-                        return [3 /*break*/, 12];
-                    case 11: throw new Error("Unsupported platform: ".concat(platform));
-                    case 12: return [3 /*break*/, 14];
-                    case 13:
-                        error_4 = _a.sent();
-                        core_1.logger.error('[ScreenCapture] Screen capture failed:', error_4);
-                        // Provide helpful error messages
-                        if (platform === 'linux' && error_4.message.includes('command not found')) {
-                            throw new Error('Screen capture tool not found. Install with: sudo apt-get install scrot');
-                        }
-                        throw error_4;
-                    case 14: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    ScreenCaptureService.prototype.getActiveTile = function () {
-        if (!this.lastCapture || !this.lastCapture.tiles[this.activeTileIndex]) {
-            return null;
-        }
-        return this.lastCapture.tiles[this.activeTileIndex];
-    };
-    ScreenCaptureService.prototype.getAllTiles = function () {
-        var _a;
-        return ((_a = this.lastCapture) === null || _a === void 0 ? void 0 : _a.tiles) || [];
-    };
-    ScreenCaptureService.prototype.getProcessedTiles = function () {
-        var _a;
-        return ((_a = this.lastCapture) === null || _a === void 0 ? void 0 : _a.tiles.filter(function (t) { return t.analysis; })) || [];
-    };
-    return ScreenCaptureService;
-}());
-exports.ScreenCaptureService = ScreenCaptureService;
-//# sourceMappingURL=screen-capture.js.map

package/dist/screen-capture.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"screen-capture.js","sourceRoot":"","sources":["../src/screen-capture.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sCAAuC;AAEvC,+CAAqC;AACrC,6BAAiC;AACjC,gCAAkC;AAClC,2BAA6B;AAC7B,+BAA0B;AAE1B,IAAM,SAAS,GAAG,IAAA,gBAAS,EAAC,oBAAI,CAAC,CAAC;AAElC;IAKE,8BAAY,MAAoB;QAHxB,oBAAe,GAAG,CAAC,CAAC;QACpB,gBAAW,GAAyB,IAAI,CAAC;QAG/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEK,4CAAa,GAAnB;;;;;;;wBACQ,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;;;;6BAG5B,CAAA,QAAQ,KAAK,QAAQ,CAAA,EAArB,wBAAqB;wBAEJ,qBAAM,SAAS,CAAC,0CAA0C,CAAC,EAAA;;wBAAtE,MAAM,GAAK,CAAA,SAA2D,CAAA,OAAhE;wBACR,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;wBAEhC,IAAI,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;4BACpD,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;4BACrC,UAAU,GAAG,MAAA,MAAA,OAAO,CAAC,MAAM,0CAAG,CAAC,CAAC,0CAAE,iBAAiB,CAAC;4BAC1D,IAAI,UAAU,EAAE,CAAC;gCACT,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gCAChD,IAAI,KAAK,EAAE,CAAC;oCACV,sBAAO;4CACL,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;4CAC7B,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;yCAC/B,EAAC;gCACJ,CAAC;4BACH,CAAC;wBACH,CAAC;;;6BACQ,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,wBAAoB;wBAEV,qBAAM,SAAS,CAAC,oCAAoC,CAAC,EAAA;;wBAAhE,MAAM,GAAK,CAAA,SAAqD,CAAA,OAA1D;wBACR,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;wBAC1C,IAAI,KAAK,EAAE,CAAC;4BACV,sBAAO;oCACL,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;oCAC7B,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;iCAC/B,EAAC;wBACJ,CAAC;;;6BACQ,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,wBAAoB;wBAEV,qBAAM,SAAS,CAChC,kGAAkG,CACnG,EAAA;;wBAFO,MAAM,GAAK,CAAA,SAElB,CAAA,OAFa;wBAGR,KAAK,GAAG,MAAA,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,0CAAG,CAAC,CAAC,CAAC;wBAC/D,MAAM,GAAG,MAAA,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,0CAAG,CAAC,CAAC,CAAC;wBACpE,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;4BACpB,sBAAO;oCACL,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;oCAC1B,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;iCAC7B,EAAC;wBACJ,CAAC;;;;;wBAGH,aAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,OAAK,CAAC,CAAC;;;oBAGpE,mBAAmB;oBACnB,sBAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,EAAC;;;;KACtC;IAEK,4CAAa,GAAnB;;;;;;wBACQ,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,sBAAe,IAAI,CAAC,GAAG,EAAE,SAAM,CAAC,CAAC;;;;wBAGzE,qBAAqB;wBACrB,qBAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,EAAA;;wBADxC,qBAAqB;wBACrB,SAAwC,CAAC;wBAGrB,qBAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAA;;wBAAzC,WAAW,GAAG,SAA2B;wBACzC,KAAK,GAAG,IAAA,eAAK,EAAC,WAAW,CAAC,CAAC;wBAChB,qBAAM,KAAK,CAAC,QAAQ,EAAE,EAAA;;wBAAjC,QAAQ,GAAG,SAAsB;wBAEjC,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC;wBAC/B,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC;wBAGjC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,GAAG,CAAC;wBACvC,KAAK,GAAiB,EAAE,CAAC;wBAE/B,KAAS,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;4BAC5D,KAAS,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gCACrD,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC;gCACnB,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC;gCACnB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gCAC1C,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gCAElD,KAAK,CAAC,IAAI,CAAC;oCACT,EAAE,EAAE,eAAQ,GAAG,cAAI,GAAG,CAAE;oCACxB,GAAG,KAAA;oCACH,GAAG,KAAA;oCACH,CAAC,GAAA;oCACD,CAAC,GAAA;oCACD,KAAK,EAAE,SAAS;oCAChB,MAAM,EAAE,UAAU;iCACnB,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;wBAED,qCAAqC;wBACrC,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,KAAK,UAAU,EAAE,CAAC;4BAE7C,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC;4BACvE,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC;4BAC/E,IAAI,CAAC,eAAe,GAAG,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,SAAS,CAAC;wBAC7E,CAAC;6BAAM,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,KAAK,QAAQ,EAAE,CAAC;4BACxD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;wBAClE,CAAC;6BAAM,CAAC;4BACN,aAAa;4BACb,IAAI,CAAC,eAAe,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;wBACnE,CAAC;wBAGK,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;6BAC3C,UAAU,EAAV,wBAAU;;;;wBAES,qBAAM,KAAK;iCAC3B,OAAO,CAAC;gCACP,IAAI,EAAE,UAAU,CAAC,CAAC;gCAClB,GAAG,EAAE,UAAU,CAAC,CAAC;gCACjB,KAAK,EAAE,UAAU,CAAC,KAAK;gCACvB,MAAM,EAAE,UAAU,CAAC,MAAM;6BAC1B,CAAC;iCACD,GAAG,EAAE;iCACL,QAAQ,EAAE,EAAA;;wBARP,UAAU,GAAG,SAQN;wBAEb,UAAU,CAAC,IAAI,GAAG,UAAU,CAAC;;;;wBAE7B,aAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,OAAK,CAAC,CAAC;;;oBAInE,qBAAqB;oBACrB,qBAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,cAAO,CAAC,CAAC,EAAA;;wBADzC,qBAAqB;wBACrB,SAAyC,CAAC;wBAGpC,OAAO,GAAkB;4BAC7B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;4BACrB,KAAK,OAAA;4BACL,MAAM,QAAA;4BACN,IAAI,EAAE,WAAW;4BACjB,KAAK,OAAA;yBACN,CAAC;wBAEF,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;wBAC3B,sBAAO,OAAO,EAAC;;;wBAEf,8BAA8B;wBAC9B,qBAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,cAAO,CAAC,CAAC,EAAA;;wBADzC,8BAA8B;wBAC9B,SAAyC,CAAC;wBAC1C,MAAM,OAAK,CAAC;;;;;KAEf;IAEa,kDAAmB,GAAjC,UAAkC,UAAkB;;;;;;wBAC5C,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;;;;6BAG5B,CAAA,QAAQ,KAAK,QAAQ,CAAA,EAArB,wBAAqB;wBACvB,2BAA2B;wBAC3B,qBAAM,SAAS,CAAC,6BAAqB,UAAU,OAAG,CAAC,EAAA;;wBADnD,2BAA2B;wBAC3B,SAAmD,CAAC;;;6BAC3C,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,wBAAoB;;;;wBAG3B,qBAAM,SAAS,CAAC,kBAAU,UAAU,OAAG,CAAC,EAAA;;wBAAxC,SAAwC,CAAC;;;;wBAEzC,+BAA+B;wBAC/B,qBAAM,SAAS,CAAC,gCAAwB,UAAU,OAAG,CAAC,EAAA;;wBADtD,+BAA+B;wBAC/B,SAAsD,CAAC;;;;6BAEhD,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,yBAAoB;wBAEvB,MAAM,GAAG,udAOG,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,gFAGlD,CAAC;wBACF,qBAAM,SAAS,CAAC,gCAAwB,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,OAAG,CAAC,EAAA;;wBAAtE,SAAsE,CAAC;;6BAEvE,MAAM,IAAI,KAAK,CAAC,gCAAyB,QAAQ,CAAE,CAAC,CAAC;;;;wBAGvD,aAAM,CAAC,KAAK,CAAC,wCAAwC,EAAE,OAAK,CAAC,CAAC;wBAE9D,iCAAiC;wBACjC,IAAI,QAAQ,KAAK,OAAO,IAAI,OAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;4BACxE,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;wBAC7F,CAAC;wBACD,MAAM,OAAK,CAAC;;;;;KAEf;IAED,4CAAa,GAAb;QACE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACtD,CAAC;IAED,0CAAW,GAAX;;QACE,OAAO,CAAA,MAAA,IAAI,CAAC,WAAW,0CAAE,KAAK,KAAI,EAAE,CAAC;IACvC,CAAC;IAED,gDAAiB,GAAjB;;QACE,OAAO,CAAA,MAAA,IAAI,CAAC,WAAW,0CAAE,KAAK,CAAC,MAAM,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,QAAQ,EAAV,CAAU,CAAC,KAAI,EAAE,CAAC;IACjE,CAAC;IACH,2BAAC;AAAD,CAAC,AApND,IAoNC;AApNY,oDAAoB","sourcesContent":["import { logger } from '@elizaos/core';\nimport type { ScreenCapture, ScreenTile, VisionConfig } from './types';\nimport { exec } from 'child_process';\nimport { promisify } from 'util';\nimport * as fs from 'fs/promises';\nimport * as path from 'path';\nimport sharp from 'sharp';\n\nconst execAsync = promisify(exec);\n\nexport class ScreenCaptureService {\n  private config: VisionConfig;\n  private activeTileIndex = 0;\n  private lastCapture: ScreenCapture | null = null;\n\n  constructor(config: VisionConfig) {\n    this.config = config;\n  }\n\n  async getScreenInfo(): Promise<{ width: number; height: number } | null> {\n    const platform = process.platform;\n\n    try {\n      if (platform === 'darwin') {\n        // macOS: Use system_profiler\n        const { stdout } = await execAsync('system_profiler SPDisplaysDataType -json');\n        const data = JSON.parse(stdout);\n\n        if (data.SPDisplaysDataType && data.SPDisplaysDataType[0]) {\n          const display = data.SPDisplaysDataType[0];\n          const resolution = display._items?.[0]?.native_resolution;\n          if (resolution) {\n            const match = resolution.match(/(\\d+) x (\\d+)/);\n            if (match) {\n              return {\n                width: parseInt(match[1], 10),\n                height: parseInt(match[2], 10),\n              };\n            }\n          }\n        }\n      } else if (platform === 'linux') {\n        // Linux: Use xrandr\n        const { stdout } = await execAsync('xrandr | grep \" connected primary\"');\n        const match = stdout.match(/(\\d+)x(\\d+)/);\n        if (match) {\n          return {\n            width: parseInt(match[1], 10),\n            height: parseInt(match[2], 10),\n          };\n        }\n      } else if (platform === 'win32') {\n        // Windows: Use wmic\n        const { stdout } = await execAsync(\n          'wmic path Win32_VideoController get CurrentHorizontalResolution,CurrentVerticalResolution /value'\n        );\n        const width = stdout.match(/CurrentHorizontalResolution=(\\d+)/)?.[1];\n        const height = stdout.match(/CurrentVerticalResolution=(\\d+)/)?.[1];\n        if (width && height) {\n          return {\n            width: parseInt(width, 10),\n            height: parseInt(height, 10),\n          };\n        }\n      }\n    } catch (error) {\n      logger.error('[ScreenCapture] Failed to get screen info:', error);\n    }\n\n    // Default fallback\n    return { width: 1920, height: 1080 };\n  }\n\n  async captureScreen(): Promise<ScreenCapture> {\n    const tempFile = path.join(process.cwd(), `temp_screen_${Date.now()}.png`);\n\n    try {\n      // Capture the screen\n      await this.captureScreenToFile(tempFile);\n\n      // Load and process the image\n      const imageBuffer = await fs.readFile(tempFile);\n      const image = sharp(imageBuffer);\n      const metadata = await image.metadata();\n\n      const width = metadata.width || 1920;\n      const height = metadata.height || 1080;\n\n      // Create tiles\n      const tileSize = this.config.tileSize || 256;\n      const tiles: ScreenTile[] = [];\n\n      for (let row = 0; row < Math.ceil(height / tileSize); row++) {\n        for (let col = 0; col < Math.ceil(width / tileSize); col++) {\n          const x = col * tileSize;\n          const y = row * tileSize;\n          const tileWidth = Math.min(tileSize, width - x);\n          const tileHeight = Math.min(tileSize, height - y);\n\n          tiles.push({\n            id: `tile-${row}-${col}`,\n            row,\n            col,\n            x,\n            y,\n            width: tileWidth,\n            height: tileHeight,\n          });\n        }\n      }\n\n      // Process active tile based on order\n      if (this.config.tileProcessingOrder === 'priority') {\n        // Focus on center tiles first\n        const centerRow = Math.floor(tiles.length / 2 / Math.ceil(width / tileSize));\n        const centerCol = Math.floor((tiles.length / 2) % Math.ceil(width / tileSize));\n        this.activeTileIndex = centerRow * Math.ceil(width / tileSize) + centerCol;\n      } else if (this.config.tileProcessingOrder === 'random') {\n        this.activeTileIndex = Math.floor(Math.random() * tiles.length);\n      } else {\n        // Sequential\n        this.activeTileIndex = (this.activeTileIndex + 1) % tiles.length;\n      }\n\n      // Extract active tile data\n      const activeTile = tiles[this.activeTileIndex];\n      if (activeTile) {\n        try {\n          const tileBuffer = await image\n            .extract({\n              left: activeTile.x,\n              top: activeTile.y,\n              width: activeTile.width,\n              height: activeTile.height,\n            })\n            .png()\n            .toBuffer();\n\n          activeTile.data = tileBuffer;\n        } catch (error) {\n          logger.error('[ScreenCapture] Failed to extract tile:', error);\n        }\n      }\n\n      // Clean up temp file\n      await fs.unlink(tempFile).catch(() => {});\n\n      // Create screen capture object\n      const capture: ScreenCapture = {\n        timestamp: Date.now(),\n        width,\n        height,\n        data: imageBuffer,\n        tiles,\n      };\n\n      this.lastCapture = capture;\n      return capture;\n    } catch (error) {\n      // Clean up temp file on error\n      await fs.unlink(tempFile).catch(() => {});\n      throw error;\n    }\n  }\n\n  private async captureScreenToFile(outputPath: string): Promise<void> {\n    const platform = process.platform;\n\n    try {\n      if (platform === 'darwin') {\n        // macOS: Use screencapture\n        await execAsync(`screencapture -x \"${outputPath}\"`);\n      } else if (platform === 'linux') {\n        // Linux: Use scrot or gnome-screenshot\n        try {\n          await execAsync(`scrot \"${outputPath}\"`);\n        } catch (_error) {\n          // Fallback to gnome-screenshot\n          await execAsync(`gnome-screenshot -f \"${outputPath}\"`);\n        }\n      } else if (platform === 'win32') {\n        // Windows: Use PowerShell\n        const script = `\n          Add-Type -AssemblyName System.Windows.Forms;\n          Add-Type -AssemblyName System.Drawing;\n          $screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds;\n          $bitmap = New-Object System.Drawing.Bitmap $screen.Width, $screen.Height;\n          $graphics = [System.Drawing.Graphics]::FromImage($bitmap);\n          $graphics.CopyFromScreen($screen.Location, [System.Drawing.Point]::Empty, $screen.Size);\n          $bitmap.Save('${outputPath.replace(/\\\\/g, '\\\\\\\\')}');\n          $graphics.Dispose();\n          $bitmap.Dispose();\n        `;\n        await execAsync(`powershell -Command \"${script.replace(/\\n/g, ' ')}\"`);\n      } else {\n        throw new Error(`Unsupported platform: ${platform}`);\n      }\n    } catch (error: any) {\n      logger.error('[ScreenCapture] Screen capture failed:', error);\n\n      // Provide helpful error messages\n      if (platform === 'linux' && error.message.includes('command not found')) {\n        throw new Error('Screen capture tool not found. Install with: sudo apt-get install scrot');\n      }\n      throw error;\n    }\n  }\n\n  getActiveTile(): ScreenTile | null {\n    if (!this.lastCapture || !this.lastCapture.tiles[this.activeTileIndex]) {\n      return null;\n    }\n    return this.lastCapture.tiles[this.activeTileIndex];\n  }\n\n  getAllTiles(): ScreenTile[] {\n    return this.lastCapture?.tiles || [];\n  }\n\n  getProcessedTiles(): ScreenTile[] {\n    return this.lastCapture?.tiles.filter((t) => t.analysis) || [];\n  }\n}\n"]}

package/dist/service.d.ts

@@ -1,73 +0,0 @@
-import { Service, type IAgentRuntime, type ServiceTypeName } from '@elizaos/core';
-import { EntityTracker } from './entity-tracker';
-import { FaceRecognition } from './face-recognition';
-import { VisionMode, type CameraInfo, type EnhancedSceneDescription, type SceneDescription, type ScreenCapture, type VisionFrame } from './types';
-export declare class VisionService extends Service {
-    static serviceType: ServiceTypeName;
-    capabilityDescription: string;
-    private visionConfig;
-    private camera;
-    private lastFrame;
-    private lastSceneDescription;
-    private frameProcessingInterval;
-    private screenProcessingInterval;
-    private isProcessing;
-    private isProcessingScreen;
-    private visionModels;
-    private faceRecognition;
-    private entityTracker;
-    private audioCapture;
-    private streamingAudioCapture;
-    private screenCapture;
-    private florence2;
-    private ocrService;
-    private lastScreenCapture;
-    private lastEnhancedScene;
-    private workerManager;
-    private lastTfUpdateTime;
-    private lastVlmUpdateTime;
-    private lastTfDescription;
-    private readonly DEFAULT_CONFIG;
-    constructor(runtime: IAgentRuntime);
-    private parseConfig;
-    static start(runtime: IAgentRuntime): Promise<VisionService>;
-    private checkCameraTools;
-    private initialize;
-    private initializeScreenVision;
-    private initializeCameraVision;
-    private initializeAudioCapture;
-    private storeAudioTranscription;
-    private startProcessing;
-    private startFrameProcessing;
-    private captureAndProcessFrame;
-    private processFrameData;
-    private calculatePixelChange;
-    private updateSceneDescription;
-    private describeSceneWithVLM;
-    private detectMotionObjects;
-    private mergeAdjacentObjects;
-    private classifyObjectBySize;
-    private detectPeopleFromMotion;
-    private startScreenProcessing;
-    private captureAndProcessScreen;
-    private analyzeTile;
-    private updateEnhancedSceneDescription;
-    getCurrentFrame(): Promise<VisionFrame | null>;
-    getSceneDescription(): Promise<SceneDescription | null>;
-    getEnhancedSceneDescription(): Promise<EnhancedSceneDescription | null>;
-    getScreenCapture(): Promise<ScreenCapture | null>;
-    getVisionMode(): VisionMode;
-    setVisionMode(mode: VisionMode): Promise<void>;
-    private stopProcessing;
-    getCameraInfo(): CameraInfo | null;
-    isActive(): boolean;
-    private calculateBoxOverlap;
-    private getDominantExpression;
-    getEntityTracker(): EntityTracker;
-    getFaceRecognition(): FaceRecognition;
-    stop(): Promise<void>;
-    private findCamera;
-    private listCameras;
-    private createCameraDevice;
-    captureImage(): Promise<Buffer | null>;
-}