@elizaos/plugin-vision 1.2.1

package/dist/config.js

@@ -0,0 +1,254 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConfigurationManager = exports.VisionConfigSchema = exports.defaultVisionConfig = void 0;
+var zod_1 = require("zod");
+var core_1 = require("@elizaos/core");
+// Default configuration for backward compatibility
+exports.defaultVisionConfig = {
+    pixelChangeThreshold: 50,
+    updateInterval: 100,
+    enablePoseDetection: false,
+    enableObjectDetection: false,
+    tfUpdateInterval: 1000,
+    vlmUpdateInterval: 10000,
+    tfChangeThreshold: 10,
+    vlmChangeThreshold: 50,
+    visionMode: 'CAMERA',
+    screenCaptureInterval: 2000,
+    tileSize: 256,
+    tileProcessingOrder: 'priority',
+    ocrEnabled: true,
+    florence2Enabled: true,
+};
+// Configuration schema using Zod for validation
+exports.VisionConfigSchema = zod_1.z.object({
+    // Camera configuration
+    cameraName: zod_1.z.string().optional(),
+    enableCamera: zod_1.z.boolean().default(true),
+    // Vision processing
+    pixelChangeThreshold: zod_1.z.number().min(0).max(100).default(50),
+    updateInterval: zod_1.z.number().min(10).max(10000).default(100),
+    // Object detection
+    enableObjectDetection: zod_1.z.boolean().default(false),
+    objectConfidenceThreshold: zod_1.z.number().min(0).max(1).default(0.5),
+    // Pose detection
+    enablePoseDetection: zod_1.z.boolean().default(false),
+    poseConfidenceThreshold: zod_1.z.number().min(0).max(1).default(0.5),
+    // Update intervals
+    tfUpdateInterval: zod_1.z.number().min(100).max(60000).default(1000),
+    vlmUpdateInterval: zod_1.z.number().min(1000).max(300000).default(10000),
+    tfChangeThreshold: zod_1.z.number().min(0).max(100).default(10),
+    vlmChangeThreshold: zod_1.z.number().min(0).max(100).default(50),
+    // Vision mode
+    visionMode: zod_1.z.enum(['OFF', 'CAMERA', 'SCREEN', 'BOTH']).default('CAMERA'),
+    // Screen capture
+    screenCaptureInterval: zod_1.z.number().min(100).max(60000).default(2000),
+    tileSize: zod_1.z.number().min(64).max(1024).default(256),
+    tileProcessingOrder: zod_1.z.enum(['sequential', 'priority', 'random']).default('priority'),
+    maxConcurrentTiles: zod_1.z.number().min(1).max(10).default(3),
+    // OCR configuration
+    ocrEnabled: zod_1.z.boolean().default(true),
+    ocrLanguage: zod_1.z.string().default('eng'),
+    ocrConfidenceThreshold: zod_1.z.number().min(0).max(100).default(60),
+    // Florence-2 configuration
+    florence2Enabled: zod_1.z.boolean().default(true),
+    florence2Provider: zod_1.z.enum(['local', 'azure', 'huggingface', 'replicate']).optional(),
+    florence2Endpoint: zod_1.z.string().url().optional(),
+    florence2ApiKey: zod_1.z.string().optional(),
+    florence2Timeout: zod_1.z.number().min(1000).max(300000).default(30000),
+    // Face recognition
+    enableFaceRecognition: zod_1.z.boolean().default(false),
+    faceMatchThreshold: zod_1.z.number().min(0).max(1).default(0.6),
+    maxFaceProfiles: zod_1.z.number().min(10).max(10000).default(1000),
+    // Entity tracking
+    entityTimeout: zod_1.z.number().min(1000).max(300000).default(30000),
+    maxTrackedEntities: zod_1.z.number().min(10).max(1000).default(100),
+    // Performance
+    enableGPUAcceleration: zod_1.z.boolean().default(true),
+    maxMemoryUsageMB: zod_1.z.number().min(100).max(8000).default(2000),
+    // Logging
+    debugMode: zod_1.z.boolean().default(false),
+    logLevel: zod_1.z.enum(['error', 'warn', 'info', 'debug']).default('info'),
+});
+var ConfigurationManager = /** @class */ (function () {
+    function ConfigurationManager(runtime) {
+        this.runtime = runtime;
+        this.config = this.loadConfiguration();
+    }
+    ConfigurationManager.prototype.loadConfiguration = function () {
+        // Load from environment variables and runtime settings
+        var rawConfig = {
+            // Camera
+            cameraName: this.getSetting('CAMERA_NAME') || this.getSetting('VISION_CAMERA_NAME'),
+            enableCamera: this.getBooleanSetting('ENABLE_CAMERA', true),
+            // Vision processing
+            pixelChangeThreshold: this.getNumberSetting('PIXEL_CHANGE_THRESHOLD', 50),
+            updateInterval: this.getNumberSetting('UPDATE_INTERVAL', 100),
+            // Object detection
+            enableObjectDetection: this.getBooleanSetting('ENABLE_OBJECT_DETECTION', false),
+            objectConfidenceThreshold: this.getNumberSetting('OBJECT_CONFIDENCE_THRESHOLD', 0.5),
+            // Pose detection
+            enablePoseDetection: this.getBooleanSetting('ENABLE_POSE_DETECTION', false),
+            poseConfidenceThreshold: this.getNumberSetting('POSE_CONFIDENCE_THRESHOLD', 0.5),
+            // Update intervals
+            tfUpdateInterval: this.getNumberSetting('TF_UPDATE_INTERVAL', 1000),
+            vlmUpdateInterval: this.getNumberSetting('VLM_UPDATE_INTERVAL', 10000),
+            tfChangeThreshold: this.getNumberSetting('TF_CHANGE_THRESHOLD', 10),
+            vlmChangeThreshold: this.getNumberSetting('VLM_CHANGE_THRESHOLD', 50),
+            // Vision mode
+            visionMode: this.getSetting('VISION_MODE'),
+            // Screen capture
+            screenCaptureInterval: this.getNumberSetting('SCREEN_CAPTURE_INTERVAL', 2000),
+            tileSize: this.getNumberSetting('TILE_SIZE', 256),
+            tileProcessingOrder: this.getSetting('TILE_PROCESSING_ORDER'),
+            maxConcurrentTiles: this.getNumberSetting('MAX_CONCURRENT_TILES', 3),
+            // OCR
+            ocrEnabled: this.getBooleanSetting('OCR_ENABLED', true),
+            ocrLanguage: this.getSetting('OCR_LANGUAGE') || 'eng',
+            ocrConfidenceThreshold: this.getNumberSetting('OCR_CONFIDENCE_THRESHOLD', 60),
+            // Florence-2
+            florence2Enabled: this.getBooleanSetting('FLORENCE2_ENABLED', true),
+            florence2Provider: this.getSetting('FLORENCE2_PROVIDER'),
+            florence2Endpoint: this.getSetting('FLORENCE2_ENDPOINT'),
+            florence2ApiKey: this.getSetting('FLORENCE2_API_KEY'),
+            florence2Timeout: this.getNumberSetting('FLORENCE2_TIMEOUT', 30000),
+            // Face recognition
+            enableFaceRecognition: this.getBooleanSetting('ENABLE_FACE_RECOGNITION', false),
+            faceMatchThreshold: this.getNumberSetting('FACE_MATCH_THRESHOLD', 0.6),
+            maxFaceProfiles: this.getNumberSetting('MAX_FACE_PROFILES', 1000),
+            // Entity tracking
+            entityTimeout: this.getNumberSetting('ENTITY_TIMEOUT', 30000),
+            maxTrackedEntities: this.getNumberSetting('MAX_TRACKED_ENTITIES', 100),
+            // Performance
+            enableGPUAcceleration: this.getBooleanSetting('ENABLE_GPU_ACCELERATION', true),
+            maxMemoryUsageMB: this.getNumberSetting('MAX_MEMORY_USAGE_MB', 2000),
+            // Logging
+            debugMode: this.getBooleanSetting('DEBUG_MODE', false),
+            logLevel: this.getSetting('LOG_LEVEL') || 'info',
+        };
+        // Validate and parse configuration
+        try {
+            var parsed = exports.VisionConfigSchema.parse(rawConfig);
+            core_1.logger.info('[ConfigurationManager] Configuration loaded successfully');
+            if (parsed.debugMode) {
+                core_1.logger.debug('[ConfigurationManager] Configuration:', parsed);
+            }
+            return parsed;
+        }
+        catch (error) {
+            core_1.logger.error('[ConfigurationManager] Invalid configuration:', error);
+            if (error instanceof zod_1.z.ZodError) {
+                core_1.logger.error('[ConfigurationManager] Validation errors:', error.errors);
+            }
+            // Return default configuration on error
+            return exports.VisionConfigSchema.parse({});
+        }
+    };
+    ConfigurationManager.prototype.getSetting = function (key) {
+        // Try with VISION_ prefix first
+        var visionKey = "VISION_".concat(key);
+        var value = this.runtime.getSetting(visionKey) || this.runtime.getSetting(key);
+        return value || undefined;
+    };
+    ConfigurationManager.prototype.getBooleanSetting = function (key, defaultValue) {
+        var value = this.getSetting(key);
+        if (value === undefined) {
+            return defaultValue;
+        }
+        return value.toLowerCase() === 'true';
+    };
+    ConfigurationManager.prototype.getNumberSetting = function (key, defaultValue) {
+        var value = this.getSetting(key);
+        if (value === undefined) {
+            return defaultValue;
+        }
+        var parsed = Number(value);
+        return isNaN(parsed) ? defaultValue : parsed;
+    };
+    // Public API
+    ConfigurationManager.prototype.get = function () {
+        return __assign({}, this.config);
+    };
+    ConfigurationManager.prototype.update = function (updates) {
+        try {
+            var newConfig = __assign(__assign({}, this.config), updates);
+            var parsed = exports.VisionConfigSchema.parse(newConfig);
+            this.config = parsed;
+            core_1.logger.info('[ConfigurationManager] Configuration updated');
+        }
+        catch (error) {
+            core_1.logger.error('[ConfigurationManager] Failed to update configuration:', error);
+            throw error;
+        }
+    };
+    // Convert to legacy VisionConfig format for backward compatibility
+    ConfigurationManager.prototype.toLegacyFormat = function () {
+        return {
+            cameraName: this.config.cameraName,
+            pixelChangeThreshold: this.config.pixelChangeThreshold,
+            updateInterval: this.config.updateInterval,
+            enableObjectDetection: this.config.enableObjectDetection,
+            enablePoseDetection: this.config.enablePoseDetection,
+            tfUpdateInterval: this.config.tfUpdateInterval,
+            vlmUpdateInterval: this.config.vlmUpdateInterval,
+            tfChangeThreshold: this.config.tfChangeThreshold,
+            vlmChangeThreshold: this.config.vlmChangeThreshold,
+            visionMode: this.config.visionMode,
+            screenCaptureInterval: this.config.screenCaptureInterval,
+            tileSize: this.config.tileSize,
+            tileProcessingOrder: this.config.tileProcessingOrder,
+            ocrEnabled: this.config.ocrEnabled,
+            florence2Enabled: this.config.florence2Enabled,
+        };
+    };
+    // Configuration presets
+    ConfigurationManager.getPreset = function (name) {
+        var presets = {
+            'high-performance': {
+                updateInterval: 50,
+                tfUpdateInterval: 500,
+                vlmUpdateInterval: 5000,
+                enableGPUAcceleration: true,
+                maxConcurrentTiles: 5,
+            },
+            'low-resource': {
+                updateInterval: 200,
+                tfUpdateInterval: 2000,
+                vlmUpdateInterval: 20000,
+                enableObjectDetection: false,
+                enablePoseDetection: false,
+                maxMemoryUsageMB: 500,
+                maxConcurrentTiles: 1,
+            },
+            'security-monitoring': {
+                enableObjectDetection: true,
+                enablePoseDetection: true,
+                enableFaceRecognition: true,
+                updateInterval: 100,
+                entityTimeout: 60000,
+            },
+            'screen-reader': {
+                visionMode: 'SCREEN',
+                ocrEnabled: true,
+                florence2Enabled: true,
+                screenCaptureInterval: 1000,
+                tileProcessingOrder: 'priority',
+            },
+        };
+        return presets[name] || {};
+    };
+    return ConfigurationManager;
+}());
+exports.ConfigurationManager = ConfigurationManager;
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,2BAAwB;AACxB,sCAAuC;AAGvC,mDAAmD;AACtC,QAAA,mBAAmB,GAAiB;IAC/C,oBAAoB,EAAE,EAAE;IACxB,cAAc,EAAE,GAAG;IACnB,mBAAmB,EAAE,KAAK;IAC1B,qBAAqB,EAAE,KAAK;IAC5B,gBAAgB,EAAE,IAAI;IACtB,iBAAiB,EAAE,KAAK;IACxB,iBAAiB,EAAE,EAAE;IACrB,kBAAkB,EAAE,EAAE;IACtB,UAAU,EAAE,QAAsB;IAClC,qBAAqB,EAAE,IAAI;IAC3B,QAAQ,EAAE,GAAG;IACb,mBAAmB,EAAE,UAAU;IAC/B,UAAU,EAAE,IAAI;IAChB,gBAAgB,EAAE,IAAI;CACvB,CAAC;AAEF,gDAAgD;AACnC,QAAA,kBAAkB,GAAG,OAAC,CAAC,MAAM,CAAC;IACzC,uBAAuB;IACvB,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,YAAY,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAEvC,oBAAoB;IACpB,oBAAoB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC5D,cAAc,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAE1D,mBAAmB;IACnB,qBAAqB,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACjD,yBAAyB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAEhE,iBAAiB;IACjB,mBAAmB,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IAC/C,uBAAuB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAE9D,mBAAmB;IACnB,gBAAgB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAC9D,iBAAiB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClE,iBAAiB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACzD,kBAAkB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAE1D,cAAc;IACd,UAAU,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;IAEzE,iBAAiB;IACjB,qBAAqB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACnE,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IACnD,mBAAmB,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IACrF,kBAAkB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAExD,oBAAoB;IACpB,UAAU,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACrC,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACtC,sBAAsB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAE9D,2BAA2B;IAC3B,gBAAgB,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3C,iBAAiB,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;IACpF,iBAAiB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IAC9C,eAAe,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,gBAAgB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAEjE,mBAAmB;IACnB,qBAAqB,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACjD,kBAAkB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IACzD,eAAe,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAE5D,kBAAkB;IAClB,aAAa,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAC9D,kBAAkB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAE7D,cAAc;IACd,qBAAqB,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAChD,gBAAgB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAE7D,UAAU;IACV,SAAS,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACrC,QAAQ,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;CACrE,CAAC,CAAC;AAKH;IAIE,8BAAY,OAAY;QACtB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;IACzC,CAAC;IAEO,gDAAiB,GAAzB;QACE,uDAAuD;QACvD,IAAM,SAAS,GAA+B;YAC5C,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC;YACnF,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,eAAe,EAAE,IAAI,CAAC;YAE3D,oBAAoB;YACpB,oBAAoB,EAAE,IAAI,CAAC,gBAAgB,CAAC,wBAAwB,EAAE,EAAE,CAAC;YACzE,cAAc,EAAE,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,GAAG,CAAC;YAE7D,mBAAmB;YACnB,qBAAqB,EAAE,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,EAAE,KAAK,CAAC;YAC/E,yBAAyB,EAAE,IAAI,CAAC,gBAAgB,CAAC,6BAA6B,EAAE,GAAG,CAAC;YAEpF,iBAAiB;YACjB,mBAAmB,EAAE,IAAI,CAAC,iBAAiB,CAAC,uBAAuB,EAAE,KAAK,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,2BAA2B,EAAE,GAAG,CAAC;YAEhF,mBAAmB;YACnB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,EAAE,IAAI,CAAC;YACnE,iBAAiB,EAAE,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,KAAK,CAAC;YACtE,iBAAiB,EAAE,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,EAAE,CAAC;YACnE,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,sBAAsB,EAAE,EAAE,CAAC;YAErE,cAAc;YACd,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,aAAa,CAAe;YAExD,iBAAiB;YACjB,qBAAqB,EAAE,IAAI,CAAC,gBAAgB,CAAC,yBAAyB,EAAE,IAAI,CAAC;YAC7E,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,GAAG,CAAC;YACjD,mBAAmB,EAAE,IAAI,CAAC,UAAU,CAAC,uBAAuB,CAAQ;YACpE,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,sBAAsB,EAAE,CAAC,CAAC;YAEpE,MAAM;YACN,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,IAAI,CAAC;YACvD,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,KAAK;YACrD,sBAAsB,EAAE,IAAI,CAAC,gBAAgB,CAAC,0BAA0B,EAAE,EAAE,CAAC;YAE7E,aAAa;YACb,gBAAgB,EAAE,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,EAAE,IAAI,CAAC;YACnE,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAQ;YAC/D,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC;YACxD,eAAe,EAAE,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;YACrD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,KAAK,CAAC;YAEnE,mBAAmB;YACnB,qBAAqB,EAAE,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,EAAE,KAAK,CAAC;YAC/E,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,sBAAsB,EAAE,GAAG,CAAC;YACtE,eAAe,EAAE,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,IAAI,CAAC;YAEjE,kBAAkB;YAClB,aAAa,EAAE,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,KAAK,CAAC;YAC7D,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,sBAAsB,EAAE,GAAG,CAAC;YAEtE,cAAc;YACd,qBAAqB,EAAE,IAAI,CAAC,iBAAiB,CAAC,yBAAyB,EAAE,IAAI,CAAC;YAC9E,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,EAAE,IAAI,CAAC;YAEpE,UAAU;YACV,SAAS,EAAE,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,KAAK,CAAC;YACtD,QAAQ,EAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAS,IAAI,MAAM;SAC1D,CAAC;QAEF,mCAAmC;QACnC,IAAI,CAAC;YACH,IAAM,MAAM,GAAG,0BAAkB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACnD,aAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;YAExE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,aAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,MAAM,CAAC,CAAC;YAChE,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE,KAAK,CAAC,CAAC;YACrE,IAAI,KAAK,YAAY,OAAC,CAAC,QAAQ,EAAE,CAAC;gBAChC,aAAM,CAAC,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1E,CAAC;YAED,wCAAwC;YACxC,OAAO,0BAAkB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,yCAAU,GAAlB,UAAmB,GAAW;QAC5B,gCAAgC;QAChC,IAAM,SAAS,GAAG,iBAAU,GAAG,CAAE,CAAC;QAClC,IAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACjF,OAAO,KAAK,IAAI,SAAS,CAAC;IAC5B,CAAC;IAEO,gDAAiB,GAAzB,UAA0B,GAAW,EAAE,YAAqB;QAC1D,IAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,YAAY,CAAC;QACtB,CAAC;QACD,OAAO,KAAK,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;IACxC,CAAC;IAEO,+CAAgB,GAAxB,UAAyB,GAAW,EAAE,YAAoB;QACxD,IAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,YAAY,CAAC;QACtB,CAAC;QACD,IAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/C,CAAC;IAED,aAAa;IACb,kCAAG,GAAH;QACE,oBAAY,IAAI,CAAC,MAAM,EAAG;IAC5B,CAAC;IAED,qCAAM,GAAN,UAAO,OAAmC;QACxC,IAAI,CAAC;YACH,IAAM,SAAS,yBAAQ,IAAI,CAAC,MAAM,GAAK,OAAO,CAAE,CAAC;YACjD,IAAM,MAAM,GAAG,0BAAkB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACnD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;YACrB,aAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAM,CAAC,KAAK,CAAC,wDAAwD,EAAE,KAAK,CAAC,CAAC;YAC9E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,6CAAc,GAAd;QACE,OAAO;YACL,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAClC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB;YACtD,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;YAC1C,qBAAqB,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB;YACxD,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB;YACpD,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB;YAC9C,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;YAChD,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;YAChD,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB;YAClD,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAwB;YAChD,qBAAqB,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB;YACxD,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;YAC9B,mBAAmB,EAAE,IAAI,CAAC,MAAM,CAAC,mBAAmB;YACpD,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;YAClC,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB;SAC/C,CAAC;IACJ,CAAC;IAED,wBAAwB;IACjB,8BAAS,GAAhB,UAAiB,IAAY;QAC3B,IAAM,OAAO,GAA+C;YAC1D,kBAAkB,EAAE;gBAClB,cAAc,EAAE,EAAE;gBAClB,gBAAgB,EAAE,GAAG;gBACrB,iBAAiB,EAAE,IAAI;gBACvB,qBAAqB,EAAE,IAAI;gBAC3B,kBAAkB,EAAE,CAAC;aACtB;YACD,cAAc,EAAE;gBACd,cAAc,EAAE,GAAG;gBACnB,gBAAgB,EAAE,IAAI;gBACtB,iBAAiB,EAAE,KAAK;gBACxB,qBAAqB,EAAE,KAAK;gBAC5B,mBAAmB,EAAE,KAAK;gBAC1B,gBAAgB,EAAE,GAAG;gBACrB,kBAAkB,EAAE,CAAC;aACtB;YACD,qBAAqB,EAAE;gBACrB,qBAAqB,EAAE,IAAI;gBAC3B,mBAAmB,EAAE,IAAI;gBACzB,qBAAqB,EAAE,IAAI;gBAC3B,cAAc,EAAE,GAAG;gBACnB,aAAa,EAAE,KAAK;aACrB;YACD,eAAe,EAAE;gBACf,UAAU,EAAE,QAAQ;gBACpB,UAAU,EAAE,IAAI;gBAChB,gBAAgB,EAAE,IAAI;gBACtB,qBAAqB,EAAE,IAAI;gBAC3B,mBAAmB,EAAE,UAAU;aAChC;SACF,CAAC;QAEF,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;IACH,2BAAC;AAAD,CAAC,AAjMD,IAiMC;AAjMY,oDAAoB","sourcesContent":["import { z } from 'zod';\nimport { logger } from '@elizaos/core';\nimport type { VisionConfig, VisionMode } from './types';\n\n// Default configuration for backward compatibility\nexport const defaultVisionConfig: VisionConfig = {\n  pixelChangeThreshold: 50,\n  updateInterval: 100,\n  enablePoseDetection: false,\n  enableObjectDetection: false,\n  tfUpdateInterval: 1000,\n  vlmUpdateInterval: 10000,\n  tfChangeThreshold: 10,\n  vlmChangeThreshold: 50,\n  visionMode: 'CAMERA' as VisionMode,\n  screenCaptureInterval: 2000,\n  tileSize: 256,\n  tileProcessingOrder: 'priority',\n  ocrEnabled: true,\n  florence2Enabled: true,\n};\n\n// Configuration schema using Zod for validation\nexport const VisionConfigSchema = z.object({\n  // Camera configuration\n  cameraName: z.string().optional(),\n  enableCamera: z.boolean().default(true),\n\n  // Vision processing\n  pixelChangeThreshold: z.number().min(0).max(100).default(50),\n  updateInterval: z.number().min(10).max(10000).default(100),\n\n  // Object detection\n  enableObjectDetection: z.boolean().default(false),\n  objectConfidenceThreshold: z.number().min(0).max(1).default(0.5),\n\n  // Pose detection\n  enablePoseDetection: z.boolean().default(false),\n  poseConfidenceThreshold: z.number().min(0).max(1).default(0.5),\n\n  // Update intervals\n  tfUpdateInterval: z.number().min(100).max(60000).default(1000),\n  vlmUpdateInterval: z.number().min(1000).max(300000).default(10000),\n  tfChangeThreshold: z.number().min(0).max(100).default(10),\n  vlmChangeThreshold: z.number().min(0).max(100).default(50),\n\n  // Vision mode\n  visionMode: z.enum(['OFF', 'CAMERA', 'SCREEN', 'BOTH']).default('CAMERA'),\n\n  // Screen capture\n  screenCaptureInterval: z.number().min(100).max(60000).default(2000),\n  tileSize: z.number().min(64).max(1024).default(256),\n  tileProcessingOrder: z.enum(['sequential', 'priority', 'random']).default('priority'),\n  maxConcurrentTiles: z.number().min(1).max(10).default(3),\n\n  // OCR configuration\n  ocrEnabled: z.boolean().default(true),\n  ocrLanguage: z.string().default('eng'),\n  ocrConfidenceThreshold: z.number().min(0).max(100).default(60),\n\n  // Florence-2 configuration\n  florence2Enabled: z.boolean().default(true),\n  florence2Provider: z.enum(['local', 'azure', 'huggingface', 'replicate']).optional(),\n  florence2Endpoint: z.string().url().optional(),\n  florence2ApiKey: z.string().optional(),\n  florence2Timeout: z.number().min(1000).max(300000).default(30000),\n\n  // Face recognition\n  enableFaceRecognition: z.boolean().default(false),\n  faceMatchThreshold: z.number().min(0).max(1).default(0.6),\n  maxFaceProfiles: z.number().min(10).max(10000).default(1000),\n\n  // Entity tracking\n  entityTimeout: z.number().min(1000).max(300000).default(30000),\n  maxTrackedEntities: z.number().min(10).max(1000).default(100),\n\n  // Performance\n  enableGPUAcceleration: z.boolean().default(true),\n  maxMemoryUsageMB: z.number().min(100).max(8000).default(2000),\n\n  // Logging\n  debugMode: z.boolean().default(false),\n  logLevel: z.enum(['error', 'warn', 'info', 'debug']).default('info'),\n});\n\nexport type VisionConfigInput = z.input<typeof VisionConfigSchema>;\nexport type VisionConfigOutput = z.output<typeof VisionConfigSchema>;\n\nexport class ConfigurationManager {\n  private config: VisionConfigOutput;\n  private runtime: any;\n\n  constructor(runtime: any) {\n    this.runtime = runtime;\n    this.config = this.loadConfiguration();\n  }\n\n  private loadConfiguration(): VisionConfigOutput {\n    // Load from environment variables and runtime settings\n    const rawConfig: Partial<VisionConfigInput> = {\n      // Camera\n      cameraName: this.getSetting('CAMERA_NAME') || this.getSetting('VISION_CAMERA_NAME'),\n      enableCamera: this.getBooleanSetting('ENABLE_CAMERA', true),\n\n      // Vision processing\n      pixelChangeThreshold: this.getNumberSetting('PIXEL_CHANGE_THRESHOLD', 50),\n      updateInterval: this.getNumberSetting('UPDATE_INTERVAL', 100),\n\n      // Object detection\n      enableObjectDetection: this.getBooleanSetting('ENABLE_OBJECT_DETECTION', false),\n      objectConfidenceThreshold: this.getNumberSetting('OBJECT_CONFIDENCE_THRESHOLD', 0.5),\n\n      // Pose detection\n      enablePoseDetection: this.getBooleanSetting('ENABLE_POSE_DETECTION', false),\n      poseConfidenceThreshold: this.getNumberSetting('POSE_CONFIDENCE_THRESHOLD', 0.5),\n\n      // Update intervals\n      tfUpdateInterval: this.getNumberSetting('TF_UPDATE_INTERVAL', 1000),\n      vlmUpdateInterval: this.getNumberSetting('VLM_UPDATE_INTERVAL', 10000),\n      tfChangeThreshold: this.getNumberSetting('TF_CHANGE_THRESHOLD', 10),\n      vlmChangeThreshold: this.getNumberSetting('VLM_CHANGE_THRESHOLD', 50),\n\n      // Vision mode\n      visionMode: this.getSetting('VISION_MODE') as VisionMode,\n\n      // Screen capture\n      screenCaptureInterval: this.getNumberSetting('SCREEN_CAPTURE_INTERVAL', 2000),\n      tileSize: this.getNumberSetting('TILE_SIZE', 256),\n      tileProcessingOrder: this.getSetting('TILE_PROCESSING_ORDER') as any,\n      maxConcurrentTiles: this.getNumberSetting('MAX_CONCURRENT_TILES', 3),\n\n      // OCR\n      ocrEnabled: this.getBooleanSetting('OCR_ENABLED', true),\n      ocrLanguage: this.getSetting('OCR_LANGUAGE') || 'eng',\n      ocrConfidenceThreshold: this.getNumberSetting('OCR_CONFIDENCE_THRESHOLD', 60),\n\n      // Florence-2\n      florence2Enabled: this.getBooleanSetting('FLORENCE2_ENABLED', true),\n      florence2Provider: this.getSetting('FLORENCE2_PROVIDER') as any,\n      florence2Endpoint: this.getSetting('FLORENCE2_ENDPOINT'),\n      florence2ApiKey: this.getSetting('FLORENCE2_API_KEY'),\n      florence2Timeout: this.getNumberSetting('FLORENCE2_TIMEOUT', 30000),\n\n      // Face recognition\n      enableFaceRecognition: this.getBooleanSetting('ENABLE_FACE_RECOGNITION', false),\n      faceMatchThreshold: this.getNumberSetting('FACE_MATCH_THRESHOLD', 0.6),\n      maxFaceProfiles: this.getNumberSetting('MAX_FACE_PROFILES', 1000),\n\n      // Entity tracking\n      entityTimeout: this.getNumberSetting('ENTITY_TIMEOUT', 30000),\n      maxTrackedEntities: this.getNumberSetting('MAX_TRACKED_ENTITIES', 100),\n\n      // Performance\n      enableGPUAcceleration: this.getBooleanSetting('ENABLE_GPU_ACCELERATION', true),\n      maxMemoryUsageMB: this.getNumberSetting('MAX_MEMORY_USAGE_MB', 2000),\n\n      // Logging\n      debugMode: this.getBooleanSetting('DEBUG_MODE', false),\n      logLevel: (this.getSetting('LOG_LEVEL') as any) || 'info',\n    };\n\n    // Validate and parse configuration\n    try {\n      const parsed = VisionConfigSchema.parse(rawConfig);\n      logger.info('[ConfigurationManager] Configuration loaded successfully');\n\n      if (parsed.debugMode) {\n        logger.debug('[ConfigurationManager] Configuration:', parsed);\n      }\n\n      return parsed;\n    } catch (error) {\n      logger.error('[ConfigurationManager] Invalid configuration:', error);\n      if (error instanceof z.ZodError) {\n        logger.error('[ConfigurationManager] Validation errors:', error.errors);\n      }\n\n      // Return default configuration on error\n      return VisionConfigSchema.parse({});\n    }\n  }\n\n  private getSetting(key: string): string | undefined {\n    // Try with VISION_ prefix first\n    const visionKey = `VISION_${key}`;\n    const value = this.runtime.getSetting(visionKey) || this.runtime.getSetting(key);\n    return value || undefined;\n  }\n\n  private getBooleanSetting(key: string, defaultValue: boolean): boolean {\n    const value = this.getSetting(key);\n    if (value === undefined) {\n      return defaultValue;\n    }\n    return value.toLowerCase() === 'true';\n  }\n\n  private getNumberSetting(key: string, defaultValue: number): number {\n    const value = this.getSetting(key);\n    if (value === undefined) {\n      return defaultValue;\n    }\n    const parsed = Number(value);\n    return isNaN(parsed) ? defaultValue : parsed;\n  }\n\n  // Public API\n  get(): VisionConfigOutput {\n    return { ...this.config };\n  }\n\n  update(updates: Partial<VisionConfigInput>): void {\n    try {\n      const newConfig = { ...this.config, ...updates };\n      const parsed = VisionConfigSchema.parse(newConfig);\n      this.config = parsed;\n      logger.info('[ConfigurationManager] Configuration updated');\n    } catch (error) {\n      logger.error('[ConfigurationManager] Failed to update configuration:', error);\n      throw error;\n    }\n  }\n\n  // Convert to legacy VisionConfig format for backward compatibility\n  toLegacyFormat(): VisionConfig {\n    return {\n      cameraName: this.config.cameraName,\n      pixelChangeThreshold: this.config.pixelChangeThreshold,\n      updateInterval: this.config.updateInterval,\n      enableObjectDetection: this.config.enableObjectDetection,\n      enablePoseDetection: this.config.enablePoseDetection,\n      tfUpdateInterval: this.config.tfUpdateInterval,\n      vlmUpdateInterval: this.config.vlmUpdateInterval,\n      tfChangeThreshold: this.config.tfChangeThreshold,\n      vlmChangeThreshold: this.config.vlmChangeThreshold,\n      visionMode: this.config.visionMode as VisionMode,\n      screenCaptureInterval: this.config.screenCaptureInterval,\n      tileSize: this.config.tileSize,\n      tileProcessingOrder: this.config.tileProcessingOrder,\n      ocrEnabled: this.config.ocrEnabled,\n      florence2Enabled: this.config.florence2Enabled,\n    };\n  }\n\n  // Configuration presets\n  static getPreset(name: string): Partial<VisionConfigInput> {\n    const presets: Record<string, Partial<VisionConfigInput>> = {\n      'high-performance': {\n        updateInterval: 50,\n        tfUpdateInterval: 500,\n        vlmUpdateInterval: 5000,\n        enableGPUAcceleration: true,\n        maxConcurrentTiles: 5,\n      },\n      'low-resource': {\n        updateInterval: 200,\n        tfUpdateInterval: 2000,\n        vlmUpdateInterval: 20000,\n        enableObjectDetection: false,\n        enablePoseDetection: false,\n        maxMemoryUsageMB: 500,\n        maxConcurrentTiles: 1,\n      },\n      'security-monitoring': {\n        enableObjectDetection: true,\n        enablePoseDetection: true,\n        enableFaceRecognition: true,\n        updateInterval: 100,\n        entityTimeout: 60000,\n      },\n      'screen-reader': {\n        visionMode: 'SCREEN',\n        ocrEnabled: true,\n        florence2Enabled: true,\n        screenCaptureInterval: 1000,\n        tileProcessingOrder: 'priority',\n      },\n    };\n\n    return presets[name] || {};\n  }\n}\n"]}

package/dist/entity-tracker.d.ts

@@ -0,0 +1,32 @@
+import { type IAgentRuntime } from '@elizaos/core';
+import type { TrackedEntity, WorldState, DetectedObject, PersonInfo } from './types';
+export declare class EntityTracker {
+    private worldState;
+    private readonly POSITION_THRESHOLD;
+    private readonly MISSING_THRESHOLD;
+    private readonly CLEANUP_THRESHOLD;
+    constructor(worldId: string);
+    updateEntities(detectedObjects: DetectedObject[], people: PersonInfo[], faceProfiles?: Map<string, string>, // Maps person ID to face profile ID
+    runtime?: IAgentRuntime): Promise<TrackedEntity[]>;
+    private trackPerson;
+    private trackObject;
+    private findMatchingEntity;
+    private calculateDistance;
+    private updateWorldState;
+    private syncWithRuntime;
+    getWorldState(): WorldState;
+    getActiveEntities(): TrackedEntity[];
+    getEntity(entityId: string): TrackedEntity | undefined;
+    getRecentlyLeft(): Array<{
+        entity: TrackedEntity;
+        leftAt: number;
+    }>;
+    assignNameToEntity(entityId: string, name: string): boolean;
+    getStatistics(): {
+        totalEntities: number;
+        activeEntities: number;
+        recentlyLeft: number;
+        people: number;
+        objects: number;
+    };
+}