@elizaos/plugin-vision 1.2.1 → 2.0.0-alpha.1

package/dist/florence2-model.js

@@ -1,503 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
-    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Florence2Model = void 0;
-var core_1 = require("@elizaos/core");
-var florence2_local_1 = require("./florence2-local");
-var Florence2Model = /** @class */ (function () {
-    function Florence2Model() {
-        this.initialized = false;
-        this.localModel = new florence2_local_1.Florence2Local();
-    }
-    Florence2Model.prototype.initialize = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var error_1;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (this.initialized) {
-                            return [2 /*return*/];
-                        }
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 3, , 4]);
-                        core_1.logger.info('[Florence2] Initializing local Florence-2 model with TensorFlow.js...');
-                        return [4 /*yield*/, this.localModel.initialize()];
-                    case 2:
-                        _a.sent();
-                        this.initialized = true;
-                        core_1.logger.info('[Florence2] Local model initialized successfully');
-                        return [3 /*break*/, 4];
-                    case 3:
-                        error_1 = _a.sent();
-                        core_1.logger.error('[Florence2] Failed to initialize local model:', error_1);
-                        // Don't throw - we have good fallbacks
-                        this.initialized = true;
-                        core_1.logger.warn('[Florence2] Running with enhanced fallback mode');
-                        return [3 /*break*/, 4];
-                    case 4: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Florence2Model.prototype.analyzeTile = function (tile) {
-        return __awaiter(this, void 0, void 0, function () {
-            var result_1, _modelError_1, result, error_2;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (!!this.initialized) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.initialize()];
-                    case 1:
-                        _a.sent();
-                        _a.label = 2;
-                    case 2:
-                        if (!tile.data) {
-                            throw new Error('Tile has no image data');
-                        }
-                        _a.label = 3;
-                    case 3:
-                        _a.trys.push([3, 9, , 10]);
-                        _a.label = 4;
-                    case 4:
-                        _a.trys.push([4, 6, , 7]);
-                        return [4 /*yield*/, this.localModel.analyzeImage(tile.data)];
-                    case 5:
-                        result_1 = _a.sent();
-                        core_1.logger.debug("[Florence2] Analyzed tile ".concat(tile.id, ": ").concat(result_1.caption));
-                        return [2 /*return*/, result_1];
-                    case 6:
-                        _modelError_1 = _a.sent();
-                        core_1.logger.warn('[Florence2] Local model analysis failed, falling back:', _modelError_1);
-                        return [3 /*break*/, 7];
-                    case 7: return [4 /*yield*/, this.mockAnalyze(tile)];
-                    case 8:
-                        result = _a.sent();
-                        core_1.logger.debug("[Florence2] Mock analyzed tile ".concat(tile.id, ": ").concat(result.caption));
-                        return [2 /*return*/, result];
-                    case 9:
-                        error_2 = _a.sent();
-                        core_1.logger.error('[Florence2] Analysis failed:', error_2);
-                        throw error_2;
-                    case 10: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Florence2Model.prototype.analyzeImage = function (imageBuffer) {
-        return __awaiter(this, void 0, void 0, function () {
-            var result_2, _modelError_2, result, error_3;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (!!this.initialized) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.initialize()];
-                    case 1:
-                        _a.sent();
-                        _a.label = 2;
-                    case 2:
-                        _a.trys.push([2, 8, , 9]);
-                        _a.label = 3;
-                    case 3:
-                        _a.trys.push([3, 5, , 6]);
-                        return [4 /*yield*/, this.localModel.analyzeImage(imageBuffer)];
-                    case 4:
-                        result_2 = _a.sent();
-                        core_1.logger.debug("[Florence2] Analyzed image: ".concat(result_2.caption));
-                        return [2 /*return*/, result_2];
-                    case 5:
-                        _modelError_2 = _a.sent();
-                        core_1.logger.warn('[Florence2] Local model analysis failed, falling back:', _modelError_2);
-                        return [3 /*break*/, 6];
-                    case 6: return [4 /*yield*/, this.mockAnalyzeBuffer(imageBuffer)];
-                    case 7:
-                        result = _a.sent();
-                        core_1.logger.debug("[Florence2] Mock analyzed image: ".concat(result.caption));
-                        return [2 /*return*/, result];
-                    case 8:
-                        error_3 = _a.sent();
-                        core_1.logger.error('[Florence2] Image analysis failed:', error_3);
-                        throw error_3;
-                    case 9: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Florence2Model.prototype.mockAnalyze = function (tile) {
-        return __awaiter(this, void 0, void 0, function () {
-            var isUpperRegion, isLeftRegion, caption, objects, regions, tags, buttonCount, i, textRegions, i;
-            return __generator(this, function (_a) {
-                isUpperRegion = tile.row < 2;
-                isLeftRegion = tile.col < 2;
-                caption = 'Desktop screen region';
-                objects = [];
-                regions = [];
-                tags = [];
-                if (isUpperRegion) {
-                    caption = 'Application window with menu bar';
-                    objects.push({
-                        label: 'window',
-                        bbox: { x: 0, y: 0, width: tile.width, height: 50 },
-                        confidence: 0.9,
-                    });
-                    objects.push({
-                        label: 'menu_bar',
-                        bbox: { x: 0, y: 0, width: tile.width, height: 30 },
-                        confidence: 0.85,
-                    });
-                    tags.push('ui', 'application', 'desktop');
-                }
-                if (isLeftRegion) {
-                    caption = 'Sidebar or navigation area';
-                    objects.push({
-                        label: 'sidebar',
-                        bbox: { x: 0, y: 0, width: 100, height: tile.height },
-                        confidence: 0.8,
-                    });
-                    tags.push('navigation', 'sidebar');
-                }
-                buttonCount = Math.floor(Math.random() * 3) + 1;
-                for (i = 0; i < buttonCount; i++) {
-                    objects.push({
-                        label: 'button',
-                        bbox: {
-                            x: Math.random() * (tile.width - 100),
-                            y: Math.random() * (tile.height - 40),
-                            width: 100,
-                            height: 40,
-                        },
-                        confidence: 0.7 + Math.random() * 0.2,
-                    });
-                }
-                textRegions = Math.floor(Math.random() * 2) + 1;
-                for (i = 0; i < textRegions; i++) {
-                    regions.push({
-                        description: 'Text content area',
-                        bbox: {
-                            x: Math.random() * (tile.width - 200),
-                            y: Math.random() * (tile.height - 100),
-                            width: 200,
-                            height: 100,
-                        },
-                    });
-                }
-                tags.push('screen', 'interface', 'computer');
-                return [2 /*return*/, {
-                        caption: caption,
-                        objects: objects,
-                        regions: regions,
-                        tags: tags,
-                    }];
-            });
-        });
-    };
-    Florence2Model.prototype.detectUIElements = function (imageBuffer) {
-        return __awaiter(this, void 0, void 0, function () {
-            var result, _modelError_3, error_4;
-            var _this = this;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        if (!!this.initialized) return [3 /*break*/, 2];
-                        return [4 /*yield*/, this.initialize()];
-                    case 1:
-                        _a.sent();
-                        _a.label = 2;
-                    case 2:
-                        _a.trys.push([2, 8, , 9]);
-                        result = void 0;
-                        _a.label = 3;
-                    case 3:
-                        _a.trys.push([3, 5, , 7]);
-                        return [4 /*yield*/, this.localModel.analyzeImage(imageBuffer)];
-                    case 4:
-                        result = _a.sent();
-                        return [3 /*break*/, 7];
-                    case 5:
-                        _modelError_3 = _a.sent();
-                        core_1.logger.warn('[Florence2] Local model failed for UI detection, using fallback');
-                        return [4 /*yield*/, this.mockAnalyzeBuffer(imageBuffer)];
-                    case 6:
-                        result = _a.sent();
-                        return [3 /*break*/, 7];
-                    case 7: 
-                    // Convert Florence-2 objects to UI elements
-                    return [2 /*return*/, (result.objects || []).map(function (obj) { return ({
-                            type: _this.mapToUIElementType(obj.label),
-                            bbox: obj.bbox,
-                            confidence: obj.confidence,
-                        }); })];
-                    case 8:
-                        error_4 = _a.sent();
-                        core_1.logger.error('[Florence2] UI element detection failed:', error_4);
-                        return [2 /*return*/, []];
-                    case 9: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Florence2Model.prototype.mockAnalyzeBuffer = function (_imageBuffer) {
-        return __awaiter(this, void 0, void 0, function () {
-            var scenarios, scenario;
-            return __generator(this, function (_a) {
-                scenarios = [
-                    {
-                        caption: 'Indoor scene with a person in front of a computer',
-                        objects: [
-                            {
-                                label: 'person',
-                                bbox: { x: 300, y: 200, width: 200, height: 300 },
-                                confidence: 0.9,
-                            },
-                            {
-                                label: 'computer',
-                                bbox: { x: 400, y: 350, width: 150, height: 100 },
-                                confidence: 0.85,
-                            },
-                            {
-                                label: 'desk',
-                                bbox: { x: 350, y: 400, width: 250, height: 100 },
-                                confidence: 0.8,
-                            },
-                        ],
-                        tags: ['indoor', 'office', 'workspace', 'person', 'computer'],
-                    },
-                    {
-                        caption: 'Room interior with furniture and lighting',
-                        objects: [
-                            {
-                                label: 'chair',
-                                bbox: { x: 200, y: 300, width: 100, height: 150 },
-                                confidence: 0.85,
-                            },
-                            {
-                                label: 'table',
-                                bbox: { x: 350, y: 350, width: 150, height: 100 },
-                                confidence: 0.8,
-                            },
-                            {
-                                label: 'lamp',
-                                bbox: { x: 500, y: 200, width: 50, height: 100 },
-                                confidence: 0.75,
-                            },
-                        ],
-                        tags: ['indoor', 'room', 'furniture', 'interior'],
-                    },
-                    {
-                        caption: 'Person working at a desk with computer monitor',
-                        objects: [
-                            {
-                                label: 'person',
-                                bbox: { x: 250, y: 150, width: 250, height: 350 },
-                                confidence: 0.92,
-                            },
-                            {
-                                label: 'monitor',
-                                bbox: { x: 450, y: 300, width: 120, height: 80 },
-                                confidence: 0.88,
-                            },
-                            {
-                                label: 'keyboard',
-                                bbox: { x: 430, y: 380, width: 100, height: 30 },
-                                confidence: 0.82,
-                            },
-                        ],
-                        tags: ['person', 'working', 'computer', 'desk', 'office'],
-                    },
-                ];
-                scenario = scenarios[Math.floor(Math.random() * scenarios.length)];
-                return [2 /*return*/, {
-                        caption: scenario.caption,
-                        objects: scenario.objects,
-                        regions: [],
-                        tags: scenario.tags,
-                    }];
-            });
-        });
-    };
-    Florence2Model.prototype.mapToUIElementType = function (label) {
-        var mapping = {
-            button: 'button',
-            text_field: 'input',
-            text_area: 'textarea',
-            checkbox: 'checkbox',
-            radio_button: 'radio',
-            dropdown: 'select',
-            menu: 'menu',
-            menu_bar: 'menubar',
-            toolbar: 'toolbar',
-            window: 'window',
-            dialog: 'dialog',
-            icon: 'icon',
-            image: 'image',
-            video: 'video',
-            link: 'link',
-            heading: 'heading',
-            paragraph: 'text',
-            list: 'list',
-            table: 'table',
-            scrollbar: 'scrollbar',
-            tab: 'tab',
-            panel: 'panel',
-        };
-        return mapping[label.toLowerCase()] || 'unknown';
-    };
-    Florence2Model.prototype.generateSceneGraph = function (tiles) {
-        return __awaiter(this, void 0, void 0, function () {
-            var nodes, edges, _i, tiles_1, tile, analysis, _a, _b, obj, nodeId, i, j, relation;
-            return __generator(this, function (_c) {
-                switch (_c.label) {
-                    case 0:
-                        nodes = [];
-                        edges = [];
-                        _i = 0, tiles_1 = tiles;
-                        _c.label = 1;
-                    case 1:
-                        if (!(_i < tiles_1.length)) return [3 /*break*/, 4];
-                        tile = tiles_1[_i];
-                        if (!tile.data) {
-                            return [3 /*break*/, 3];
-                        }
-                        return [4 /*yield*/, this.analyzeTile(tile)];
-                    case 2:
-                        analysis = _c.sent();
-                        // Add objects as nodes
-                        if (analysis.objects) {
-                            for (_a = 0, _b = analysis.objects; _a < _b.length; _a++) {
-                                obj = _b[_a];
-                                nodeId = "".concat(tile.id, "-").concat(obj.label, "-").concat(nodes.length);
-                                nodes.push({
-                                    id: nodeId,
-                                    type: obj.label,
-                                    label: obj.label,
-                                    position: {
-                                        x: tile.x + obj.bbox.x,
-                                        y: tile.y + obj.bbox.y,
-                                        width: obj.bbox.width,
-                                        height: obj.bbox.height,
-                                    },
-                                });
-                            }
-                        }
-                        _c.label = 3;
-                    case 3:
-                        _i++;
-                        return [3 /*break*/, 1];
-                    case 4:
-                        // Infer spatial relationships
-                        for (i = 0; i < nodes.length; i++) {
-                            for (j = i + 1; j < nodes.length; j++) {
-                                relation = this.inferSpatialRelation(nodes[i].position, nodes[j].position);
-                                if (relation) {
-                                    edges.push({
-                                        source: nodes[i].id,
-                                        target: nodes[j].id,
-                                        relation: relation,
-                                    });
-                                }
-                            }
-                        }
-                        return [2 /*return*/, { nodes: nodes, edges: edges }];
-                }
-            });
-        });
-    };
-    Florence2Model.prototype.inferSpatialRelation = function (box1, box2) {
-        var center1 = {
-            x: box1.x + box1.width / 2,
-            y: box1.y + box1.height / 2,
-        };
-        var center2 = {
-            x: box2.x + box2.width / 2,
-            y: box2.y + box2.height / 2,
-        };
-        // Check containment
-        if (this.contains(box1, box2)) {
-            return 'contains';
-        }
-        if (this.contains(box2, box1)) {
-            return 'contained_by';
-        }
-        // Check overlap
-        if (this.overlaps(box1, box2)) {
-            return 'overlaps';
-        }
-        // Check adjacency and direction
-        var dx = center2.x - center1.x;
-        var dy = center2.y - center1.y;
-        var distance = Math.sqrt(dx * dx + dy * dy);
-        if (distance < 100) {
-            // Close proximity
-            if (Math.abs(dx) > Math.abs(dy)) {
-                return dx > 0 ? 'right_of' : 'left_of';
-            }
-            else {
-                return dy > 0 ? 'below' : 'above';
-            }
-        }
-        return null;
-    };
-    Florence2Model.prototype.contains = function (box1, box2) {
-        return (box1.x <= box2.x &&
-            box1.y <= box2.y &&
-            box1.x + box1.width >= box2.x + box2.width &&
-            box1.y + box1.height >= box2.y + box2.height);
-    };
-    Florence2Model.prototype.overlaps = function (box1, box2) {
-        return !(box1.x + box1.width < box2.x ||
-            box2.x + box2.width < box1.x ||
-            box1.y + box1.height < box2.y ||
-            box2.y + box2.height < box1.y);
-    };
-    Florence2Model.prototype.isInitialized = function () {
-        return this.initialized;
-    };
-    Florence2Model.prototype.dispose = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            return __generator(this, function (_a) {
-                // Clean up resources if needed
-                this.initialized = false;
-                core_1.logger.info('[Florence2] Model disposed');
-                return [2 /*return*/];
-            });
-        });
-    };
-    return Florence2Model;
-}());
-exports.Florence2Model = Florence2Model;
-//# sourceMappingURL=florence2-model.js.map

package/dist/florence2-model.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"florence2-model.js","sourceRoot":"","sources":["../src/florence2-model.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sCAAuC;AAEvC,qDAAmD;AAEnD;IAIE;QAHQ,gBAAW,GAAG,KAAK,CAAC;QAI1B,IAAI,CAAC,UAAU,GAAG,IAAI,gCAAc,EAAE,CAAC;IACzC,CAAC;IAEK,mCAAU,GAAhB;;;;;;wBACE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,sBAAO;wBACT,CAAC;;;;wBAGC,aAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;wBAErF,qBAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,EAAA;;wBAAlC,SAAkC,CAAC;wBAEnC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBACxB,aAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;;;;wBAEhE,aAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE,OAAK,CAAC,CAAC;wBACrE,uCAAuC;wBACvC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBACxB,aAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;;;;;;KAElE;IAEK,oCAAW,GAAjB,UAAkB,IAAgB;;;;;;6BAC5B,CAAC,IAAI,CAAC,WAAW,EAAjB,wBAAiB;wBACnB,qBAAM,IAAI,CAAC,UAAU,EAAE,EAAA;;wBAAvB,SAAuB,CAAC;;;wBAG1B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;4BACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;wBAC5C,CAAC;;;;;;;wBAKkB,qBAAM,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAA;;wBAAtD,WAAS,SAA6C;wBAC5D,aAAM,CAAC,KAAK,CAAC,oCAA6B,IAAI,CAAC,EAAE,eAAK,QAAM,CAAC,OAAO,CAAE,CAAC,CAAC;wBACxE,sBAAO,QAAM,EAAC;;;wBAEd,aAAM,CAAC,IAAI,CAAC,wDAAwD,EAAE,aAAW,CAAC,CAAC;;4BAItE,qBAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAA;;wBAArC,MAAM,GAAG,SAA4B;wBAC3C,aAAM,CAAC,KAAK,CAAC,yCAAkC,IAAI,CAAC,EAAE,eAAK,MAAM,CAAC,OAAO,CAAE,CAAC,CAAC;wBAC7E,sBAAO,MAAM,EAAC;;;wBAEd,aAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,OAAK,CAAC,CAAC;wBACpD,MAAM,OAAK,CAAC;;;;;KAEf;IAEK,qCAAY,GAAlB,UAAmB,WAAmB;;;;;;6BAChC,CAAC,IAAI,CAAC,WAAW,EAAjB,wBAAiB;wBACnB,qBAAM,IAAI,CAAC,UAAU,EAAE,EAAA;;wBAAvB,SAAuB,CAAC;;;;;;;wBAMP,qBAAM,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,WAAW,CAAC,EAAA;;wBAAxD,WAAS,SAA+C;wBAC9D,aAAM,CAAC,KAAK,CAAC,sCAA+B,QAAM,CAAC,OAAO,CAAE,CAAC,CAAC;wBAC9D,sBAAO,QAAM,EAAC;;;wBAEd,aAAM,CAAC,IAAI,CAAC,wDAAwD,EAAE,aAAW,CAAC,CAAC;;4BAItE,qBAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAA;;wBAAlD,MAAM,GAAG,SAAyC;wBACxD,aAAM,CAAC,KAAK,CAAC,2CAAoC,MAAM,CAAC,OAAO,CAAE,CAAC,CAAC;wBACnE,sBAAO,MAAM,EAAC;;;wBAEd,aAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,OAAK,CAAC,CAAC;wBAC1D,MAAM,OAAK,CAAC;;;;;KAEf;IAEa,oCAAW,GAAzB,UAA0B,IAAgB;;;;gBAIlC,aAAa,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;gBAC7B,YAAY,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;gBAG9B,OAAO,GAAG,uBAAuB,CAAC;gBAChC,OAAO,GAAoE,EAAE,CAAC;gBAC9E,OAAO,GAAsD,EAAE,CAAC;gBAChE,IAAI,GAAa,EAAE,CAAC;gBAE1B,IAAI,aAAa,EAAE,CAAC;oBAClB,OAAO,GAAG,kCAAkC,CAAC;oBAC7C,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,QAAQ;wBACf,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;wBACnD,UAAU,EAAE,GAAG;qBAChB,CAAC,CAAC;oBACH,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,UAAU;wBACjB,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;wBACnD,UAAU,EAAE,IAAI;qBACjB,CAAC,CAAC;oBACH,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;gBAC5C,CAAC;gBAED,IAAI,YAAY,EAAE,CAAC;oBACjB,OAAO,GAAG,4BAA4B,CAAC;oBACvC,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,SAAS;wBAChB,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;wBACrD,UAAU,EAAE,GAAG;qBAChB,CAAC,CAAC;oBACH,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;gBACrC,CAAC;gBAGK,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;gBACtD,KAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,QAAQ;wBACf,IAAI,EAAE;4BACJ,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;4BACrC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;4BACrC,KAAK,EAAE,GAAG;4BACV,MAAM,EAAE,EAAE;yBACX;wBACD,UAAU,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG;qBACtC,CAAC,CAAC;gBACL,CAAC;gBAGK,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;gBACtD,KAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,CAAC;wBACX,WAAW,EAAE,mBAAmB;wBAChC,IAAI,EAAE;4BACJ,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;4BACrC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;4BACtC,KAAK,EAAE,GAAG;4BACV,MAAM,EAAE,GAAG;yBACZ;qBACF,CAAC,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;gBAE7C,sBAAO;wBACL,OAAO,SAAA;wBACP,OAAO,SAAA;wBACP,OAAO,SAAA;wBACP,IAAI,MAAA;qBACL,EAAC;;;KACH;IAEK,yCAAgB,GAAtB,UAAuB,WAAmB;;;;;;;6BAQpC,CAAC,IAAI,CAAC,WAAW,EAAjB,wBAAiB;wBACnB,qBAAM,IAAI,CAAC,UAAU,EAAE,EAAA;;wBAAvB,SAAuB,CAAC;;;;wBAKpB,MAAM,SAAiB,CAAC;;;;wBAGjB,qBAAM,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,WAAW,CAAC,EAAA;;wBAAxD,MAAM,GAAG,SAA+C,CAAC;;;;wBAEzD,aAAM,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;wBACtE,qBAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,EAAA;;wBAAlD,MAAM,GAAG,SAAyC,CAAC;;;oBAGrD,4CAA4C;oBAC5C,sBAAO,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,UAAC,GAAG,IAAK,OAAA,CAAC;4BAC1C,IAAI,EAAE,KAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC;4BACxC,IAAI,EAAE,GAAG,CAAC,IAAI;4BACd,UAAU,EAAE,GAAG,CAAC,UAAU;yBAC3B,CAAC,EAJyC,CAIzC,CAAC,EAAC;;;wBAEJ,aAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,OAAK,CAAC,CAAC;wBAChE,sBAAO,EAAE,EAAC;;;;;KAEb;IAEa,0CAAiB,GAA/B,UAAgC,YAAoB;;;;gBAI5C,SAAS,GAAG;oBAChB;wBACE,OAAO,EAAE,mDAAmD;wBAC5D,OAAO,EAAE;4BACP;gCACE,KAAK,EAAE,QAAQ;gCACf,IAAI,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;gCACjD,UAAU,EAAE,GAAG;6BAChB;4BACD;gCACE,KAAK,EAAE,UAAU;gCACjB,IAAI,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;gCACjD,UAAU,EAAE,IAAI;6BACjB;4BACD;gCACE,KAAK,EAAE,MAAM;gCACb,IAAI,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;gCACjD,UAAU,EAAE,GAAG;6BAChB;yBACF;wBACD,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC;qBAC9D;oBACD;wBACE,OAAO,EAAE,2CAA2C;wBACpD,OAAO,EAAE;4BACP;gCACE,KAAK,EAAE,OAAO;gCACd,IAAI,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;gCACjD,UAAU,EAAE,IAAI;6BACjB;4BACD;gCACE,KAAK,EAAE,OAAO;gCACd,IAAI,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;gCACjD,UAAU,EAAE,GAAG;6BAChB;4BACD;gCACE,KAAK,EAAE,MAAM;gCACb,IAAI,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE;gCAChD,UAAU,EAAE,IAAI;6BACjB;yBACF;wBACD,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,CAAC;qBAClD;oBACD;wBACE,OAAO,EAAE,gDAAgD;wBACzD,OAAO,EAAE;4BACP;gCACE,KAAK,EAAE,QAAQ;gCACf,IAAI,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;gCACjD,UAAU,EAAE,IAAI;6BACjB;4BACD;gCACE,KAAK,EAAE,SAAS;gCAChB,IAAI,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE;gCAChD,UAAU,EAAE,IAAI;6BACjB;4BACD;gCACE,KAAK,EAAE,UAAU;gCACjB,IAAI,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE;gCAChD,UAAU,EAAE,IAAI;6BACjB;yBACF;wBACD,IAAI,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC;qBAC1D;iBACF,CAAC;gBAGI,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;gBAEzE,sBAAO;wBACL,OAAO,EAAE,QAAQ,CAAC,OAAO;wBACzB,OAAO,EAAE,QAAQ,CAAC,OAAO;wBACzB,OAAO,EAAE,EAAE;wBACX,IAAI,EAAE,QAAQ,CAAC,IAAI;qBACpB,EAAC;;;KACH;IAEO,2CAAkB,GAA1B,UAA2B,KAAa;QACtC,IAAM,OAAO,GAA2B;YACtC,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,OAAO;YACnB,SAAS,EAAE,UAAU;YACrB,QAAQ,EAAE,UAAU;YACpB,YAAY,EAAE,OAAO;YACrB,QAAQ,EAAE,QAAQ;YAClB,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,SAAS;YACnB,OAAO,EAAE,SAAS;YAClB,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,OAAO;YACd,KAAK,EAAE,OAAO;YACd,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,MAAM;YACjB,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,OAAO;YACd,SAAS,EAAE,WAAW;YACtB,GAAG,EAAE,KAAK;YACV,KAAK,EAAE,OAAO;SACf,CAAC;QAEF,OAAO,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,SAAS,CAAC;IACnD,CAAC;IAEK,2CAAkB,GAAxB,UAAyB,KAAmB;;;;;;wBAIpC,KAAK,GAA8E,EAAE,CAAC;wBACtF,KAAK,GAAgE,EAAE,CAAC;8BAGtD,EAAL,eAAK;;;6BAAL,CAAA,mBAAK,CAAA;wBAAb,IAAI;wBACb,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;4BACf,wBAAS;wBACX,CAAC;wBAEgB,qBAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAA;;wBAAvC,QAAQ,GAAG,SAA4B;wBAE7C,uBAAuB;wBACvB,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;4BACrB,WAAkC,EAAhB,KAAA,QAAQ,CAAC,OAAO,EAAhB,cAAgB,EAAhB,IAAgB,EAAE,CAAC;gCAA1B,GAAG;gCACN,MAAM,GAAG,UAAG,IAAI,CAAC,EAAE,cAAI,GAAG,CAAC,KAAK,cAAI,KAAK,CAAC,MAAM,CAAE,CAAC;gCACzD,KAAK,CAAC,IAAI,CAAC;oCACT,EAAE,EAAE,MAAM;oCACV,IAAI,EAAE,GAAG,CAAC,KAAK;oCACf,KAAK,EAAE,GAAG,CAAC,KAAK;oCAChB,QAAQ,EAAE;wCACR,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;wCACtB,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;wCACtB,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK;wCACrB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM;qCACxB;iCACF,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;;;wBAvBgB,IAAK,CAAA;;;wBA0BxB,8BAA8B;wBAC9B,KAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;4BACtC,KAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gCACpC,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gCACjF,IAAI,QAAQ,EAAE,CAAC;oCACb,KAAK,CAAC,IAAI,CAAC;wCACT,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;wCACnB,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;wCACnB,QAAQ,UAAA;qCACT,CAAC,CAAC;gCACL,CAAC;4BACH,CAAC;wBACH,CAAC;wBAED,sBAAO,EAAE,KAAK,OAAA,EAAE,KAAK,OAAA,EAAE,EAAC;;;;KACzB;IAEO,6CAAoB,GAA5B,UAA6B,IAAiB,EAAE,IAAiB;QAC/D,IAAM,OAAO,GAAG;YACd,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;YAC1B,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;SAC5B,CAAC;QACF,IAAM,OAAO,GAAG;YACd,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;YAC1B,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;SAC5B,CAAC;QAEF,oBAAoB;QACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAC9B,OAAO,UAAU,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAC9B,OAAO,cAAc,CAAC;QACxB,CAAC;QAED,gBAAgB;QAChB,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAC9B,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,gCAAgC;QAChC,IAAM,EAAE,GAAG,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;QACjC,IAAM,EAAE,GAAG,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;QACjC,IAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAE9C,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YACnB,kBAAkB;YAClB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAChC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;YACpC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iCAAQ,GAAhB,UAAiB,IAAiB,EAAE,IAAiB;QACnD,OAAO,CACL,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;YAChB,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;YAChB,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK;YAC1C,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAC7C,CAAC;IACJ,CAAC;IAEO,iCAAQ,GAAhB,UAAiB,IAAiB,EAAE,IAAiB;QACnD,OAAO,CAAC,CACN,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;YAC5B,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;YAC5B,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;YAC7B,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAC9B,CAAC;IACJ,CAAC;IAED,sCAAa,GAAb;QACE,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAEK,gCAAO,GAAb;;;gBACE,+BAA+B;gBAC/B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;gBACzB,aAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;;;;KAC3C;IACH,qBAAC;AAAD,CAAC,AAtaD,IAsaC;AAtaY,wCAAc","sourcesContent":["import { logger } from '@elizaos/core';\nimport type { Florence2Result, ScreenTile, BoundingBox } from './types';\nimport { Florence2Local } from './florence2-local';\n\nexport class Florence2Model {\n  private initialized = false;\n  private localModel: Florence2Local;\n\n  constructor() {\n    this.localModel = new Florence2Local();\n  }\n\n  async initialize(): Promise<void> {\n    if (this.initialized) {\n      return;\n    }\n\n    try {\n      logger.info('[Florence2] Initializing local Florence-2 model with TensorFlow.js...');\n\n      await this.localModel.initialize();\n\n      this.initialized = true;\n      logger.info('[Florence2] Local model initialized successfully');\n    } catch (error) {\n      logger.error('[Florence2] Failed to initialize local model:', error);\n      // Don't throw - we have good fallbacks\n      this.initialized = true;\n      logger.warn('[Florence2] Running with enhanced fallback mode');\n    }\n  }\n\n  async analyzeTile(tile: ScreenTile): Promise<Florence2Result> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    if (!tile.data) {\n      throw new Error('Tile has no image data');\n    }\n\n    try {\n      // Use local model\n      try {\n        const result = await this.localModel.analyzeImage(tile.data);\n        logger.debug(`[Florence2] Analyzed tile ${tile.id}: ${result.caption}`);\n        return result;\n      } catch (_modelError) {\n        logger.warn('[Florence2] Local model analysis failed, falling back:', _modelError);\n      }\n\n      // Fall back to mock analysis\n      const result = await this.mockAnalyze(tile);\n      logger.debug(`[Florence2] Mock analyzed tile ${tile.id}: ${result.caption}`);\n      return result;\n    } catch (error) {\n      logger.error('[Florence2] Analysis failed:', error);\n      throw error;\n    }\n  }\n\n  async analyzeImage(imageBuffer: Buffer): Promise<Florence2Result> {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    try {\n      // Use local model\n      try {\n        const result = await this.localModel.analyzeImage(imageBuffer);\n        logger.debug(`[Florence2] Analyzed image: ${result.caption}`);\n        return result;\n      } catch (_modelError) {\n        logger.warn('[Florence2] Local model analysis failed, falling back:', _modelError);\n      }\n\n      // Fall back to mock analysis\n      const result = await this.mockAnalyzeBuffer(imageBuffer);\n      logger.debug(`[Florence2] Mock analyzed image: ${result.caption}`);\n      return result;\n    } catch (error) {\n      logger.error('[Florence2] Image analysis failed:', error);\n      throw error;\n    }\n  }\n\n  private async mockAnalyze(tile: ScreenTile): Promise<Florence2Result> {\n    // Mock implementation that simulates Florence-2 output\n    // In production, this would be replaced with actual API calls\n\n    const isUpperRegion = tile.row < 2;\n    const isLeftRegion = tile.col < 2;\n\n    // Simulate different UI regions\n    let caption = 'Desktop screen region';\n    const objects: Array<{ label: string; bbox: BoundingBox; confidence: number }> = [];\n    const regions: Array<{ description: string; bbox: BoundingBox }> = [];\n    const tags: string[] = [];\n\n    if (isUpperRegion) {\n      caption = 'Application window with menu bar';\n      objects.push({\n        label: 'window',\n        bbox: { x: 0, y: 0, width: tile.width, height: 50 },\n        confidence: 0.9,\n      });\n      objects.push({\n        label: 'menu_bar',\n        bbox: { x: 0, y: 0, width: tile.width, height: 30 },\n        confidence: 0.85,\n      });\n      tags.push('ui', 'application', 'desktop');\n    }\n\n    if (isLeftRegion) {\n      caption = 'Sidebar or navigation area';\n      objects.push({\n        label: 'sidebar',\n        bbox: { x: 0, y: 0, width: 100, height: tile.height },\n        confidence: 0.8,\n      });\n      tags.push('navigation', 'sidebar');\n    }\n\n    // Add some common UI elements\n    const buttonCount = Math.floor(Math.random() * 3) + 1;\n    for (let i = 0; i < buttonCount; i++) {\n      objects.push({\n        label: 'button',\n        bbox: {\n          x: Math.random() * (tile.width - 100),\n          y: Math.random() * (tile.height - 40),\n          width: 100,\n          height: 40,\n        },\n        confidence: 0.7 + Math.random() * 0.2,\n      });\n    }\n\n    // Add text regions\n    const textRegions = Math.floor(Math.random() * 2) + 1;\n    for (let i = 0; i < textRegions; i++) {\n      regions.push({\n        description: 'Text content area',\n        bbox: {\n          x: Math.random() * (tile.width - 200),\n          y: Math.random() * (tile.height - 100),\n          width: 200,\n          height: 100,\n        },\n      });\n    }\n\n    tags.push('screen', 'interface', 'computer');\n\n    return {\n      caption,\n      objects,\n      regions,\n      tags,\n    };\n  }\n\n  async detectUIElements(imageBuffer: Buffer): Promise<\n    Array<{\n      type: string;\n      bbox: BoundingBox;\n      confidence: number;\n      text?: string;\n    }>\n  > {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    try {\n      // Use local model to analyze image\n      let result: Florence2Result;\n\n      try {\n        result = await this.localModel.analyzeImage(imageBuffer);\n      } catch (_modelError) {\n        logger.warn('[Florence2] Local model failed for UI detection, using fallback');\n        result = await this.mockAnalyzeBuffer(imageBuffer);\n      }\n\n      // Convert Florence-2 objects to UI elements\n      return (result.objects || []).map((obj) => ({\n        type: this.mapToUIElementType(obj.label),\n        bbox: obj.bbox,\n        confidence: obj.confidence,\n      }));\n    } catch (error) {\n      logger.error('[Florence2] UI element detection failed:', error);\n      return [];\n    }\n  }\n\n  private async mockAnalyzeBuffer(_imageBuffer: Buffer): Promise<Florence2Result> {\n    // Enhanced mock for when API is not available\n    // Provides more realistic descriptions based on common scenarios\n\n    const scenarios = [\n      {\n        caption: 'Indoor scene with a person in front of a computer',\n        objects: [\n          {\n            label: 'person',\n            bbox: { x: 300, y: 200, width: 200, height: 300 },\n            confidence: 0.9,\n          },\n          {\n            label: 'computer',\n            bbox: { x: 400, y: 350, width: 150, height: 100 },\n            confidence: 0.85,\n          },\n          {\n            label: 'desk',\n            bbox: { x: 350, y: 400, width: 250, height: 100 },\n            confidence: 0.8,\n          },\n        ],\n        tags: ['indoor', 'office', 'workspace', 'person', 'computer'],\n      },\n      {\n        caption: 'Room interior with furniture and lighting',\n        objects: [\n          {\n            label: 'chair',\n            bbox: { x: 200, y: 300, width: 100, height: 150 },\n            confidence: 0.85,\n          },\n          {\n            label: 'table',\n            bbox: { x: 350, y: 350, width: 150, height: 100 },\n            confidence: 0.8,\n          },\n          {\n            label: 'lamp',\n            bbox: { x: 500, y: 200, width: 50, height: 100 },\n            confidence: 0.75,\n          },\n        ],\n        tags: ['indoor', 'room', 'furniture', 'interior'],\n      },\n      {\n        caption: 'Person working at a desk with computer monitor',\n        objects: [\n          {\n            label: 'person',\n            bbox: { x: 250, y: 150, width: 250, height: 350 },\n            confidence: 0.92,\n          },\n          {\n            label: 'monitor',\n            bbox: { x: 450, y: 300, width: 120, height: 80 },\n            confidence: 0.88,\n          },\n          {\n            label: 'keyboard',\n            bbox: { x: 430, y: 380, width: 100, height: 30 },\n            confidence: 0.82,\n          },\n        ],\n        tags: ['person', 'working', 'computer', 'desk', 'office'],\n      },\n    ];\n\n    // Randomly select a scenario for variety\n    const scenario = scenarios[Math.floor(Math.random() * scenarios.length)];\n\n    return {\n      caption: scenario.caption,\n      objects: scenario.objects,\n      regions: [],\n      tags: scenario.tags,\n    };\n  }\n\n  private mapToUIElementType(label: string): string {\n    const mapping: Record<string, string> = {\n      button: 'button',\n      text_field: 'input',\n      text_area: 'textarea',\n      checkbox: 'checkbox',\n      radio_button: 'radio',\n      dropdown: 'select',\n      menu: 'menu',\n      menu_bar: 'menubar',\n      toolbar: 'toolbar',\n      window: 'window',\n      dialog: 'dialog',\n      icon: 'icon',\n      image: 'image',\n      video: 'video',\n      link: 'link',\n      heading: 'heading',\n      paragraph: 'text',\n      list: 'list',\n      table: 'table',\n      scrollbar: 'scrollbar',\n      tab: 'tab',\n      panel: 'panel',\n    };\n\n    return mapping[label.toLowerCase()] || 'unknown';\n  }\n\n  async generateSceneGraph(tiles: ScreenTile[]): Promise<{\n    nodes: Array<{ id: string; type: string; label: string; position: BoundingBox }>;\n    edges: Array<{ source: string; target: string; relation: string }>;\n  }> {\n    const nodes: Array<{ id: string; type: string; label: string; position: BoundingBox }> = [];\n    const edges: Array<{ source: string; target: string; relation: string }> = [];\n\n    // Analyze each tile\n    for (const tile of tiles) {\n      if (!tile.data) {\n        continue;\n      }\n\n      const analysis = await this.analyzeTile(tile);\n\n      // Add objects as nodes\n      if (analysis.objects) {\n        for (const obj of analysis.objects) {\n          const nodeId = `${tile.id}-${obj.label}-${nodes.length}`;\n          nodes.push({\n            id: nodeId,\n            type: obj.label,\n            label: obj.label,\n            position: {\n              x: tile.x + obj.bbox.x,\n              y: tile.y + obj.bbox.y,\n              width: obj.bbox.width,\n              height: obj.bbox.height,\n            },\n          });\n        }\n      }\n    }\n\n    // Infer spatial relationships\n    for (let i = 0; i < nodes.length; i++) {\n      for (let j = i + 1; j < nodes.length; j++) {\n        const relation = this.inferSpatialRelation(nodes[i].position, nodes[j].position);\n        if (relation) {\n          edges.push({\n            source: nodes[i].id,\n            target: nodes[j].id,\n            relation,\n          });\n        }\n      }\n    }\n\n    return { nodes, edges };\n  }\n\n  private inferSpatialRelation(box1: BoundingBox, box2: BoundingBox): string | null {\n    const center1 = {\n      x: box1.x + box1.width / 2,\n      y: box1.y + box1.height / 2,\n    };\n    const center2 = {\n      x: box2.x + box2.width / 2,\n      y: box2.y + box2.height / 2,\n    };\n\n    // Check containment\n    if (this.contains(box1, box2)) {\n      return 'contains';\n    }\n    if (this.contains(box2, box1)) {\n      return 'contained_by';\n    }\n\n    // Check overlap\n    if (this.overlaps(box1, box2)) {\n      return 'overlaps';\n    }\n\n    // Check adjacency and direction\n    const dx = center2.x - center1.x;\n    const dy = center2.y - center1.y;\n    const distance = Math.sqrt(dx * dx + dy * dy);\n\n    if (distance < 100) {\n      // Close proximity\n      if (Math.abs(dx) > Math.abs(dy)) {\n        return dx > 0 ? 'right_of' : 'left_of';\n      } else {\n        return dy > 0 ? 'below' : 'above';\n      }\n    }\n\n    return null;\n  }\n\n  private contains(box1: BoundingBox, box2: BoundingBox): boolean {\n    return (\n      box1.x <= box2.x &&\n      box1.y <= box2.y &&\n      box1.x + box1.width >= box2.x + box2.width &&\n      box1.y + box1.height >= box2.y + box2.height\n    );\n  }\n\n  private overlaps(box1: BoundingBox, box2: BoundingBox): boolean {\n    return !(\n      box1.x + box1.width < box2.x ||\n      box2.x + box2.width < box1.x ||\n      box1.y + box1.height < box2.y ||\n      box2.y + box2.height < box1.y\n    );\n  }\n\n  isInitialized(): boolean {\n    return this.initialized;\n  }\n\n  async dispose(): Promise<void> {\n    // Clean up resources if needed\n    this.initialized = false;\n    logger.info('[Florence2] Model disposed');\n  }\n}\n"]}

package/dist/index.d.ts

@@ -1,3 +0,0 @@
-import { Plugin } from '@elizaos/core';
-export declare const visionPlugin: Plugin;
-export default visionPlugin;

package/dist/ocr-service-real.d.ts

@@ -1,32 +0,0 @@
-import type { OCRResult, ScreenTile, BoundingBox } from './types';
-export declare class RealOCRService {
-    private worker;
-    private initialized;
-    private initPromise;
-    initialize(): Promise<void>;
-    private _initialize;
-    extractText(imageBuffer: Buffer): Promise<OCRResult>;
-    extractFromTile(tile: ScreenTile): Promise<OCRResult>;
-    private preprocessImage;
-    private convertTesseractResult;
-    extractStructuredData(imageBuffer: Buffer): Promise<{
-        tables?: Array<{
-            rows: string[][];
-            bbox: BoundingBox;
-        }>;
-        forms?: Array<{
-            label: string;
-            value: string;
-            bbox: BoundingBox;
-        }>;
-        lists?: Array<{
-            items: string[];
-            bbox: BoundingBox;
-        }>;
-    }>;
-    private detectTables;
-    private detectForms;
-    private detectLists;
-    isInitialized(): boolean;
-    dispose(): Promise<void>;
-}