@elizaos/plugin-vision 1.2.1

package/dist/face-recognition.js

@@ -0,0 +1,332 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FaceRecognition = void 0;
+var faceapi = require("face-api.js");
+var core_1 = require("@elizaos/core");
+var path = require("path");
+var url_1 = require("url");
+var __filename = (0, url_1.fileURLToPath)(import.meta.url);
+var __dirname = path.dirname(__filename);
+// Dynamic imports for optional dependencies
+var Canvas, Image, ImageData;
+function initializeCanvas() {
+    return __awaiter(this, void 0, void 0, function () {
+        var canvas, error_1;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    _a.trys.push([0, 2, , 3]);
+                    return [4 /*yield*/, Promise.resolve().then(function () { return require('canvas'); })];
+                case 1:
+                    canvas = _a.sent();
+                    Canvas = canvas.Canvas;
+                    Image = canvas.Image;
+                    ImageData = canvas.ImageData;
+                    // Polyfill for face-api.js
+                    faceapi.env.monkeyPatch({ Canvas: Canvas, Image: Image, ImageData: ImageData });
+                    return [3 /*break*/, 3];
+                case 2:
+                    error_1 = _a.sent();
+                    core_1.logger.error('[FaceRecognition] Canvas module not available:', error_1);
+                    throw new Error('Canvas module is required for face recognition. Install with: npm install canvas');
+                case 3: return [2 /*return*/];
+            }
+        });
+    });
+}
+var FaceRecognition = /** @class */ (function () {
+    function FaceRecognition() {
+        this.initialized = false;
+        this.faceLibrary = {
+            faces: new Map(),
+            embeddings: new Map(),
+        };
+        this.modelPath = path.join(__dirname, '..', 'models', 'face-api');
+        // Thresholds
+        this.FACE_MATCH_THRESHOLD = 0.6; // Euclidean distance threshold
+        this.MIN_FACE_SIZE = 50; // Minimum face size in pixels
+    }
+    FaceRecognition.prototype.initialize = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var error_2;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (this.initialized) {
+                            return [2 /*return*/];
+                        }
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 8, , 9]);
+                        core_1.logger.info('[FaceRecognition] Loading face detection models...');
+                        // Initialize canvas first
+                        return [4 /*yield*/, initializeCanvas()];
+                    case 2:
+                        // Initialize canvas first
+                        _a.sent();
+                        // Load face-api.js models
+                        return [4 /*yield*/, faceapi.nets.ssdMobilenetv1.loadFromDisk(this.modelPath)];
+                    case 3:
+                        // Load face-api.js models
+                        _a.sent();
+                        return [4 /*yield*/, faceapi.nets.faceLandmark68Net.loadFromDisk(this.modelPath)];
+                    case 4:
+                        _a.sent();
+                        return [4 /*yield*/, faceapi.nets.faceRecognitionNet.loadFromDisk(this.modelPath)];
+                    case 5:
+                        _a.sent();
+                        return [4 /*yield*/, faceapi.nets.faceExpressionNet.loadFromDisk(this.modelPath)];
+                    case 6:
+                        _a.sent();
+                        return [4 /*yield*/, faceapi.nets.ageGenderNet.loadFromDisk(this.modelPath)];
+                    case 7:
+                        _a.sent();
+                        this.initialized = true;
+                        core_1.logger.info('[FaceRecognition] Models loaded successfully');
+                        return [3 /*break*/, 9];
+                    case 8:
+                        error_2 = _a.sent();
+                        core_1.logger.error('[FaceRecognition] Failed to load models:', error_2);
+                        core_1.logger.info('[FaceRecognition] Download models from: https://github.com/justadudewhohacks/face-api.js-models');
+                        throw error_2;
+                    case 9: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    FaceRecognition.prototype.detectFaces = function (imageData, width, height) {
+        return __awaiter(this, void 0, void 0, function () {
+            var expectedSize, canvas, ctx, imageDataObj, detections, error_3;
+            var _this = this;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!!this.initialized) return [3 /*break*/, 2];
+                        return [4 /*yield*/, this.initialize()];
+                    case 1:
+                        _a.sent();
+                        _a.label = 2;
+                    case 2:
+                        // Validate input parameters
+                        if (!imageData || imageData.length === 0 || width <= 0 || height <= 0) {
+                            core_1.logger.warn('[FaceRecognition] Invalid input parameters:', {
+                                dataLength: (imageData === null || imageData === void 0 ? void 0 : imageData.length) || 0,
+                                width: width,
+                                height: height,
+                            });
+                            return [2 /*return*/, []];
+                        }
+                        expectedSize = width * height * 4;
+                        if (imageData.length !== expectedSize) {
+                            core_1.logger.warn('[FaceRecognition] Buffer size mismatch:', {
+                                expected: expectedSize,
+                                actual: imageData.length,
+                                width: width,
+                                height: height,
+                            });
+                            return [2 /*return*/, []];
+                        }
+                        _a.label = 3;
+                    case 3:
+                        _a.trys.push([3, 5, , 6]);
+                        canvas = new Canvas(width, height);
+                        ctx = canvas.getContext('2d');
+                        imageDataObj = new ImageData(new Uint8ClampedArray(imageData), width, height);
+                        ctx.putImageData(imageDataObj, 0, 0);
+                        return [4 /*yield*/, faceapi
+                                .detectAllFaces(canvas, new faceapi.SsdMobilenetv1Options({
+                                minConfidence: 0.5,
+                                maxResults: 10,
+                            }))
+                                .withFaceLandmarks()
+                                .withFaceDescriptors()
+                                .withFaceExpressions()
+                                .withAgeAndGender()];
+                    case 4:
+                        detections = _a.sent();
+                        return [2 /*return*/, detections.filter(function (d) {
+                                var box = d.detection.box;
+                                return box.width >= _this.MIN_FACE_SIZE && box.height >= _this.MIN_FACE_SIZE;
+                            })];
+                    case 5:
+                        error_3 = _a.sent();
+                        core_1.logger.error('[FaceRecognition] Face detection failed:', error_3);
+                        return [2 /*return*/, []];
+                    case 6: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    FaceRecognition.prototype.recognizeFace = function (descriptor) {
+        return __awaiter(this, void 0, void 0, function () {
+            var bestMatch, minDistance, _i, _a, _b, profileId, embeddings, _c, embeddings_1, knownEmbedding, distance;
+            return __generator(this, function (_d) {
+                bestMatch = null;
+                minDistance = Infinity;
+                // Compare with all known faces
+                for (_i = 0, _a = this.faceLibrary.embeddings; _i < _a.length; _i++) {
+                    _b = _a[_i], profileId = _b[0], embeddings = _b[1];
+                    for (_c = 0, embeddings_1 = embeddings; _c < embeddings_1.length; _c++) {
+                        knownEmbedding = embeddings_1[_c];
+                        distance = this.euclideanDistance(descriptor, knownEmbedding);
+                        if (distance < this.FACE_MATCH_THRESHOLD && distance < minDistance) {
+                            minDistance = distance;
+                            bestMatch = { profileId: profileId, distance: distance };
+                        }
+                    }
+                }
+                return [2 /*return*/, bestMatch];
+            });
+        });
+    };
+    FaceRecognition.prototype.addOrUpdateFace = function (descriptor, attributes) {
+        return __awaiter(this, void 0, void 0, function () {
+            var match, profile, embeddings, profileId, profile;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.recognizeFace(descriptor)];
+                    case 1:
+                        match = _a.sent();
+                        if (match) {
+                            profile = this.faceLibrary.faces.get(match.profileId);
+                            profile.lastSeen = Date.now();
+                            profile.seenCount++;
+                            embeddings = this.faceLibrary.embeddings.get(match.profileId);
+                            if (embeddings.length < 10) {
+                                // Keep up to 10 embeddings per person
+                                embeddings.push(Array.from(descriptor));
+                            }
+                            // Update attributes if provided
+                            if (attributes) {
+                                Object.assign(profile, attributes);
+                            }
+                            return [2 /*return*/, match.profileId];
+                        }
+                        else {
+                            profileId = "face-".concat(Date.now(), "-").concat(Math.random().toString(36).substr(2, 9));
+                            profile = __assign({ id: profileId, embeddings: [Array.from(descriptor)], firstSeen: Date.now(), lastSeen: Date.now(), seenCount: 1 }, attributes);
+                            this.faceLibrary.faces.set(profileId, profile);
+                            this.faceLibrary.embeddings.set(profileId, [Array.from(descriptor)]);
+                            core_1.logger.info("[FaceRecognition] New face registered: ".concat(profileId));
+                            return [2 /*return*/, profileId];
+                        }
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    FaceRecognition.prototype.getFaceProfile = function (profileId) {
+        return this.faceLibrary.faces.get(profileId);
+    };
+    FaceRecognition.prototype.getAllProfiles = function () {
+        return Array.from(this.faceLibrary.faces.values());
+    };
+    FaceRecognition.prototype.euclideanDistance = function (a, b) {
+        var sum = 0;
+        var aArray = Array.from(a);
+        for (var i = 0; i < aArray.length; i++) {
+            sum += Math.pow(aArray[i] - b[i], 2);
+        }
+        return Math.sqrt(sum);
+    };
+    // Persistence methods
+    FaceRecognition.prototype.saveFaceLibrary = function (path) {
+        return __awaiter(this, void 0, void 0, function () {
+            var data, fs;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        data = {
+                            faces: Array.from(this.faceLibrary.faces.entries()),
+                            embeddings: Array.from(this.faceLibrary.embeddings.entries()),
+                        };
+                        return [4 /*yield*/, Promise.resolve().then(function () { return require('fs/promises'); })];
+                    case 1:
+                        fs = _a.sent();
+                        return [4 /*yield*/, fs.writeFile(path, JSON.stringify(data, null, 2))];
+                    case 2:
+                        _a.sent();
+                        core_1.logger.info("[FaceRecognition] Face library saved to ".concat(path));
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    FaceRecognition.prototype.loadFaceLibrary = function (path) {
+        return __awaiter(this, void 0, void 0, function () {
+            var fs, data, _a, _b, error_4;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
+                    case 0:
+                        _c.trys.push([0, 3, , 4]);
+                        return [4 /*yield*/, Promise.resolve().then(function () { return require('fs/promises'); })];
+                    case 1:
+                        fs = _c.sent();
+                        _b = (_a = JSON).parse;
+                        return [4 /*yield*/, fs.readFile(path, 'utf-8')];
+                    case 2:
+                        data = _b.apply(_a, [_c.sent()]);
+                        this.faceLibrary.faces = new Map(data.faces);
+                        this.faceLibrary.embeddings = new Map(data.embeddings);
+                        core_1.logger.info("[FaceRecognition] Loaded ".concat(this.faceLibrary.faces.size, " face profiles"));
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_4 = _c.sent();
+                        core_1.logger.warn('[FaceRecognition] Could not load face library:', error_4);
+                        return [3 /*break*/, 4];
+                    case 4: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    return FaceRecognition;
+}());
+exports.FaceRecognition = FaceRecognition;
+//# sourceMappingURL=face-recognition.js.map

package/dist/face-recognition.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"face-recognition.js","sourceRoot":"","sources":["../src/face-recognition.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAuC;AACvC,sCAAuC;AACvC,2BAA6B;AAC7B,2BAAoC;AAGpC,IAAM,UAAU,GAAG,IAAA,mBAAa,EAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,IAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,4CAA4C;AAC5C,IAAI,MAAW,EAAE,KAAU,EAAE,SAAc,CAAC;AAE5C,SAAe,gBAAgB;;;;;;;oBAEZ,yEAAa,QAAQ,OAAC;;oBAA/B,MAAM,GAAG,SAAsB;oBACrC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;oBACvB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;oBACrB,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;oBAE7B,2BAA2B;oBAC3B,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,MAAM,QAAA,EAAE,KAAK,OAAA,EAAE,SAAS,WAAA,EAAE,CAAC,CAAC;;;;oBAEtD,aAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,OAAK,CAAC,CAAC;oBACtE,MAAM,IAAI,KAAK,CACb,kFAAkF,CACnF,CAAC;;;;;CAEL;AAED;IAAA;QACU,gBAAW,GAAG,KAAK,CAAC;QACpB,gBAAW,GAAgB;YACjC,KAAK,EAAE,IAAI,GAAG,EAAE;YAChB,UAAU,EAAE,IAAI,GAAG,EAAE;SACtB,CAAC;QACM,cAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAErE,aAAa;QACI,yBAAoB,GAAG,GAAG,CAAC,CAAC,+BAA+B;QAC3D,kBAAa,GAAG,EAAE,CAAC,CAAC,8BAA8B;IA2NrE,CAAC;IAzNO,oCAAU,GAAhB;;;;;;wBACE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,sBAAO;wBACT,CAAC;;;;wBAGC,aAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;wBAElE,0BAA0B;wBAC1B,qBAAM,gBAAgB,EAAE,EAAA;;wBADxB,0BAA0B;wBAC1B,SAAwB,CAAC;wBAEzB,0BAA0B;wBAC1B,qBAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAA;;wBAD9D,0BAA0B;wBAC1B,SAA8D,CAAC;wBAC/D,qBAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAA;;wBAAjE,SAAiE,CAAC;wBAClE,qBAAM,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAA;;wBAAlE,SAAkE,CAAC;wBACnE,qBAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAA;;wBAAjE,SAAiE,CAAC;wBAClE,qBAAM,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAA;;wBAA5D,SAA4D,CAAC;wBAE7D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBACxB,aAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;;;;wBAE5D,aAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,OAAK,CAAC,CAAC;wBAChE,aAAM,CAAC,IAAI,CACT,iGAAiG,CAClG,CAAC;wBACF,MAAM,OAAK,CAAC;;;;;KAEf;IAEK,qCAAW,GAAjB,UACE,SAAiB,EACjB,KAAa,EACb,MAAc;;;;;;;6BAUV,CAAC,IAAI,CAAC,WAAW,EAAjB,wBAAiB;wBACnB,qBAAM,IAAI,CAAC,UAAU,EAAE,EAAA;;wBAAvB,SAAuB,CAAC;;;wBAG1B,4BAA4B;wBAC5B,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;4BACtE,aAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE;gCACzD,UAAU,EAAE,CAAA,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,KAAI,CAAC;gCAClC,KAAK,OAAA;gCACL,MAAM,QAAA;6BACP,CAAC,CAAC;4BACH,sBAAO,EAAE,EAAC;wBACZ,CAAC;wBAGK,YAAY,GAAG,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC;wBACxC,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;4BACtC,aAAM,CAAC,IAAI,CAAC,yCAAyC,EAAE;gCACrD,QAAQ,EAAE,YAAY;gCACtB,MAAM,EAAE,SAAS,CAAC,MAAM;gCACxB,KAAK,OAAA;gCACL,MAAM,QAAA;6BACP,CAAC,CAAC;4BACH,sBAAO,EAAE,EAAC;wBACZ,CAAC;;;;wBAIO,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;wBACnC,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;wBAC9B,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,iBAAiB,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;wBACpF,GAAG,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;wBAGlB,qBAAM,OAAO;iCAC7B,cAAc,CACb,MAAa,EACb,IAAI,OAAO,CAAC,qBAAqB,CAAC;gCAChC,aAAa,EAAE,GAAG;gCAClB,UAAU,EAAE,EAAE;6BACf,CAAC,CACH;iCACA,iBAAiB,EAAE;iCACnB,mBAAmB,EAAE;iCACrB,mBAAmB,EAAE;iCACrB,gBAAgB,EAAE,EAAA;;wBAXf,UAAU,GAAG,SAWE;wBAErB,sBAAO,UAAU,CAAC,MAAM,CACtB,UAAC,CAOA;gCACC,IAAM,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC;gCAC5B,OAAO,GAAG,CAAC,KAAK,IAAI,KAAI,CAAC,aAAa,IAAI,GAAG,CAAC,MAAM,IAAI,KAAI,CAAC,aAAa,CAAC;4BAC7E,CAAC,CACF,EAAC;;;wBAEF,aAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,OAAK,CAAC,CAAC;wBAChE,sBAAO,EAAE,EAAC;;;;;KAEb;IAEK,uCAAa,GAAnB,UACE,UAAwB;;;;gBAEpB,SAAS,GAAmD,IAAI,CAAC;gBACjE,WAAW,GAAG,QAAQ,CAAC;gBAE3B,+BAA+B;gBAC/B,WAAiE,EAA3B,KAAA,IAAI,CAAC,WAAW,CAAC,UAAU,EAA3B,cAA2B,EAA3B,IAA2B,EAAE,CAAC;oBAAzD,WAAuB,EAAtB,SAAS,QAAA,EAAE,UAAU,QAAA;oBAC/B,WAAuC,EAAV,yBAAU,EAAV,wBAAU,EAAV,IAAU,EAAE,CAAC;wBAA/B,cAAc;wBACjB,QAAQ,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;wBAEpE,IAAI,QAAQ,GAAG,IAAI,CAAC,oBAAoB,IAAI,QAAQ,GAAG,WAAW,EAAE,CAAC;4BACnE,WAAW,GAAG,QAAQ,CAAC;4BACvB,SAAS,GAAG,EAAE,SAAS,WAAA,EAAE,QAAQ,UAAA,EAAE,CAAC;wBACtC,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,sBAAO,SAAS,EAAC;;;KAClB;IAEK,yCAAe,GAArB,UACE,UAAwB,EACxB,UAAiC;;;;;4BAGnB,qBAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAA;;wBAA5C,KAAK,GAAG,SAAoC;wBAElD,IAAI,KAAK,EAAE,CAAC;4BAEJ,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAE,CAAC;4BAC7D,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;4BAC9B,OAAO,CAAC,SAAS,EAAE,CAAC;4BAGd,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAE,CAAC;4BACrE,IAAI,UAAU,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gCAC3B,sCAAsC;gCACtC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;4BAC1C,CAAC;4BAED,gCAAgC;4BAChC,IAAI,UAAU,EAAE,CAAC;gCACf,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;4BACrC,CAAC;4BAED,sBAAO,KAAK,CAAC,SAAS,EAAC;wBACzB,CAAC;6BAAM,CAAC;4BAEA,SAAS,GAAG,eAAQ,IAAI,CAAC,GAAG,EAAE,cAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAE,CAAC;4BAC5E,OAAO,cACX,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EACpC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EACrB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EACpB,SAAS,EAAE,CAAC,IACT,UAAU,CACd,CAAC;4BAEF,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;4BAC/C,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;4BAErE,aAAM,CAAC,IAAI,CAAC,iDAA0C,SAAS,CAAE,CAAC,CAAC;4BACnE,sBAAO,SAAS,EAAC;wBACnB,CAAC;;;;;KACF;IAED,wCAAc,GAAd,UAAe,SAAiB;QAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;IAED,wCAAc,GAAd;QACE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACrD,CAAC;IAEO,2CAAiB,GAAzB,UAA0B,CAA0B,EAAE,CAAW;QAC/D,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAED,sBAAsB;IAChB,yCAAe,GAArB,UAAsB,IAAY;;;;;;wBAC1B,IAAI,GAAG;4BACX,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;4BACnD,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;yBAC9D,CAAC;wBAES,yEAAa,aAAa,OAAC;;wBAAhC,EAAE,GAAG,SAA2B;wBACtC,qBAAM,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAA;;wBAAvD,SAAuD,CAAC;wBACxD,aAAM,CAAC,IAAI,CAAC,kDAA2C,IAAI,CAAE,CAAC,CAAC;;;;;KAChE;IAEK,yCAAe,GAArB,UAAsB,IAAY;;;;;;;wBAEnB,yEAAa,aAAa,OAAC;;wBAAhC,EAAE,GAAG,SAA2B;wBACzB,KAAA,CAAA,KAAA,IAAI,CAAA,CAAC,KAAK,CAAA;wBAAC,qBAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,EAAA;;wBAAlD,IAAI,GAAG,cAAW,SAAgC,EAAC;wBAEzD,IAAI,CAAC,WAAW,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC7C,IAAI,CAAC,WAAW,CAAC,UAAU,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;wBAEvD,aAAM,CAAC,IAAI,CAAC,mCAA4B,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,mBAAgB,CAAC,CAAC;;;;wBAErF,aAAM,CAAC,IAAI,CAAC,gDAAgD,EAAE,OAAK,CAAC,CAAC;;;;;;KAExE;IACH,sBAAC;AAAD,CAAC,AArOD,IAqOC;AArOY,0CAAe","sourcesContent":["import * as faceapi from 'face-api.js';\nimport { logger } from '@elizaos/core';\nimport * as path from 'path';\nimport { fileURLToPath } from 'url';\nimport type { FaceProfile, FaceLibrary } from './types';\n\nconst __filename = fileURLToPath(import.meta.url);\nconst __dirname = path.dirname(__filename);\n\n// Dynamic imports for optional dependencies\nlet Canvas: any, Image: any, ImageData: any;\n\nasync function initializeCanvas() {\n  try {\n    const canvas = await import('canvas');\n    Canvas = canvas.Canvas;\n    Image = canvas.Image;\n    ImageData = canvas.ImageData;\n\n    // Polyfill for face-api.js\n    faceapi.env.monkeyPatch({ Canvas, Image, ImageData });\n  } catch (error) {\n    logger.error('[FaceRecognition] Canvas module not available:', error);\n    throw new Error(\n      'Canvas module is required for face recognition. Install with: npm install canvas'\n    );\n  }\n}\n\nexport class FaceRecognition {\n  private initialized = false;\n  private faceLibrary: FaceLibrary = {\n    faces: new Map(),\n    embeddings: new Map(),\n  };\n  private modelPath = path.join(__dirname, '..', 'models', 'face-api');\n\n  // Thresholds\n  private readonly FACE_MATCH_THRESHOLD = 0.6; // Euclidean distance threshold\n  private readonly MIN_FACE_SIZE = 50; // Minimum face size in pixels\n\n  async initialize(): Promise<void> {\n    if (this.initialized) {\n      return;\n    }\n\n    try {\n      logger.info('[FaceRecognition] Loading face detection models...');\n\n      // Initialize canvas first\n      await initializeCanvas();\n\n      // Load face-api.js models\n      await faceapi.nets.ssdMobilenetv1.loadFromDisk(this.modelPath);\n      await faceapi.nets.faceLandmark68Net.loadFromDisk(this.modelPath);\n      await faceapi.nets.faceRecognitionNet.loadFromDisk(this.modelPath);\n      await faceapi.nets.faceExpressionNet.loadFromDisk(this.modelPath);\n      await faceapi.nets.ageGenderNet.loadFromDisk(this.modelPath);\n\n      this.initialized = true;\n      logger.info('[FaceRecognition] Models loaded successfully');\n    } catch (error) {\n      logger.error('[FaceRecognition] Failed to load models:', error);\n      logger.info(\n        '[FaceRecognition] Download models from: https://github.com/justadudewhohacks/face-api.js-models'\n      );\n      throw error;\n    }\n  }\n\n  async detectFaces(\n    imageData: Buffer,\n    width: number,\n    height: number\n  ): Promise<\n    Array<{\n      detection: faceapi.FaceDetection;\n      landmarks: faceapi.FaceLandmarks68;\n      descriptor: Float32Array;\n      expressions: faceapi.FaceExpressions;\n      ageGender?: { age: number; gender: string; genderProbability: number };\n    }>\n  > {\n    if (!this.initialized) {\n      await this.initialize();\n    }\n\n    // Validate input parameters\n    if (!imageData || imageData.length === 0 || width <= 0 || height <= 0) {\n      logger.warn('[FaceRecognition] Invalid input parameters:', {\n        dataLength: imageData?.length || 0,\n        width,\n        height,\n      });\n      return [];\n    }\n\n    // Validate that the buffer size matches expected dimensions\n    const expectedSize = width * height * 4; // RGBA\n    if (imageData.length !== expectedSize) {\n      logger.warn('[FaceRecognition] Buffer size mismatch:', {\n        expected: expectedSize,\n        actual: imageData.length,\n        width,\n        height,\n      });\n      return [];\n    }\n\n    try {\n      // Create canvas from image data\n      const canvas = new Canvas(width, height);\n      const ctx = canvas.getContext('2d');\n      const imageDataObj = new ImageData(new Uint8ClampedArray(imageData), width, height);\n      ctx.putImageData(imageDataObj, 0, 0);\n\n      // Detect faces with full analysis\n      const detections = await faceapi\n        .detectAllFaces(\n          canvas as any,\n          new faceapi.SsdMobilenetv1Options({\n            minConfidence: 0.5,\n            maxResults: 10,\n          })\n        )\n        .withFaceLandmarks()\n        .withFaceDescriptors()\n        .withFaceExpressions()\n        .withAgeAndGender();\n\n      return detections.filter(\n        (d: {\n          detection: {\n            box: {\n              width: number;\n              height: number;\n            };\n          };\n        }) => {\n          const box = d.detection.box;\n          return box.width >= this.MIN_FACE_SIZE && box.height >= this.MIN_FACE_SIZE;\n        }\n      );\n    } catch (error) {\n      logger.error('[FaceRecognition] Face detection failed:', error);\n      return [];\n    }\n  }\n\n  async recognizeFace(\n    descriptor: Float32Array\n  ): Promise<{ profileId: string; distance: number } | null> {\n    let bestMatch: { profileId: string; distance: number } | null = null;\n    let minDistance = Infinity;\n\n    // Compare with all known faces\n    for (const [profileId, embeddings] of this.faceLibrary.embeddings) {\n      for (const knownEmbedding of embeddings) {\n        const distance = this.euclideanDistance(descriptor, knownEmbedding);\n\n        if (distance < this.FACE_MATCH_THRESHOLD && distance < minDistance) {\n          minDistance = distance;\n          bestMatch = { profileId, distance };\n        }\n      }\n    }\n\n    return bestMatch;\n  }\n\n  async addOrUpdateFace(\n    descriptor: Float32Array,\n    attributes?: Partial<FaceProfile>\n  ): Promise<string> {\n    // Check if this face already exists\n    const match = await this.recognizeFace(descriptor);\n\n    if (match) {\n      // Update existing profile\n      const profile = this.faceLibrary.faces.get(match.profileId)!;\n      profile.lastSeen = Date.now();\n      profile.seenCount++;\n\n      // Add new embedding for better recognition\n      const embeddings = this.faceLibrary.embeddings.get(match.profileId)!;\n      if (embeddings.length < 10) {\n        // Keep up to 10 embeddings per person\n        embeddings.push(Array.from(descriptor));\n      }\n\n      // Update attributes if provided\n      if (attributes) {\n        Object.assign(profile, attributes);\n      }\n\n      return match.profileId;\n    } else {\n      // Create new profile\n      const profileId = `face-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;\n      const profile: FaceProfile = {\n        id: profileId,\n        embeddings: [Array.from(descriptor)],\n        firstSeen: Date.now(),\n        lastSeen: Date.now(),\n        seenCount: 1,\n        ...attributes,\n      };\n\n      this.faceLibrary.faces.set(profileId, profile);\n      this.faceLibrary.embeddings.set(profileId, [Array.from(descriptor)]);\n\n      logger.info(`[FaceRecognition] New face registered: ${profileId}`);\n      return profileId;\n    }\n  }\n\n  getFaceProfile(profileId: string): FaceProfile | undefined {\n    return this.faceLibrary.faces.get(profileId);\n  }\n\n  getAllProfiles(): FaceProfile[] {\n    return Array.from(this.faceLibrary.faces.values());\n  }\n\n  private euclideanDistance(a: Float32Array | number[], b: number[]): number {\n    let sum = 0;\n    const aArray = Array.from(a);\n    for (let i = 0; i < aArray.length; i++) {\n      sum += Math.pow(aArray[i] - b[i], 2);\n    }\n    return Math.sqrt(sum);\n  }\n\n  // Persistence methods\n  async saveFaceLibrary(path: string): Promise<void> {\n    const data = {\n      faces: Array.from(this.faceLibrary.faces.entries()),\n      embeddings: Array.from(this.faceLibrary.embeddings.entries()),\n    };\n\n    const fs = await import('fs/promises');\n    await fs.writeFile(path, JSON.stringify(data, null, 2));\n    logger.info(`[FaceRecognition] Face library saved to ${path}`);\n  }\n\n  async loadFaceLibrary(path: string): Promise<void> {\n    try {\n      const fs = await import('fs/promises');\n      const data = JSON.parse(await fs.readFile(path, 'utf-8'));\n\n      this.faceLibrary.faces = new Map(data.faces);\n      this.faceLibrary.embeddings = new Map(data.embeddings);\n\n      logger.info(`[FaceRecognition] Loaded ${this.faceLibrary.faces.size} face profiles`);\n    } catch (error) {\n      logger.warn('[FaceRecognition] Could not load face library:', error);\n    }\n  }\n}\n"]}

package/dist/florence2-local.d.ts

@@ -0,0 +1,25 @@
+import type { Florence2Result } from './types';
+interface Florence2LocalConfig {
+    modelPath?: string;
+    modelUrl?: string;
+    cacheDir?: string;
+}
+export declare class Florence2Local {
+    private model;
+    private initialized;
+    private config;
+    private readonly IMAGE_SIZE;
+    private readonly VOCAB_SIZE;
+    constructor(config?: Florence2LocalConfig);
+    initialize(): Promise<void>;
+    analyzeImage(imageBuffer: Buffer): Promise<Florence2Result>;
+    private preprocessImage;
+    private runInference;
+    private parseModelOutput;
+    private generateCaptionFromFeatures;
+    private extractTagsFromCaption;
+    private enhancedFallback;
+    isInitialized(): boolean;
+    dispose(): Promise<void>;
+}
+export {};