@elizaos/plugin-vision 2.0.0-alpha.3 → 2.0.0-alpha.5

package/dist/index.js

@@ -1,4 +1,20 @@
 import { createRequire } from "node:module";
+var __create = Object.create;
+var __getProtoOf = Object.getPrototypeOf;
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __toESM = (mod, isNodeMode, target) => {
+  target = mod != null ? __create(__getProtoOf(mod)) : {};
+  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
+  for (let key of __getOwnPropNames(mod))
+    if (!__hasOwnProp.call(to, key))
+      __defProp(to, key, {
+        get: () => mod[key],
+        enumerable: true
+      });
+  return to;
+};
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // src/action.ts
@@ -2264,8 +2280,19 @@ import { logger as logger8 } from "@elizaos/core";
 
 // src/florence2-local.ts
 import { logger as logger7 } from "@elizaos/core";
-import * as tf from "@tensorflow/tfjs-node";
 import sharp from "sharp";
+var tf = null;
+async function ensureTf() {
+  if (tf)
+    return true;
+  try {
+    tf = await import("@tensorflow/tfjs-node");
+    return true;
+  } catch {
+    logger7.warn("[Florence2Local] TensorFlow.js native addon not available — using heuristic analysis.");
+    return false;
+  }
+}
 
 class Florence2Local {
   model = null;
@@ -2281,6 +2308,11 @@ class Florence2Local {
     if (this.initialized) {
       return;
     }
+    const tfAvailable = await ensureTf();
+    if (!tfAvailable || !tf) {
+      this.initialized = true;
+      return;
+    }
     try {
       logger7.info("[VisionModel] Initializing MobileNet model for image analysis...");
       this.model = await tf.loadGraphModel(this.config.modelUrl);
@@ -2295,22 +2327,22 @@ class Florence2Local {
     if (!this.initialized) {
       await this.initialize();
     }
+    if (!tf || !this.model) {
+      return await this.enhancedFallback(imageBuffer);
+    }
     try {
       const preprocessed = await this.preprocessImage(imageBuffer);
-      if (this.model) {
-        const predictions = await this.runInference(preprocessed);
-        preprocessed.dispose();
-        return this.parseModelOutput(predictions);
-      } else {
-        preprocessed.dispose();
-        return await this.enhancedFallback(imageBuffer);
-      }
+      const predictions = await this.runInference(preprocessed);
+      preprocessed.dispose?.();
+      return this.parseModelOutput(predictions);
     } catch (error) {
       logger7.error("[VisionModel] Analysis failed:", error);
       return await this.enhancedFallback(imageBuffer);
     }
   }
   async preprocessImage(imageBuffer) {
+    if (!tf)
+      throw new Error("TensorFlow.js not available");
     const resized = await sharp(imageBuffer).resize(224, 224).raw().toBuffer();
     const tensor = tf.node.decodeImage(resized, 3);
     const normalized = tf.div(tensor, 255);
@@ -3252,19 +3284,39 @@ class ScreenCaptureService {
 
 // src/vision-models.ts
 import { logger as logger12 } from "@elizaos/core";
-import * as tf2 from "@tensorflow/tfjs-node";
-import * as cocoSsd from "@tensorflow-models/coco-ssd";
-import * as poseDetection from "@tensorflow-models/pose-detection";
+var tf2 = null;
+var cocoSsd = null;
+var poseDetection = null;
+async function loadTfModules() {
+  if (tf2)
+    return true;
+  try {
+    tf2 = await import("@tensorflow/tfjs-node");
+    cocoSsd = await import("@tensorflow-models/coco-ssd");
+    poseDetection = await import("@tensorflow-models/pose-detection");
+    return true;
+  } catch (err) {
+    logger12.warn("[VisionModels] TensorFlow.js native addon not available — " + "falling back to description-based detection. Run `npm rebuild @tensorflow/tfjs-node --build-addon-from-source` to enable hardware-accelerated vision.");
+    return false;
+  }
+}
 
 class VisionModels {
   objectDetectionModel = null;
   poseDetector = null;
   initialized = false;
+  tfAvailable = false;
   async initialize(config) {
     if (this.initialized) {
       return;
     }
     logger12.info("[VisionModels] Initializing vision models...");
+    this.tfAvailable = await loadTfModules();
+    if (!this.tfAvailable || !tf2 || !cocoSsd || !poseDetection) {
+      this.initialized = true;
+      logger12.info("[VisionModels] Initialized without TensorFlow (fallback mode)");
+      return;
+    }
     try {
       await tf2.ready();
       logger12.info("[VisionModels] TensorFlow.js backend ready");
@@ -3308,7 +3360,7 @@ class VisionModels {
     return this.poseDetector !== null;
   }
   async detectObjects(imageData, _width, _height, description) {
-    if (!this.objectDetectionModel) {
+    if (!this.objectDetectionModel || !tf2) {
       logger12.warn("[VisionModels] Object detection model not loaded");
       return this.enhancedObjectDetection(description);
     }
@@ -3394,7 +3446,7 @@ class VisionModels {
     };
   }
   async detectPoses(imageData, width, height, description) {
-    if (!this.poseDetector) {
+    if (!this.poseDetector || !tf2) {
       logger12.warn("[VisionModels] Pose detection model not loaded");
       return this.enhancedPoseDetection(description);
     }
@@ -3601,7 +3653,7 @@ import * as path4 from "node:path";
 import { TextDecoder } from "node:util";
 import { Worker } from "node:worker_threads";
 import { logger as logger13 } from "@elizaos/core";
-var __dirname = "/Users/shawwalters/eliza-ok/plugins/plugin-vision/typescript/src";
+var __dirname = "/Users/shawwalters/eliza-workspace/plugins/plugin-vision/typescript/src";
 
 class VisionWorkerManager {
   config;
@@ -6041,11 +6093,11 @@ ${Object.entries(captureData.statistics.poseCounts).map(([pose, count]) => `- **
 `) || "- No people detected"}
 
 ## Sample Scene Descriptions
-${captureData.captures.filter((c) => c.scene?.description).slice(0, 5).map((c, _i) => `### Capture ${c.index} (${c.elapsedMs}ms)
-"${c.scene.description}"
-- Change: ${c.scene.changePercentage?.toFixed(1)}%
-- Objects: ${c.scene.objectCount}
-- People: ${c.scene.peopleCount}`).join(`
+${captureData.captures.filter((c) => c.scene?.description).slice(0, 5).map((c, _i) => `### Capture ${c.index} (${c.elapsedMs ?? 0}ms)
+"${c.scene?.description ?? ""}"
+- Change: ${c.scene?.changePercentage?.toFixed(1) ?? "0"}%
+- Objects: ${c.scene?.objectCount ?? 0}
+- People: ${c.scene?.peopleCount ?? 0}`).join(`
 
 `)}
 
@@ -6078,7 +6130,6 @@ ${captureData.captures.filter((c) => c.scene?.description).slice(0, 5).map((c, _
 var vision_capture_log_default = new VisionCaptureLogTestSuite;
 // src/tests/e2e/vision-runtime.ts
 import { logger as logger15 } from "@elizaos/core";
-
 class VisionRuntimeTestSuite {
   name = "vision-runtime-tests";
   description = "Real runtime tests for vision plugin functionality";
@@ -6087,7 +6138,7 @@ class VisionRuntimeTestSuite {
       name: "Vision service initialization",
       fn: async (runtime) => {
         logger15.info("[Test] Testing vision service initialization...");
-        const visionService = runtime.getService("VISION");
+        const visionService = runtime.getService(VisionServiceType.VISION);
         if (!visionService) {
           throw new Error("Vision service not found in runtime");
         }
@@ -6096,7 +6147,7 @@ class VisionRuntimeTestSuite {
         }
         const isActive = visionService.isActive();
         logger15.info(`[Test] Vision service active: ${isActive}`);
-        if (!isActive && runtime.getSetting("VISION_MODE") !== "OFF") {
+        if (!isActive && runtime.getSetting("VISION_MODE") !== "OFF" /* OFF */) {
           throw new Error("Vision service should be active but is not");
         }
         logger15.info("[Test] ✅ Vision service initialization test passed");
@@ -6106,7 +6157,7 @@ class VisionRuntimeTestSuite {
       name: "Scene description functionality",
       fn: async (runtime) => {
         logger15.info("[Test] Testing scene description...");
-        const visionService = runtime.getService("VISION");
+        const visionService = runtime.getService(VisionServiceType.VISION);
         if (!visionService) {
           throw new Error("Vision service not found");
         }
@@ -6136,13 +6187,18 @@ class VisionRuntimeTestSuite {
       name: "Vision mode switching",
       fn: async (runtime) => {
         logger15.info("[Test] Testing vision mode switching...");
-        const visionService = runtime.getService("VISION");
+        const visionService = runtime.getService(VisionServiceType.VISION);
         if (!visionService) {
           throw new Error("Vision service not found");
         }
         const originalMode = visionService.getVisionMode();
         logger15.info(`[Test] Original mode: ${originalMode}`);
-        const testModes = ["CAMERA", "SCREEN", "BOTH", "OFF"];
+        const testModes = [
+          "CAMERA" /* CAMERA */,
+          "SCREEN" /* SCREEN */,
+          "BOTH" /* BOTH */,
+          "OFF" /* OFF */
+        ];
         for (const mode of testModes) {
           logger15.info(`[Test] Switching to mode: ${mode}`);
           await visionService.setVisionMode(mode);
@@ -6185,7 +6241,7 @@ class VisionRuntimeTestSuite {
           }
           return [];
         };
-        await action.handler(runtime, message, {}, {}, callback);
+        await action.handler(runtime, message, { values: {}, data: {}, text: "" }, {}, callback);
         if (!responseReceived) {
           throw new Error("DESCRIBE_SCENE action did not produce a response");
         }
@@ -6226,7 +6282,7 @@ class VisionRuntimeTestSuite {
       name: "Florence-2 model initialization",
       fn: async (runtime) => {
         logger15.info("[Test] Testing Florence-2 model...");
-        const visionService = runtime.getService("VISION");
+        const visionService = runtime.getService(VisionServiceType.VISION);
         if (!visionService) {
           throw new Error("Vision service not found");
         }
@@ -6236,7 +6292,7 @@ class VisionRuntimeTestSuite {
           return;
         }
         const mode = visionService.getVisionMode();
-        if (mode === "SCREEN" || mode === "BOTH") {
+        if (mode === "SCREEN" /* SCREEN */ || mode === "BOTH" /* BOTH */) {
           const screenCapture = await visionService.getScreenCapture();
           if (screenCapture) {
             logger15.info("[Test] Screen capture available");
@@ -6251,7 +6307,7 @@ class VisionRuntimeTestSuite {
       name: "OCR service functionality",
       fn: async (runtime) => {
         logger15.info("[Test] Testing OCR service...");
-        const visionService = runtime.getService("VISION");
+        const visionService = runtime.getService(VisionServiceType.VISION);
         if (!visionService) {
           throw new Error("Vision service not found");
         }
@@ -6261,7 +6317,7 @@ class VisionRuntimeTestSuite {
           return;
         }
         const mode = visionService.getVisionMode();
-        if (mode === "SCREEN" || mode === "BOTH") {
+        if (mode === "SCREEN" /* SCREEN */ || mode === "BOTH" /* BOTH */) {
           const enhancedScene = await visionService.getEnhancedSceneDescription();
           if (enhancedScene?.screenAnalysis) {
             const ocrText = enhancedScene.screenAnalysis.fullScreenOCR;
@@ -6278,7 +6334,7 @@ class VisionRuntimeTestSuite {
       name: "Entity tracking system",
       fn: async (runtime) => {
         logger15.info("[Test] Testing entity tracking...");
-        const visionService = runtime.getService("VISION");
+        const visionService = runtime.getService(VisionServiceType.VISION);
         if (!visionService) {
           throw new Error("Vision service not found");
         }
@@ -6289,10 +6345,10 @@ class VisionRuntimeTestSuite {
         const entities = entityTracker.getActiveEntities();
         logger15.info(`[Test] Active entities: ${entities.length}`);
         for (const entity of entities) {
-          if (!entity.id || !entity.type || !entity.lastSeen) {
+          if (!entity.id || !entity.entityType || !entity.lastSeen) {
             throw new Error("Entity missing required fields");
           }
-          logger15.info(`[Test] Entity ${entity.id}: type=${entity.type}, tracked=${entity.trackingDuration}ms`);
+          logger15.info(`[Test] Entity ${entity.id}: type=${entity.entityType}, tracked=${entity.lastSeen - entity.firstSeen}ms`);
         }
         logger15.info("[Test] ✅ Entity tracking test passed");
       }
@@ -6703,4 +6759,4 @@ export {
   src_default as default
 };
 
-//# debugId=F5001EC904380B0D64756E2164756E21
+//# debugId=4F137AC378EA73BB64756E2164756E21