@elizaos/plugin-vision 2.0.0-alpha.4 → 2.0.0-alpha.5

package/dist/index.js

@@ -1,4 +1,20 @@
 import { createRequire } from "node:module";
+var __create = Object.create;
+var __getProtoOf = Object.getPrototypeOf;
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __toESM = (mod, isNodeMode, target) => {
+  target = mod != null ? __create(__getProtoOf(mod)) : {};
+  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
+  for (let key of __getOwnPropNames(mod))
+    if (!__hasOwnProp.call(to, key))
+      __defProp(to, key, {
+        get: () => mod[key],
+        enumerable: true
+      });
+  return to;
+};
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // src/action.ts
@@ -2264,8 +2280,19 @@ import { logger as logger8 } from "@elizaos/core";
 
 // src/florence2-local.ts
 import { logger as logger7 } from "@elizaos/core";
-import * as tf from "@tensorflow/tfjs-node";
 import sharp from "sharp";
+var tf = null;
+async function ensureTf() {
+  if (tf)
+    return true;
+  try {
+    tf = await import("@tensorflow/tfjs-node");
+    return true;
+  } catch {
+    logger7.warn("[Florence2Local] TensorFlow.js native addon not available — using heuristic analysis.");
+    return false;
+  }
+}
 
 class Florence2Local {
   model = null;
@@ -2281,6 +2308,11 @@ class Florence2Local {
     if (this.initialized) {
       return;
     }
+    const tfAvailable = await ensureTf();
+    if (!tfAvailable || !tf) {
+      this.initialized = true;
+      return;
+    }
     try {
       logger7.info("[VisionModel] Initializing MobileNet model for image analysis...");
       this.model = await tf.loadGraphModel(this.config.modelUrl);
@@ -2295,22 +2327,22 @@ class Florence2Local {
     if (!this.initialized) {
       await this.initialize();
     }
+    if (!tf || !this.model) {
+      return await this.enhancedFallback(imageBuffer);
+    }
     try {
       const preprocessed = await this.preprocessImage(imageBuffer);
-      if (this.model) {
-        const predictions = await this.runInference(preprocessed);
-        preprocessed.dispose();
-        return this.parseModelOutput(predictions);
-      } else {
-        preprocessed.dispose();
-        return await this.enhancedFallback(imageBuffer);
-      }
+      const predictions = await this.runInference(preprocessed);
+      preprocessed.dispose?.();
+      return this.parseModelOutput(predictions);
     } catch (error) {
       logger7.error("[VisionModel] Analysis failed:", error);
       return await this.enhancedFallback(imageBuffer);
     }
   }
   async preprocessImage(imageBuffer) {
+    if (!tf)
+      throw new Error("TensorFlow.js not available");
     const resized = await sharp(imageBuffer).resize(224, 224).raw().toBuffer();
     const tensor = tf.node.decodeImage(resized, 3);
     const normalized = tf.div(tensor, 255);
@@ -3252,19 +3284,39 @@ class ScreenCaptureService {
 
 // src/vision-models.ts
 import { logger as logger12 } from "@elizaos/core";
-import * as tf2 from "@tensorflow/tfjs-node";
-import * as cocoSsd from "@tensorflow-models/coco-ssd";
-import * as poseDetection from "@tensorflow-models/pose-detection";
+var tf2 = null;
+var cocoSsd = null;
+var poseDetection = null;
+async function loadTfModules() {
+  if (tf2)
+    return true;
+  try {
+    tf2 = await import("@tensorflow/tfjs-node");
+    cocoSsd = await import("@tensorflow-models/coco-ssd");
+    poseDetection = await import("@tensorflow-models/pose-detection");
+    return true;
+  } catch (err) {
+    logger12.warn("[VisionModels] TensorFlow.js native addon not available — " + "falling back to description-based detection. Run `npm rebuild @tensorflow/tfjs-node --build-addon-from-source` to enable hardware-accelerated vision.");
+    return false;
+  }
+}
 
 class VisionModels {
   objectDetectionModel = null;
   poseDetector = null;
   initialized = false;
+  tfAvailable = false;
   async initialize(config) {
     if (this.initialized) {
       return;
     }
     logger12.info("[VisionModels] Initializing vision models...");
+    this.tfAvailable = await loadTfModules();
+    if (!this.tfAvailable || !tf2 || !cocoSsd || !poseDetection) {
+      this.initialized = true;
+      logger12.info("[VisionModels] Initialized without TensorFlow (fallback mode)");
+      return;
+    }
     try {
       await tf2.ready();
       logger12.info("[VisionModels] TensorFlow.js backend ready");
@@ -3308,7 +3360,7 @@ class VisionModels {
     return this.poseDetector !== null;
   }
   async detectObjects(imageData, _width, _height, description) {
-    if (!this.objectDetectionModel) {
+    if (!this.objectDetectionModel || !tf2) {
       logger12.warn("[VisionModels] Object detection model not loaded");
       return this.enhancedObjectDetection(description);
     }
@@ -3394,7 +3446,7 @@ class VisionModels {
     };
   }
   async detectPoses(imageData, width, height, description) {
-    if (!this.poseDetector) {
+    if (!this.poseDetector || !tf2) {
       logger12.warn("[VisionModels] Pose detection model not loaded");
       return this.enhancedPoseDetection(description);
     }
@@ -3601,7 +3653,7 @@ import * as path4 from "node:path";
 import { TextDecoder } from "node:util";
 import { Worker } from "node:worker_threads";
 import { logger as logger13 } from "@elizaos/core";
-var __dirname = "/Users/shawwalters/eliza-ok/plugins/plugin-vision/typescript/src";
+var __dirname = "/Users/shawwalters/eliza-workspace/plugins/plugin-vision/typescript/src";
 
 class VisionWorkerManager {
   config;
@@ -6707,4 +6759,4 @@ export {
   src_default as default
 };
 
-//# debugId=F534BCEA2CE2323664756E2164756E21
+//# debugId=4F137AC378EA73BB64756E2164756E21