@elizaos/plugin-vision 2.0.0-alpha.4 → 2.0.0-alpha.6

package/dist/index.js

@@ -1,4 +1,20 @@
 import { createRequire } from "node:module";
+var __create = Object.create;
+var __getProtoOf = Object.getPrototypeOf;
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __toESM = (mod, isNodeMode, target) => {
+  target = mod != null ? __create(__getProtoOf(mod)) : {};
+  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
+  for (let key of __getOwnPropNames(mod))
+    if (!__hasOwnProp.call(to, key))
+      __defProp(to, key, {
+        get: () => mod[key],
+        enumerable: true
+      });
+  return to;
+};
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // src/action.ts
@@ -1836,7 +1852,7 @@ class StreamingAudioCaptureService extends EventEmitter {
 }
 
 // src/entity-tracker.ts
-import { logger as logger5 } from "@elizaos/core";
+import { createUniqueUuid as createUniqueUuid2, logger as logger5 } from "@elizaos/core";
 
 class EntityTracker {
   worldState;
@@ -2014,8 +2030,9 @@ class EntityTracker {
   async syncWithRuntime(runtime, frameEntities) {
     try {
       for (const entity of frameEntities) {
+        const entityUuid = createUniqueUuid2(runtime, entity.id);
         const elizaEntity = {
-          id: entity.id,
+          id: entityUuid,
           names: [entity.attributes.name || entity.id],
           metadata: {
             type: entity.entityType,
@@ -2027,16 +2044,16 @@ class EntityTracker {
           agentId: runtime.agentId
         };
         try {
-          const existing = await runtime.getEntityById(entity.id);
+          const existing = await runtime.getEntityById(entityUuid);
           if (!existing) {
             await runtime.createEntity(elizaEntity);
-            logger5.debug(`[EntityTracker] Created entity ${entity.id} in runtime`);
+            logger5.debug(`[EntityTracker] Created entity ${entity.id} (${entityUuid}) in runtime`);
           } else {
             await runtime.updateEntity({
               ...elizaEntity,
-              id: entity.id
+              id: entityUuid
             });
-            logger5.debug(`[EntityTracker] Updated entity ${entity.id} in runtime`);
+            logger5.debug(`[EntityTracker] Updated entity ${entity.id} (${entityUuid}) in runtime`);
           }
         } catch (err) {
           logger5.debug(`[EntityTracker] Could not sync entity ${entity.id}: ${err instanceof Error ? err.message : String(err)}`);
@@ -2264,8 +2281,19 @@ import { logger as logger8 } from "@elizaos/core";
 
 // src/florence2-local.ts
 import { logger as logger7 } from "@elizaos/core";
-import * as tf from "@tensorflow/tfjs-node";
 import sharp from "sharp";
+var tf = null;
+async function ensureTf() {
+  if (tf)
+    return true;
+  try {
+    tf = await import("@tensorflow/tfjs-node");
+    return true;
+  } catch {
+    logger7.warn("[Florence2Local] TensorFlow.js native addon not available — using heuristic analysis.");
+    return false;
+  }
+}
 
 class Florence2Local {
   model = null;
@@ -2281,6 +2309,11 @@ class Florence2Local {
     if (this.initialized) {
       return;
     }
+    const tfAvailable = await ensureTf();
+    if (!tfAvailable || !tf) {
+      this.initialized = true;
+      return;
+    }
     try {
       logger7.info("[VisionModel] Initializing MobileNet model for image analysis...");
       this.model = await tf.loadGraphModel(this.config.modelUrl);
@@ -2295,22 +2328,22 @@ class Florence2Local {
     if (!this.initialized) {
       await this.initialize();
     }
+    if (!tf || !this.model) {
+      return await this.enhancedFallback(imageBuffer);
+    }
     try {
       const preprocessed = await this.preprocessImage(imageBuffer);
-      if (this.model) {
-        const predictions = await this.runInference(preprocessed);
-        preprocessed.dispose();
-        return this.parseModelOutput(predictions);
-      } else {
-        preprocessed.dispose();
-        return await this.enhancedFallback(imageBuffer);
-      }
+      const predictions = await this.runInference(preprocessed);
+      preprocessed.dispose?.();
+      return this.parseModelOutput(predictions);
     } catch (error) {
       logger7.error("[VisionModel] Analysis failed:", error);
       return await this.enhancedFallback(imageBuffer);
     }
   }
   async preprocessImage(imageBuffer) {
+    if (!tf)
+      throw new Error("TensorFlow.js not available");
     const resized = await sharp(imageBuffer).resize(224, 224).raw().toBuffer();
     const tensor = tf.node.decodeImage(resized, 3);
     const normalized = tf.div(tensor, 255);
@@ -3252,19 +3285,39 @@ class ScreenCaptureService {
 
 // src/vision-models.ts
 import { logger as logger12 } from "@elizaos/core";
-import * as tf2 from "@tensorflow/tfjs-node";
-import * as cocoSsd from "@tensorflow-models/coco-ssd";
-import * as poseDetection from "@tensorflow-models/pose-detection";
+var tf2 = null;
+var cocoSsd = null;
+var poseDetection = null;
+async function loadTfModules() {
+  if (tf2)
+    return true;
+  try {
+    tf2 = await import("@tensorflow/tfjs-node");
+    cocoSsd = await import("@tensorflow-models/coco-ssd");
+    poseDetection = await import("@tensorflow-models/pose-detection");
+    return true;
+  } catch (err) {
+    logger12.warn("[VisionModels] TensorFlow.js native addon not available — " + "falling back to description-based detection. Run `npm rebuild @tensorflow/tfjs-node --build-addon-from-source` to enable hardware-accelerated vision.");
+    return false;
+  }
+}
 
 class VisionModels {
   objectDetectionModel = null;
   poseDetector = null;
   initialized = false;
+  tfAvailable = false;
   async initialize(config) {
     if (this.initialized) {
       return;
     }
     logger12.info("[VisionModels] Initializing vision models...");
+    this.tfAvailable = await loadTfModules();
+    if (!this.tfAvailable || !tf2 || !cocoSsd || !poseDetection) {
+      this.initialized = true;
+      logger12.info("[VisionModels] Initialized without TensorFlow (fallback mode)");
+      return;
+    }
     try {
       await tf2.ready();
       logger12.info("[VisionModels] TensorFlow.js backend ready");
@@ -3308,7 +3361,7 @@ class VisionModels {
     return this.poseDetector !== null;
   }
   async detectObjects(imageData, _width, _height, description) {
-    if (!this.objectDetectionModel) {
+    if (!this.objectDetectionModel || !tf2) {
       logger12.warn("[VisionModels] Object detection model not loaded");
       return this.enhancedObjectDetection(description);
     }
@@ -3394,7 +3447,7 @@ class VisionModels {
     };
   }
   async detectPoses(imageData, width, height, description) {
-    if (!this.poseDetector) {
+    if (!this.poseDetector || !tf2) {
       logger12.warn("[VisionModels] Pose detection model not loaded");
       return this.enhancedPoseDetection(description);
     }
@@ -3601,7 +3654,7 @@ import * as path4 from "node:path";
 import { TextDecoder } from "node:util";
 import { Worker } from "node:worker_threads";
 import { logger as logger13 } from "@elizaos/core";
-var __dirname = "/Users/shawwalters/eliza-ok/plugins/plugin-vision/typescript/src";
+var __dirname = "/Users/shawwalters/eliza-workspace/plugins/plugin-vision/typescript/src";
 
 class VisionWorkerManager {
   config;
@@ -5131,7 +5184,7 @@ class VisionService extends Service {
 }
 
 // src/tests/e2e/screen-vision.ts
-import { createUniqueUuid as createUniqueUuid2 } from "@elizaos/core";
+import { createUniqueUuid as createUniqueUuid3 } from "@elizaos/core";
 class ScreenVisionE2ETestSuite {
   name = "plugin-vision-screen-e2e";
   description = "E2E tests for screen vision functionality including Florence-2 and OCR";
@@ -5250,11 +5303,11 @@ class ScreenVisionE2ETestSuite {
         }
         console.log("  Testing SET_VISION_MODE action...");
         const message = {
-          id: createUniqueUuid2(runtime, "test-msg"),
+          id: createUniqueUuid3(runtime, "test-msg"),
           entityId: runtime.agentId,
           content: { text: "set vision mode to both" },
           agentId: runtime.agentId,
-          roomId: createUniqueUuid2(runtime, "test-room"),
+          roomId: createUniqueUuid3(runtime, "test-room"),
           createdAt: Date.now()
         };
         let callbackCalled = false;
@@ -5303,11 +5356,11 @@ class ScreenVisionE2ETestSuite {
           }
         }
         const state = await runtime.composeState({
-          id: createUniqueUuid2(runtime, "test-msg"),
+          id: createUniqueUuid3(runtime, "test-msg"),
           entityId: runtime.agentId,
           content: { text: "test" },
           agentId: runtime.agentId,
-          roomId: createUniqueUuid2(runtime, "test-room"),
+          roomId: createUniqueUuid3(runtime, "test-room"),
           createdAt: Date.now()
         });
         if (state.text.includes("Vision mode: BOTH")) {
@@ -5343,7 +5396,7 @@ class ScreenVisionE2ETestSuite {
 }
 var screen_vision_default = new ScreenVisionE2ETestSuite;
 // src/tests/e2e/vision-autonomy.ts
-import { createUniqueUuid as createUniqueUuid3 } from "@elizaos/core";
+import { createUniqueUuid as createUniqueUuid4 } from "@elizaos/core";
 class VisionAutonomyE2ETestSuite {
   name = "plugin-vision-autonomy-e2e";
   description = "Tests for vision plugin integration with autonomy plugin";
@@ -5352,9 +5405,9 @@ class VisionAutonomyE2ETestSuite {
       name: "Should stop autonomous loop with kill command",
       fn: async (runtime) => {
         console.log("Testing kill autonomous action...");
-        const roomId = createUniqueUuid3(runtime, "test-room");
+        const roomId = createUniqueUuid4(runtime, "test-room");
         const message = {
-          id: createUniqueUuid3(runtime, "test-msg-kill"),
+          id: createUniqueUuid4(runtime, "test-msg-kill"),
           entityId: runtime.agentId,
           content: { text: "kill the autonomous loop" },
           agentId: runtime.agentId,
@@ -5445,9 +5498,9 @@ class VisionAutonomyE2ETestSuite {
         if (!visionService) {
           throw new Error("Vision service not available");
         }
-        const roomId = createUniqueUuid3(runtime, "test-room");
+        const roomId = createUniqueUuid4(runtime, "test-room");
         const firstMessage = {
-          id: createUniqueUuid3(runtime, "test-msg-1"),
+          id: createUniqueUuid4(runtime, "test-msg-1"),
           entityId: runtime.agentId,
           content: { text: "what do you see?" },
           agentId: runtime.agentId,
@@ -5456,7 +5509,7 @@ class VisionAutonomyE2ETestSuite {
         };
         await runtime.createMemory(firstMessage, "messages");
         const firstResponse = {
-          id: createUniqueUuid3(runtime, "test-response-1"),
+          id: createUniqueUuid4(runtime, "test-response-1"),
           entityId: runtime.agentId,
           content: {
             text: "I see a test scene",
@@ -5469,7 +5522,7 @@ class VisionAutonomyE2ETestSuite {
         await runtime.createMemory(firstResponse, "messages");
         await new Promise((resolve) => setTimeout(resolve, 500));
         const secondMessage = {
-          id: createUniqueUuid3(runtime, "test-msg-2"),
+          id: createUniqueUuid4(runtime, "test-msg-2"),
           entityId: runtime.agentId,
           content: { text: "what did you see before?" },
           agentId: runtime.agentId,
@@ -5478,7 +5531,7 @@ class VisionAutonomyE2ETestSuite {
         };
         await runtime.createMemory(secondMessage, "messages");
         const secondResponse = {
-          id: createUniqueUuid3(runtime, "test-response-2"),
+          id: createUniqueUuid4(runtime, "test-response-2"),
           entityId: runtime.agentId,
           content: {
             text: "Previously, I saw a test scene",
@@ -5506,7 +5559,7 @@ class VisionAutonomyE2ETestSuite {
       name: "Should integrate vision data with agent decision making",
       fn: async (runtime) => {
         console.log("Testing vision-based decision making...");
-        const roomId = createUniqueUuid3(runtime, "test-room");
+        const roomId = createUniqueUuid4(runtime, "test-room");
         const scenarios = [
           { text: "Is anyone in the room?", expectedContext: "people" },
           { text: "Should I turn on the lights?", expectedContext: "scene" },
@@ -5515,7 +5568,7 @@ class VisionAutonomyE2ETestSuite {
         let scenariosWithVision = 0;
         for (const scenario of scenarios) {
           const message = {
-            id: createUniqueUuid3(runtime, `test-msg-${Date.now()}`),
+            id: createUniqueUuid4(runtime, `test-msg-${Date.now()}`),
             entityId: runtime.agentId,
             content: { text: scenario.text },
             agentId: runtime.agentId,
@@ -5550,11 +5603,11 @@ class VisionAutonomyE2ETestSuite {
         const isActive = visionService.isActive();
         console.log(`  Vision service active: ${isActive}`);
         const message = {
-          id: createUniqueUuid3(runtime, "test-msg-no-vision"),
+          id: createUniqueUuid4(runtime, "test-msg-no-vision"),
           entityId: runtime.agentId,
           content: { text: "test without vision" },
           agentId: runtime.agentId,
-          roomId: createUniqueUuid3(runtime, "test-room"),
+          roomId: createUniqueUuid4(runtime, "test-room"),
           createdAt: Date.now()
         };
         const state = await runtime.composeState(message);
@@ -5583,7 +5636,7 @@ class VisionAutonomyE2ETestSuite {
 }
 var vision_autonomy_default = new VisionAutonomyE2ETestSuite;
 // src/tests/e2e/vision-basic.ts
-import { createUniqueUuid as createUniqueUuid4 } from "@elizaos/core";
+import { createUniqueUuid as createUniqueUuid5 } from "@elizaos/core";
 class VisionBasicE2ETestSuite {
   name = "plugin-vision-basic-e2e";
   description = "Basic end-to-end tests for vision plugin functionality";
@@ -5611,9 +5664,9 @@ class VisionBasicE2ETestSuite {
       name: "Should describe scene when requested",
       fn: async (runtime) => {
         console.log("Testing scene description action...");
-        const roomId = createUniqueUuid4(runtime, "test-room");
+        const roomId = createUniqueUuid5(runtime, "test-room");
         const message = {
-          id: createUniqueUuid4(runtime, "test-msg-describe"),
+          id: createUniqueUuid5(runtime, "test-msg-describe"),
           entityId: runtime.agentId,
           content: { text: "what do you see?" },
           agentId: runtime.agentId,
@@ -5677,9 +5730,9 @@ class VisionBasicE2ETestSuite {
       name: "Should capture image when requested",
       fn: async (runtime) => {
         console.log("Testing image capture action...");
-        const roomId = createUniqueUuid4(runtime, "test-room");
+        const roomId = createUniqueUuid5(runtime, "test-room");
         const message = {
-          id: createUniqueUuid4(runtime, "test-msg-capture"),
+          id: createUniqueUuid5(runtime, "test-msg-capture"),
           entityId: runtime.agentId,
           content: { text: "take a photo" },
           agentId: runtime.agentId,
@@ -5749,11 +5802,11 @@ class VisionBasicE2ETestSuite {
       fn: async (runtime) => {
         console.log("Testing vision provider...");
         const message = {
-          id: createUniqueUuid4(runtime, "test-msg-provider"),
+          id: createUniqueUuid5(runtime, "test-msg-provider"),
           entityId: runtime.agentId,
           content: { text: "test provider" },
           agentId: runtime.agentId,
-          roomId: createUniqueUuid4(runtime, "test-room"),
+          roomId: createUniqueUuid5(runtime, "test-room"),
           createdAt: Date.now()
         };
         const state = await runtime.composeState(message);
@@ -6707,4 +6760,4 @@ export {
   src_default as default
 };
 
-//# debugId=F534BCEA2CE2323664756E2164756E21
+//# debugId=8F996A8EAA2C894364756E2164756E21