@blueharford/scrypted-spatial-awareness 0.6.8 → 0.6.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.8",
+  "version": "0.6.10",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -589,11 +589,14 @@ export class SpatialReasoningEngine {
     const basicDescription = `${objectType} arrived at ${location}${source}`;
 
     // Try LLM for enhanced description with visual details
+    this.console.log(`[Entry] enableLlm=${this.config.enableLlm}, hasMediaObject=${!!mediaObject}`);
     if (this.config.enableLlm && mediaObject) {
+      this.console.log(`[Entry] Attempting LLM description for entry event`);
       const llmDescription = await this.getLlmEntryExitDescription(
         tracked, camera, landmarks, 'entry', mediaObject
       );
       if (llmDescription) {
+        this.console.log(`[Entry] LLM returned: ${llmDescription.substring(0, 50)}...`);
         return {
           description: llmDescription,
           involvedLandmarks: landmarks,
@@ -601,6 +604,9 @@ export class SpatialReasoningEngine {
           usedLlm: true,
         };
       }
+      this.console.warn(`[Entry] LLM returned null, falling back to basic`);
+    } else {
+      this.console.log(`[Entry] Skipping LLM (enableLlm=${this.config.enableLlm}, mediaObject=${!!mediaObject})`);
     }
 
     return {
@@ -695,11 +701,14 @@ export class SpatialReasoningEngine {
     const basicDescription = `${objectType} left ${location}${destination}${timeContext}${journeyContext}`;
 
     // Try LLM for enhanced description with visual details
+    this.console.log(`[Exit] enableLlm=${this.config.enableLlm}, hasMediaObject=${!!mediaObject}`);
     if (this.config.enableLlm && mediaObject) {
+      this.console.log(`[Exit] Attempting LLM description for exit event`);
       const llmDescription = await this.getLlmEntryExitDescription(
         tracked, camera, landmarks, 'exit', mediaObject, journeyContext
       );
       if (llmDescription) {
+        this.console.log(`[Exit] LLM returned: ${llmDescription.substring(0, 50)}...`);
         return {
           description: llmDescription,
           involvedLandmarks: landmarks,
@@ -707,6 +716,9 @@ export class SpatialReasoningEngine {
           usedLlm: true,
         };
       }
+      this.console.warn(`[Exit] LLM returned null, falling back to basic`);
+    } else {
+      this.console.log(`[Exit] Skipping LLM (enableLlm=${this.config.enableLlm}, mediaObject=${!!mediaObject})`);
     }
 
     return {
@@ -996,12 +1008,24 @@ export class SpatialReasoningEngine {
     mediaObject: MediaObject,
     journeyContext?: string
   ): Promise<string | null> {
+    this.console.log(`[LLM] getLlmEntryExitDescription called for ${eventType} event`);
+
     const llm = await this.findLlmDevice();
-    if (!llm || !llm.getChatCompletion) return null;
+    if (!llm) {
+      this.console.warn(`[LLM] No LLM device found for ${eventType} description`);
+      return null;
+    }
+    if (!llm.getChatCompletion) {
+      this.console.warn(`[LLM] LLM device has no getChatCompletion method`);
+      return null;
+    }
+
+    this.console.log(`[LLM] Using LLM device: ${this.llmProvider}`);
 
     try {
       // Convert image to base64 for vision LLM
       const imageData = await mediaObjectToBase64(mediaObject);
+      this.console.log(`[LLM] Image converted: ${imageData ? 'success' : 'failed'}, type: ${imageData?.mediaType}`);
 
       const landmarkNames = landmarks.map(l => l.name).join(', ') || 'none identified';
       const dwellTime = Math.round((tracked.lastSeen - tracked.firstSeen) / 1000);
@@ -1053,37 +1077,65 @@ Examples of good descriptions:
 
 Generate ONLY the description, nothing else:`;
 
-      // Build message content - use multimodal format if we have an image
-      let messageContent: any;
+      // Try multimodal format first, fall back to text-only if it fails
+      let result: any;
+      let usedVision = false;
+
       if (imageData) {
-        messageContent = [
-          { type: 'text', text: prompt },
-          buildImageContent(imageData, this.llmProviderType),
-        ];
-      } else {
-        messageContent = prompt;
+        // First attempt: Try multimodal with image
+        try {
+          this.console.log(`[LLM] Attempting multimodal ${eventType} call with image...`);
+          const multimodalContent = [
+            { type: 'text', text: prompt },
+            buildImageContent(imageData, this.llmProviderType),
+          ];
+
+          result = await llm.getChatCompletion({
+            messages: [
+              {
+                role: 'user',
+                content: multimodalContent,
+              },
+            ],
+            max_tokens: 100,
+            temperature: 0.7,
+          });
+          usedVision = true;
+        } catch (visionError: any) {
+          // If vision format fails, try text-only
+          if (isVisionFormatError(visionError)) {
+            this.console.warn(`[LLM] Vision format not supported, falling back to text-only: ${visionError.message || visionError}`);
+          } else {
+            this.console.warn(`[LLM] Multimodal call failed, trying text-only: ${visionError.message || visionError}`);
+          }
+        }
       }
 
-      // Call LLM using ChatCompletion interface
-      const result = await llm.getChatCompletion({
-        messages: [
-          {
-            role: 'user',
-            content: messageContent,
-          },
-        ],
-        max_tokens: 100,
-        temperature: 0.7,
-      });
+      // If no result yet, try text-only
+      if (!result) {
+        this.console.log(`[LLM] Calling text-only getChatCompletion for ${eventType}...`);
+        result = await llm.getChatCompletion({
+          messages: [
+            {
+              role: 'user',
+              content: prompt,
+            },
+          ],
+          max_tokens: 100,
+          temperature: 0.7,
+        });
+      }
 
       const content = result?.choices?.[0]?.message?.content;
       if (content && typeof content === 'string') {
+        this.console.log(`[LLM] Got ${eventType} description (vision=${usedVision}): ${content.trim().substring(0, 50)}...`);
         return content.trim();
       }
 
+      this.console.warn(`[LLM] No content in response for ${eventType}`);
       return null;
     } catch (e) {
-      this.console.warn(`LLM ${eventType} description generation failed:`, e);
+      this.console.warn(`[LLM] ${eventType} description generation failed:`, e);
       return null;
     }
   }

package/src/core/tracking-engine.ts

@@ -538,23 +538,28 @@ export class TrackingEngine {
 
       // Get snapshot for LLM description (if LLM is enabled)
       let mediaObject: MediaObject | undefined;
+      this.console.log(`[Entry Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
       if (this.config.useLlmDescriptions) {
         try {
           const camera = systemManager.getDeviceById<Camera>(sighting.cameraId);
+          this.console.log(`[Entry Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(ScryptedInterface.Camera)}`);
           if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
             mediaObject = await camera.takePicture();
+            this.console.log(`[Entry Alert] Got snapshot: ${!!mediaObject}`);
           }
         } catch (e) {
-          this.console.warn('Failed to get snapshot for entry description:', e);
+          this.console.warn('[Entry Alert] Failed to get snapshot:', e);
         }
       }
 
       // Generate spatial description (now async with LLM support)
+      this.console.log(`[Entry Alert] Calling generateEntryDescription with mediaObject=${!!mediaObject}`);
       const spatialResult = await this.spatialReasoning.generateEntryDescription(
         tracked,
         sighting.cameraId,
         mediaObject
       );
+      this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
 
       if (isEntryPoint) {
         // Entry point - generate property entry alert
@@ -642,18 +647,22 @@ export class TrackingEngine {
 
         // Get snapshot for LLM description (if LLM is enabled)
         let mediaObject: MediaObject | undefined;
+        this.console.log(`[Exit Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
         if (this.config.useLlmDescriptions) {
           try {
             const camera = systemManager.getDeviceById<Camera>(sighting.cameraId);
+            this.console.log(`[Exit Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(ScryptedInterface.Camera)}`);
             if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
               mediaObject = await camera.takePicture();
+              this.console.log(`[Exit Alert] Got snapshot: ${!!mediaObject}`);
             }
           } catch (e) {
-            this.console.warn('Failed to get snapshot for exit description:', e);
+            this.console.warn('[Exit Alert] Failed to get snapshot:', e);
           }
         }
 
         // Generate rich exit description using topology context (now async with LLM support)
+        this.console.log(`[Exit Alert] Calling generateExitDescription with mediaObject=${!!mediaObject}`);
         const spatialResult = await this.spatialReasoning.generateExitDescription(
           current,
           sighting.cameraId,
@@ -661,7 +670,7 @@ export class TrackingEngine {
         );
 
         this.console.log(
-          `Object ${tracked.globalId.slice(0, 8)} exited: ${spatialResult.description}`
+          `[Exit Alert] Object ${tracked.globalId.slice(0, 8)} exited: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`
         );
 
         await this.alertManager.checkAndAlert('property_exit', current, {