@blueharford/scrypted-spatial-awareness 0.6.7 → 0.6.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.7",
+  "version": "0.6.9",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -545,10 +545,11 @@ export class SpatialReasoningEngine {
   }
 
   /** Generate entry description when object enters property */
-  generateEntryDescription(
+  async generateEntryDescription(
     tracked: TrackedObject,
-    cameraId: string
-  ): SpatialReasoningResult {
+    cameraId: string,
+    mediaObject?: MediaObject
+  ): Promise<SpatialReasoningResult> {
     if (!this.topology) {
       return {
         description: `${this.capitalizeFirst(tracked.className)} entered property`,
@@ -571,11 +572,10 @@ export class SpatialReasoningEngine {
     const landmarks = getLandmarksVisibleFromCamera(this.topology, cameraId);
     const objectType = this.capitalizeFirst(tracked.className);
 
-    // Build entry description using topology context
+    // Build basic entry description using topology context
     const location = this.describeLocation(camera, landmarks, 'to');
 
     // Check if we can determine where they came from (e.g., street, neighbor)
-    const entryLandmark = landmarks.find(l => l.isEntryPoint);
     const streetLandmark = landmarks.find(l => l.type === 'street');
     const neighborLandmark = landmarks.find(l => l.type === 'neighbor');
 
@@ -586,8 +586,31 @@ export class SpatialReasoningEngine {
       source = ` from ${neighborLandmark.name}`;
     }
 
+    const basicDescription = `${objectType} arrived at ${location}${source}`;
+
+    // Try LLM for enhanced description with visual details
+    this.console.log(`[Entry] enableLlm=${this.config.enableLlm}, hasMediaObject=${!!mediaObject}`);
+    if (this.config.enableLlm && mediaObject) {
+      this.console.log(`[Entry] Attempting LLM description for entry event`);
+      const llmDescription = await this.getLlmEntryExitDescription(
+        tracked, camera, landmarks, 'entry', mediaObject
+      );
+      if (llmDescription) {
+        this.console.log(`[Entry] LLM returned: ${llmDescription.substring(0, 50)}...`);
+        return {
+          description: llmDescription,
+          involvedLandmarks: landmarks,
+          confidence: 0.9,
+          usedLlm: true,
+        };
+      }
+      this.console.warn(`[Entry] LLM returned null, falling back to basic`);
+    } else {
+      this.console.log(`[Entry] Skipping LLM (enableLlm=${this.config.enableLlm}, mediaObject=${!!mediaObject})`);
+    }
+
     return {
-      description: `${objectType} arrived at ${location}${source}`,
+      description: basicDescription,
       involvedLandmarks: landmarks,
       confidence: 0.8,
       usedLlm: false,
@@ -595,10 +618,11 @@ export class SpatialReasoningEngine {
   }
 
   /** Generate exit description when object leaves property */
-  generateExitDescription(
+  async generateExitDescription(
     tracked: TrackedObject,
-    cameraId: string
-  ): SpatialReasoningResult {
+    cameraId: string,
+    mediaObject?: MediaObject
+  ): Promise<SpatialReasoningResult> {
     if (!this.topology) {
       return {
         description: `${this.capitalizeFirst(tracked.className)} left property`,
@@ -621,7 +645,7 @@ export class SpatialReasoningEngine {
     const landmarks = getLandmarksVisibleFromCamera(this.topology, cameraId);
     const objectType = this.capitalizeFirst(tracked.className);
 
-    // Build exit description
+    // Build basic exit description
     const location = this.describeLocation(camera, landmarks, 'from');
 
     // Check for exit point landmarks
@@ -674,8 +698,31 @@ export class SpatialReasoningEngine {
       }
     }
 
+    const basicDescription = `${objectType} left ${location}${destination}${timeContext}${journeyContext}`;
+
+    // Try LLM for enhanced description with visual details
+    this.console.log(`[Exit] enableLlm=${this.config.enableLlm}, hasMediaObject=${!!mediaObject}`);
+    if (this.config.enableLlm && mediaObject) {
+      this.console.log(`[Exit] Attempting LLM description for exit event`);
+      const llmDescription = await this.getLlmEntryExitDescription(
+        tracked, camera, landmarks, 'exit', mediaObject, journeyContext
+      );
+      if (llmDescription) {
+        this.console.log(`[Exit] LLM returned: ${llmDescription.substring(0, 50)}...`);
+        return {
+          description: llmDescription,
+          involvedLandmarks: landmarks,
+          confidence: 0.9,
+          usedLlm: true,
+        };
+      }
+      this.console.warn(`[Exit] LLM returned null, falling back to basic`);
+    } else {
+      this.console.log(`[Exit] Skipping LLM (enableLlm=${this.config.enableLlm}, mediaObject=${!!mediaObject})`);
+    }
+
     return {
-      description: `${objectType} left ${location}${destination}${timeContext}${journeyContext}`,
+      description: basicDescription,
       involvedLandmarks: landmarks,
       confidence: 0.8,
       usedLlm: false,
@@ -952,6 +999,122 @@ export class SpatialReasoningEngine {
     }
   }
 
+  /** Get LLM-enhanced description for entry/exit events */
+  private async getLlmEntryExitDescription(
+    tracked: TrackedObject,
+    camera: CameraNode,
+    landmarks: Landmark[],
+    eventType: 'entry' | 'exit',
+    mediaObject: MediaObject,
+    journeyContext?: string
+  ): Promise<string | null> {
+    this.console.log(`[LLM] getLlmEntryExitDescription called for ${eventType} event`);
+
+    const llm = await this.findLlmDevice();
+    if (!llm) {
+      this.console.warn(`[LLM] No LLM device found for ${eventType} description`);
+      return null;
+    }
+    if (!llm.getChatCompletion) {
+      this.console.warn(`[LLM] LLM device has no getChatCompletion method`);
+      return null;
+    }
+
+    this.console.log(`[LLM] Using LLM device: ${this.llmProvider}`);
+
+    try {
+      // Convert image to base64 for vision LLM
+      const imageData = await mediaObjectToBase64(mediaObject);
+      this.console.log(`[LLM] Image converted: ${imageData ? 'success' : 'failed'}, type: ${imageData?.mediaType}`);
+
+      const landmarkNames = landmarks.map(l => l.name).join(', ') || 'none identified';
+      const dwellTime = Math.round((tracked.lastSeen - tracked.firstSeen) / 1000);
+
+      // Build context-aware prompt
+      const prompt = eventType === 'entry'
+        ? `You are a security camera system. Analyze this image and describe who/what just arrived.
+
+CONTEXT:
+- Camera: ${camera.name}
+- Object type: ${tracked.className}
+- Nearby landmarks: ${landmarkNames}
+
+INSTRUCTIONS:
+Look at the image and generate a single, natural sentence describing:
+1. Physical description (if person: gender, clothing, items carried; if vehicle: color, type, make)
+2. What they appear to be doing (arriving, approaching, etc.)
+3. Relevant landmark context (driveway, front door, mailbox, etc.)
+
+Examples of good descriptions:
+- "Man in gray hoodie approaching the front door"
+- "Woman in scrubs arriving with shopping bags"
+- "White delivery van pulling into the driveway"
+- "UPS driver carrying package towards the porch"
+- "Teenager on bicycle coming up the driveway"
+
+Generate ONLY the description, nothing else:`
+        : `You are a security camera system. Analyze this image and describe who/what is leaving.
+
+CONTEXT:
+- Camera: ${camera.name}
+- Object type: ${tracked.className}
+- Time on property: ${dwellTime > 60 ? Math.round(dwellTime / 60) + ' minutes' : dwellTime + ' seconds'}
+- Nearby landmarks: ${landmarkNames}
+${journeyContext ? `- Journey: ${journeyContext}` : ''}
+
+INSTRUCTIONS:
+Look at the image and generate a single, natural sentence describing:
+1. Physical description (if person: gender, clothing, items carried; if vehicle: color, type)
+2. What they did (if determinable from context)
+3. Direction they're leaving towards
+
+Examples of good descriptions:
+- "Man in black hoodie leaving after checking the mailbox"
+- "Woman in business attire heading to car in driveway"
+- "Red sedan backing out of the driveway"
+- "Delivery driver returning to FedEx truck after leaving package"
+- "Landscaper with leaf blower heading to work truck"
+
+Generate ONLY the description, nothing else:`;
+
+      // Build message content - use multimodal format if we have an image
+      let messageContent: any;
+      if (imageData) {
+        messageContent = [
+          { type: 'text', text: prompt },
+          buildImageContent(imageData, this.llmProviderType),
+        ];
+      } else {
+        messageContent = prompt;
+      }
+
+      // Call LLM using ChatCompletion interface
+      this.console.log(`[LLM] Calling getChatCompletion for ${eventType}...`);
+      const result = await llm.getChatCompletion({
+        messages: [
+          {
+            role: 'user',
+            content: messageContent,
+          },
+        ],
+        max_tokens: 100,
+        temperature: 0.7,
+      });
+
+      const content = result?.choices?.[0]?.message?.content;
+      if (content && typeof content === 'string') {
+        this.console.log(`[LLM] Got ${eventType} description: ${content.trim().substring(0, 50)}...`);
+        return content.trim();
+      }
+
+      this.console.warn(`[LLM] No content in response for ${eventType}`);
+      return null;
+    } catch (e) {
+      this.console.warn(`[LLM] ${eventType} description generation failed:`, e);
+      return null;
+    }
+  }
+
   /** Build LLM prompt with RAG context */
   private buildLlmPrompt(
     tracked: TrackedObject,

package/src/core/tracking-engine.ts

@@ -536,11 +536,30 @@ export class TrackingEngine {
       // Check if we've already alerted for this object
       if (this.isInAlertCooldown(globalId)) return;
 
-      // Generate spatial description
-      const spatialResult = this.spatialReasoning.generateEntryDescription(
+      // Get snapshot for LLM description (if LLM is enabled)
+      let mediaObject: MediaObject | undefined;
+      this.console.log(`[Entry Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
+      if (this.config.useLlmDescriptions) {
+        try {
+          const camera = systemManager.getDeviceById<Camera>(sighting.cameraId);
+          this.console.log(`[Entry Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(ScryptedInterface.Camera)}`);
+          if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
+            mediaObject = await camera.takePicture();
+            this.console.log(`[Entry Alert] Got snapshot: ${!!mediaObject}`);
+          }
+        } catch (e) {
+          this.console.warn('[Entry Alert] Failed to get snapshot:', e);
+        }
+      }
+
+      // Generate spatial description (now async with LLM support)
+      this.console.log(`[Entry Alert] Calling generateEntryDescription with mediaObject=${!!mediaObject}`);
+      const spatialResult = await this.spatialReasoning.generateEntryDescription(
         tracked,
-        sighting.cameraId
+        sighting.cameraId,
+        mediaObject
       );
+      this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
 
       if (isEntryPoint) {
         // Entry point - generate property entry alert
@@ -626,14 +645,32 @@ export class TrackingEngine {
       if (current && current.state === 'pending') {
         this.state.markExited(tracked.globalId, sighting.cameraId, sighting.cameraName);
 
-        // Generate rich exit description using topology context
-        const spatialResult = this.spatialReasoning.generateExitDescription(
+        // Get snapshot for LLM description (if LLM is enabled)
+        let mediaObject: MediaObject | undefined;
+        this.console.log(`[Exit Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
+        if (this.config.useLlmDescriptions) {
+          try {
+            const camera = systemManager.getDeviceById<Camera>(sighting.cameraId);
+            this.console.log(`[Exit Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(ScryptedInterface.Camera)}`);
+            if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
+              mediaObject = await camera.takePicture();
+              this.console.log(`[Exit Alert] Got snapshot: ${!!mediaObject}`);
+            }
+          } catch (e) {
+            this.console.warn('[Exit Alert] Failed to get snapshot:', e);
+          }
+        }
+
+        // Generate rich exit description using topology context (now async with LLM support)
+        this.console.log(`[Exit Alert] Calling generateExitDescription with mediaObject=${!!mediaObject}`);
+        const spatialResult = await this.spatialReasoning.generateExitDescription(
           current,
-          sighting.cameraId
+          sighting.cameraId,
+          mediaObject
         );
 
         this.console.log(
-          `Object ${tracked.globalId.slice(0, 8)} exited: ${spatialResult.description}`
+          `[Exit Alert] Object ${tracked.globalId.slice(0, 8)} exited: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`
         );
 
         await this.alertManager.checkAndAlert('property_exit', current, {