@blueharford/scrypted-spatial-awareness 0.6.7 → 0.6.9

package/out/main.nodejs.js

@@ -35497,7 +35497,7 @@ class SpatialReasoningEngine {
         return this.llmDevice !== null;
     }
     /** Generate entry description when object enters property */
-    generateEntryDescription(tracked, cameraId) {
+    async generateEntryDescription(tracked, cameraId, mediaObject) {
         if (!this.topology) {
             return {
                 description: `${this.capitalizeFirst(tracked.className)} entered property`,
@@ -35517,10 +35517,9 @@ class SpatialReasoningEngine {
         }
         const landmarks = (0, topology_1.getLandmarksVisibleFromCamera)(this.topology, cameraId);
         const objectType = this.capitalizeFirst(tracked.className);
-        // Build entry description using topology context
+        // Build basic entry description using topology context
         const location = this.describeLocation(camera, landmarks, 'to');
         // Check if we can determine where they came from (e.g., street, neighbor)
-        const entryLandmark = landmarks.find(l => l.isEntryPoint);
         const streetLandmark = landmarks.find(l => l.type === 'street');
         const neighborLandmark = landmarks.find(l => l.type === 'neighbor');
         let source = '';
@@ -35530,15 +35529,35 @@ class SpatialReasoningEngine {
         else if (neighborLandmark) {
             source = ` from ${neighborLandmark.name}`;
         }
+        const basicDescription = `${objectType} arrived at ${location}${source}`;
+        // Try LLM for enhanced description with visual details
+        this.console.log(`[Entry] enableLlm=${this.config.enableLlm}, hasMediaObject=${!!mediaObject}`);
+        if (this.config.enableLlm && mediaObject) {
+            this.console.log(`[Entry] Attempting LLM description for entry event`);
+            const llmDescription = await this.getLlmEntryExitDescription(tracked, camera, landmarks, 'entry', mediaObject);
+            if (llmDescription) {
+                this.console.log(`[Entry] LLM returned: ${llmDescription.substring(0, 50)}...`);
+                return {
+                    description: llmDescription,
+                    involvedLandmarks: landmarks,
+                    confidence: 0.9,
+                    usedLlm: true,
+                };
+            }
+            this.console.warn(`[Entry] LLM returned null, falling back to basic`);
+        }
+        else {
+            this.console.log(`[Entry] Skipping LLM (enableLlm=${this.config.enableLlm}, mediaObject=${!!mediaObject})`);
+        }
         return {
-            description: `${objectType} arrived at ${location}${source}`,
+            description: basicDescription,
             involvedLandmarks: landmarks,
             confidence: 0.8,
             usedLlm: false,
         };
     }
     /** Generate exit description when object leaves property */
-    generateExitDescription(tracked, cameraId) {
+    async generateExitDescription(tracked, cameraId, mediaObject) {
         if (!this.topology) {
             return {
                 description: `${this.capitalizeFirst(tracked.className)} left property`,
@@ -35558,7 +35577,7 @@ class SpatialReasoningEngine {
         }
         const landmarks = (0, topology_1.getLandmarksVisibleFromCamera)(this.topology, cameraId);
         const objectType = this.capitalizeFirst(tracked.className);
-        // Build exit description
+        // Build basic exit description
         const location = this.describeLocation(camera, landmarks, 'from');
         // Check for exit point landmarks
         const exitLandmark = landmarks.find(l => l.isExitPoint);
@@ -35603,8 +35622,28 @@ class SpatialReasoningEngine {
                 journeyContext = ` — visited ${visitedLandmarks.join(' → ')}`;
             }
         }
+        const basicDescription = `${objectType} left ${location}${destination}${timeContext}${journeyContext}`;
+        // Try LLM for enhanced description with visual details
+        this.console.log(`[Exit] enableLlm=${this.config.enableLlm}, hasMediaObject=${!!mediaObject}`);
+        if (this.config.enableLlm && mediaObject) {
+            this.console.log(`[Exit] Attempting LLM description for exit event`);
+            const llmDescription = await this.getLlmEntryExitDescription(tracked, camera, landmarks, 'exit', mediaObject, journeyContext);
+            if (llmDescription) {
+                this.console.log(`[Exit] LLM returned: ${llmDescription.substring(0, 50)}...`);
+                return {
+                    description: llmDescription,
+                    involvedLandmarks: landmarks,
+                    confidence: 0.9,
+                    usedLlm: true,
+                };
+            }
+            this.console.warn(`[Exit] LLM returned null, falling back to basic`);
+        }
+        else {
+            this.console.log(`[Exit] Skipping LLM (enableLlm=${this.config.enableLlm}, mediaObject=${!!mediaObject})`);
+        }
         return {
-            description: `${objectType} left ${location}${destination}${timeContext}${journeyContext}`,
+            description: basicDescription,
             involvedLandmarks: landmarks,
             confidence: 0.8,
             usedLlm: false,
@@ -35801,6 +35840,107 @@ class SpatialReasoningEngine {
             return null;
         }
     }
+    /** Get LLM-enhanced description for entry/exit events */
+    async getLlmEntryExitDescription(tracked, camera, landmarks, eventType, mediaObject, journeyContext) {
+        this.console.log(`[LLM] getLlmEntryExitDescription called for ${eventType} event`);
+        const llm = await this.findLlmDevice();
+        if (!llm) {
+            this.console.warn(`[LLM] No LLM device found for ${eventType} description`);
+            return null;
+        }
+        if (!llm.getChatCompletion) {
+            this.console.warn(`[LLM] LLM device has no getChatCompletion method`);
+            return null;
+        }
+        this.console.log(`[LLM] Using LLM device: ${this.llmProvider}`);
+        try {
+            // Convert image to base64 for vision LLM
+            const imageData = await mediaObjectToBase64(mediaObject);
+            this.console.log(`[LLM] Image converted: ${imageData ? 'success' : 'failed'}, type: ${imageData?.mediaType}`);
+            const landmarkNames = landmarks.map(l => l.name).join(', ') || 'none identified';
+            const dwellTime = Math.round((tracked.lastSeen - tracked.firstSeen) / 1000);
+            // Build context-aware prompt
+            const prompt = eventType === 'entry'
+                ? `You are a security camera system. Analyze this image and describe who/what just arrived.
+
+CONTEXT:
+- Camera: ${camera.name}
+- Object type: ${tracked.className}
+- Nearby landmarks: ${landmarkNames}
+
+INSTRUCTIONS:
+Look at the image and generate a single, natural sentence describing:
+1. Physical description (if person: gender, clothing, items carried; if vehicle: color, type, make)
+2. What they appear to be doing (arriving, approaching, etc.)
+3. Relevant landmark context (driveway, front door, mailbox, etc.)
+
+Examples of good descriptions:
+- "Man in gray hoodie approaching the front door"
+- "Woman in scrubs arriving with shopping bags"
+- "White delivery van pulling into the driveway"
+- "UPS driver carrying package towards the porch"
+- "Teenager on bicycle coming up the driveway"
+
+Generate ONLY the description, nothing else:`
+                : `You are a security camera system. Analyze this image and describe who/what is leaving.
+
+CONTEXT:
+- Camera: ${camera.name}
+- Object type: ${tracked.className}
+- Time on property: ${dwellTime > 60 ? Math.round(dwellTime / 60) + ' minutes' : dwellTime + ' seconds'}
+- Nearby landmarks: ${landmarkNames}
+${journeyContext ? `- Journey: ${journeyContext}` : ''}
+
+INSTRUCTIONS:
+Look at the image and generate a single, natural sentence describing:
+1. Physical description (if person: gender, clothing, items carried; if vehicle: color, type)
+2. What they did (if determinable from context)
+3. Direction they're leaving towards
+
+Examples of good descriptions:
+- "Man in black hoodie leaving after checking the mailbox"
+- "Woman in business attire heading to car in driveway"
+- "Red sedan backing out of the driveway"
+- "Delivery driver returning to FedEx truck after leaving package"
+- "Landscaper with leaf blower heading to work truck"
+
+Generate ONLY the description, nothing else:`;
+            // Build message content - use multimodal format if we have an image
+            let messageContent;
+            if (imageData) {
+                messageContent = [
+                    { type: 'text', text: prompt },
+                    buildImageContent(imageData, this.llmProviderType),
+                ];
+            }
+            else {
+                messageContent = prompt;
+            }
+            // Call LLM using ChatCompletion interface
+            this.console.log(`[LLM] Calling getChatCompletion for ${eventType}...`);
+            const result = await llm.getChatCompletion({
+                messages: [
+                    {
+                        role: 'user',
+                        content: messageContent,
+                    },
+                ],
+                max_tokens: 100,
+                temperature: 0.7,
+            });
+            const content = result?.choices?.[0]?.message?.content;
+            if (content && typeof content === 'string') {
+                this.console.log(`[LLM] Got ${eventType} description: ${content.trim().substring(0, 50)}...`);
+                return content.trim();
+            }
+            this.console.warn(`[LLM] No content in response for ${eventType}`);
+            return null;
+        }
+        catch (e) {
+            this.console.warn(`[LLM] ${eventType} description generation failed:`, e);
+            return null;
+        }
+    }
     /** Build LLM prompt with RAG context */
     buildLlmPrompt(tracked, fromCamera, toCamera, transitTime, fromLandmarks, toLandmarks, ragContext) {
         const transitSecs = Math.round(transitTime / 1000);
@@ -37106,8 +37246,26 @@ class TrackingEngine {
             // Check if we've already alerted for this object
             if (this.isInAlertCooldown(globalId))
                 return;
-            // Generate spatial description
-            const spatialResult = this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId);
+            // Get snapshot for LLM description (if LLM is enabled)
+            let mediaObject;
+            this.console.log(`[Entry Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
+            if (this.config.useLlmDescriptions) {
+                try {
+                    const camera = systemManager.getDeviceById(sighting.cameraId);
+                    this.console.log(`[Entry Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)}`);
+                    if (camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)) {
+                        mediaObject = await camera.takePicture();
+                        this.console.log(`[Entry Alert] Got snapshot: ${!!mediaObject}`);
+                    }
+                }
+                catch (e) {
+                    this.console.warn('[Entry Alert] Failed to get snapshot:', e);
+                }
+            }
+            // Generate spatial description (now async with LLM support)
+            this.console.log(`[Entry Alert] Calling generateEntryDescription with mediaObject=${!!mediaObject}`);
+            const spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
+            this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
             if (isEntryPoint) {
                 // Entry point - generate property entry alert
                 await this.alertManager.checkAndAlert('property_entry', tracked, {
@@ -37182,9 +37340,26 @@ class TrackingEngine {
             const current = this.state.getObject(tracked.globalId);
             if (current && current.state === 'pending') {
                 this.state.markExited(tracked.globalId, sighting.cameraId, sighting.cameraName);
-                // Generate rich exit description using topology context
-                const spatialResult = this.spatialReasoning.generateExitDescription(current, sighting.cameraId);
-                this.console.log(`Object ${tracked.globalId.slice(0, 8)} exited: ${spatialResult.description}`);
+                // Get snapshot for LLM description (if LLM is enabled)
+                let mediaObject;
+                this.console.log(`[Exit Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
+                if (this.config.useLlmDescriptions) {
+                    try {
+                        const camera = systemManager.getDeviceById(sighting.cameraId);
+                        this.console.log(`[Exit Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)}`);
+                        if (camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)) {
+                            mediaObject = await camera.takePicture();
+                            this.console.log(`[Exit Alert] Got snapshot: ${!!mediaObject}`);
+                        }
+                    }
+                    catch (e) {
+                        this.console.warn('[Exit Alert] Failed to get snapshot:', e);
+                    }
+                }
+                // Generate rich exit description using topology context (now async with LLM support)
+                this.console.log(`[Exit Alert] Calling generateExitDescription with mediaObject=${!!mediaObject}`);
+                const spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
+                this.console.log(`[Exit Alert] Object ${tracked.globalId.slice(0, 8)} exited: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
                 await this.alertManager.checkAndAlert('property_exit', current, {
                     cameraId: sighting.cameraId,
                     cameraName: sighting.cameraName,