@blueharford/scrypted-spatial-awareness 0.1.15 → 0.1.16

package/out/main.nodejs.js

@@ -35070,6 +35070,10 @@ class TrackingEngine {
     listeners = new Map();
     pendingTimers = new Map();
     lostCheckInterval = null;
+    /** Track last alert time per object to enforce cooldown */
+    objectLastAlertTime = new Map();
+    /** Cache for LLM device reference */
+    llmDevice = null;
     constructor(topology, state, alertManager, config, console) {
         this.topology = topology;
         this.state = state;
@@ -35172,6 +35176,68 @@ class TrackingEngine {
             await this.processSighting(sighting, camera.isEntryPoint, camera.isExitPoint);
         }
     }
+    /** Check if object passes loitering threshold */
+    passesLoiteringThreshold(tracked) {
+        const visibleDuration = tracked.lastSeen - tracked.firstSeen;
+        return visibleDuration >= this.config.loiteringThreshold;
+    }
+    /** Check if object is in alert cooldown */
+    isInAlertCooldown(globalId) {
+        const lastAlertTime = this.objectLastAlertTime.get(globalId);
+        if (!lastAlertTime)
+            return false;
+        return (Date.now() - lastAlertTime) < this.config.objectAlertCooldown;
+    }
+    /** Record that we alerted for this object */
+    recordAlertTime(globalId) {
+        this.objectLastAlertTime.set(globalId, Date.now());
+    }
+    /** Try to get LLM-enhanced description for movement */
+    async getLlmDescription(tracked, fromCamera, toCamera, cameraId) {
+        if (!this.config.useLlmDescriptions)
+            return null;
+        try {
+            // Find LLM plugin device if not cached
+            if (!this.llmDevice) {
+                for (const id of Object.keys(systemManager.getSystemState())) {
+                    const device = systemManager.getDeviceById(id);
+                    if (device?.interfaces?.includes(sdk_1.ScryptedInterface.ObjectDetection) &&
+                        device.name?.toLowerCase().includes('llm')) {
+                        this.llmDevice = device;
+                        this.console.log(`Found LLM device: ${device.name}`);
+                        break;
+                    }
+                }
+            }
+            if (!this.llmDevice)
+                return null;
+            // Get snapshot from camera for LLM analysis
+            const camera = systemManager.getDeviceById(cameraId);
+            if (!camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera))
+                return null;
+            const picture = await camera.takePicture();
+            if (!picture)
+                return null;
+            // Ask LLM to describe the movement
+            const prompt = `Describe this ${tracked.className} in one short sentence. ` +
+                `They are moving from the ${fromCamera} area towards the ${toCamera}. ` +
+                `Include details like: gender (man/woman), clothing color, vehicle color/type if applicable. ` +
+                `Example: "Man in blue jacket walking from garage towards front door" or ` +
+                `"Black SUV driving from driveway towards street"`;
+            const result = await this.llmDevice.detectObjects(picture, {
+                settings: { prompt }
+            });
+            // Extract description from LLM response
+            if (result.detections?.[0]?.label) {
+                return result.detections[0].label;
+            }
+            return null;
+        }
+        catch (e) {
+            this.console.warn('LLM description failed:', e);
+            return null;
+        }
+    }
     /** Process a single sighting */
     async processSighting(sighting, isEntryPoint, isExitPoint) {
         // Try to correlate with existing tracked objects
@@ -35197,17 +35263,23 @@ class TrackingEngine {
                 this.console.log(`Object ${tracked.globalId.slice(0, 8)} transited: ` +
                     `${lastSighting.cameraName} → ${sighting.cameraName} ` +
                     `(confidence: ${(correlation.confidence * 100).toFixed(0)}%)`);
-                // Generate movement alert for cross-camera transition
-                await this.alertManager.checkAndAlert('movement', tracked, {
-                    fromCameraId: lastSighting.cameraId,
-                    fromCameraName: lastSighting.cameraName,
-                    toCameraId: sighting.cameraId,
-                    toCameraName: sighting.cameraName,
-                    transitTime: transitDuration,
-                    objectClass: sighting.detection.className,
-                    objectLabel: sighting.detection.label,
-                    detectionId: sighting.detectionId,
-                });
+                // Check loitering threshold and per-object cooldown before alerting
+                if (this.passesLoiteringThreshold(tracked) && !this.isInAlertCooldown(tracked.globalId)) {
+                    // Try to get LLM-enhanced description
+                    const llmDescription = await this.getLlmDescription(tracked, lastSighting.cameraName, sighting.cameraName, sighting.cameraId);
+                    // Generate movement alert for cross-camera transition
+                    await this.alertManager.checkAndAlert('movement', tracked, {
+                        fromCameraId: lastSighting.cameraId,
+                        fromCameraName: lastSighting.cameraName,
+                        toCameraId: sighting.cameraId,
+                        toCameraName: sighting.cameraName,
+                        transitTime: transitDuration,
+                        objectClass: sighting.detection.className,
+                        objectLabel: llmDescription || sighting.detection.label,
+                        detectionId: sighting.detectionId,
+                    });
+                    this.recordAlertTime(tracked.globalId);
+                }
             }
             // Add sighting to tracked object
             this.state.addSighting(tracked.globalId, sighting);
@@ -35231,14 +35303,17 @@ class TrackingEngine {
             this.console.log(`New ${sighting.detection.className} detected on ${sighting.cameraName} ` +
                 `(ID: ${globalId.slice(0, 8)})`);
             // Generate entry alert if this is an entry point
-            if (isEntryPoint) {
+            // Entry alerts also respect loitering threshold and cooldown
+            if (isEntryPoint && this.passesLoiteringThreshold(tracked) && !this.isInAlertCooldown(globalId)) {
+                const llmDescription = await this.getLlmDescription(tracked, 'outside', sighting.cameraName, sighting.cameraId);
                 await this.alertManager.checkAndAlert('property_entry', tracked, {
                     cameraId: sighting.cameraId,
                     cameraName: sighting.cameraName,
                     objectClass: sighting.detection.className,
-                    objectLabel: sighting.detection.label,
+                    objectLabel: llmDescription || sighting.detection.label,
                     detectionId: sighting.detectionId,
                 });
+                this.recordAlertTime(globalId);
             }
         }
     }
@@ -36091,6 +36166,28 @@ class SpatialAwarenessPlugin extends sdk_1.ScryptedDeviceBase {
             description: 'Use visual embeddings for object correlation (requires compatible detectors)',
             group: 'Tracking',
         },
+        loiteringThreshold: {
+            title: 'Loitering Threshold (seconds)',
+            type: 'number',
+            defaultValue: 3,
+            description: 'Object must be visible for this duration before triggering movement alerts',
+            group: 'Tracking',
+        },
+        objectAlertCooldown: {
+            title: 'Per-Object Alert Cooldown (seconds)',
+            type: 'number',
+            defaultValue: 30,
+            description: 'Minimum time between alerts for the same tracked object',
+            group: 'Tracking',
+        },
+        // LLM Integration
+        useLlmDescriptions: {
+            title: 'Use LLM for Rich Descriptions',
+            type: 'boolean',
+            defaultValue: true,
+            description: 'Use LLM plugin (if installed) to generate descriptive alerts like "Man walking from garage towards front door"',
+            group: 'Tracking',
+        },
         // MQTT Settings
         enableMqtt: {
             title: 'Enable MQTT',
@@ -36233,6 +36330,9 @@ class SpatialAwarenessPlugin extends sdk_1.ScryptedDeviceBase {
             correlationThreshold: this.storageSettings.values.correlationThreshold || 0.6,
             lostTimeout: (this.storageSettings.values.lostTimeout || 300) * 1000,
             useVisualMatching: this.storageSettings.values.useVisualMatching ?? true,
+            loiteringThreshold: (this.storageSettings.values.loiteringThreshold || 3) * 1000,
+            objectAlertCooldown: (this.storageSettings.values.objectAlertCooldown || 30) * 1000,
+            useLlmDescriptions: this.storageSettings.values.useLlmDescriptions ?? true,
         };
         this.trackingEngine = new tracking_engine_1.TrackingEngine(topology, this.trackingState, this.alertManager, config, this.console);
         await this.trackingEngine.startTracking();
@@ -36471,7 +36571,10 @@ class SpatialAwarenessPlugin extends sdk_1.ScryptedDeviceBase {
             key === 'correlationWindow' ||
             key === 'correlationThreshold' ||
             key === 'lostTimeout' ||
-            key === 'useVisualMatching') {
+            key === 'useVisualMatching' ||
+            key === 'loiteringThreshold' ||
+            key === 'objectAlertCooldown' ||
+            key === 'useLlmDescriptions') {
             const topologyJson = this.storage.getItem('topology');
             if (topologyJson) {
                 try {