@blueharford/scrypted-spatial-awareness 0.1.14 → 0.1.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.1.14",
+  "version": "0.1.16",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/alerts/alert-manager.ts

@@ -186,6 +186,24 @@ export class AlertManager {
    */
   private getDefaultNotifiers(): string[] {
     try {
+      // Try new multiple notifiers setting first
+      const notifiers = this.storage.getItem('defaultNotifiers');
+      if (notifiers) {
+        // Could be JSON array or comma-separated string
+        try {
+          const parsed = JSON.parse(notifiers);
+          if (Array.isArray(parsed)) {
+            return parsed;
+          }
+        } catch {
+          // Not JSON, might be comma-separated or single value
+          if (notifiers.includes(',')) {
+            return notifiers.split(',').map(s => s.trim()).filter(Boolean);
+          }
+          return [notifiers];
+        }
+      }
+      // Fallback to old single notifier setting
       const defaultNotifier = this.storage.getItem('defaultNotifier');
       return defaultNotifier ? [defaultNotifier] : [];
     } catch {

package/src/core/tracking-engine.ts

@@ -10,6 +10,9 @@ import sdk, {
   ObjectDetectionResult,
   ScryptedInterface,
   EventListenerRegister,
+  ObjectDetection,
+  Camera,
+  MediaObject,
 } from '@scrypted/sdk';
 import { CameraTopology, findCamera, findConnection, findConnectionsFrom } from '../models/topology';
 import {
@@ -34,6 +37,12 @@ export interface TrackingEngineConfig {
   lostTimeout: number;
   /** Enable visual embedding matching */
   useVisualMatching: boolean;
+  /** Loitering threshold - object must be visible this long before alerting (ms) */
+  loiteringThreshold: number;
+  /** Per-object alert cooldown (ms) */
+  objectAlertCooldown: number;
+  /** Use LLM for enhanced descriptions */
+  useLlmDescriptions: boolean;
 }
 
 export class TrackingEngine {
@@ -46,6 +55,10 @@ export class TrackingEngine {
   private listeners: Map<string, EventListenerRegister> = new Map();
   private pendingTimers: Map<GlobalTrackingId, NodeJS.Timeout> = new Map();
   private lostCheckInterval: NodeJS.Timeout | null = null;
+  /** Track last alert time per object to enforce cooldown */
+  private objectLastAlertTime: Map<GlobalTrackingId, number> = new Map();
+  /** Cache for LLM device reference */
+  private llmDevice: ObjectDetection | null = null;
 
   constructor(
     topology: CameraTopology,
@@ -176,6 +189,79 @@ export class TrackingEngine {
     }
   }
 
+  /** Check if object passes loitering threshold */
+  private passesLoiteringThreshold(tracked: TrackedObject): boolean {
+    const visibleDuration = tracked.lastSeen - tracked.firstSeen;
+    return visibleDuration >= this.config.loiteringThreshold;
+  }
+
+  /** Check if object is in alert cooldown */
+  private isInAlertCooldown(globalId: GlobalTrackingId): boolean {
+    const lastAlertTime = this.objectLastAlertTime.get(globalId);
+    if (!lastAlertTime) return false;
+    return (Date.now() - lastAlertTime) < this.config.objectAlertCooldown;
+  }
+
+  /** Record that we alerted for this object */
+  private recordAlertTime(globalId: GlobalTrackingId): void {
+    this.objectLastAlertTime.set(globalId, Date.now());
+  }
+
+  /** Try to get LLM-enhanced description for movement */
+  private async getLlmDescription(
+    tracked: TrackedObject,
+    fromCamera: string,
+    toCamera: string,
+    cameraId: string
+  ): Promise<string | null> {
+    if (!this.config.useLlmDescriptions) return null;
+
+    try {
+      // Find LLM plugin device if not cached
+      if (!this.llmDevice) {
+        for (const id of Object.keys(systemManager.getSystemState())) {
+          const device = systemManager.getDeviceById(id);
+          if (device?.interfaces?.includes(ScryptedInterface.ObjectDetection) &&
+              device.name?.toLowerCase().includes('llm')) {
+            this.llmDevice = device as unknown as ObjectDetection;
+            this.console.log(`Found LLM device: ${device.name}`);
+            break;
+          }
+        }
+      }
+
+      if (!this.llmDevice) return null;
+
+      // Get snapshot from camera for LLM analysis
+      const camera = systemManager.getDeviceById<Camera>(cameraId);
+      if (!camera?.interfaces?.includes(ScryptedInterface.Camera)) return null;
+
+      const picture = await camera.takePicture();
+      if (!picture) return null;
+
+      // Ask LLM to describe the movement
+      const prompt = `Describe this ${tracked.className} in one short sentence. ` +
+        `They are moving from the ${fromCamera} area towards the ${toCamera}. ` +
+        `Include details like: gender (man/woman), clothing color, vehicle color/type if applicable. ` +
+        `Example: "Man in blue jacket walking from garage towards front door" or ` +
+        `"Black SUV driving from driveway towards street"`;
+
+      const result = await this.llmDevice.detectObjects(picture, {
+        settings: { prompt }
+      } as any);
+
+      // Extract description from LLM response
+      if (result.detections?.[0]?.label) {
+        return result.detections[0].label;
+      }
+
+      return null;
+    } catch (e) {
+      this.console.warn('LLM description failed:', e);
+      return null;
+    }
+  }
+
   /** Process a single sighting */
   private async processSighting(
     sighting: ObjectSighting,
@@ -212,17 +298,30 @@ export class TrackingEngine {
           `(confidence: ${(correlation.confidence * 100).toFixed(0)}%)`
         );
 
-        // Generate movement alert for cross-camera transition
-        await this.alertManager.checkAndAlert('movement', tracked, {
-          fromCameraId: lastSighting.cameraId,
-          fromCameraName: lastSighting.cameraName,
-          toCameraId: sighting.cameraId,
-          toCameraName: sighting.cameraName,
-          transitTime: transitDuration,
-          objectClass: sighting.detection.className,
-          objectLabel: sighting.detection.label,
-          detectionId: sighting.detectionId,
-        });
+        // Check loitering threshold and per-object cooldown before alerting
+        if (this.passesLoiteringThreshold(tracked) && !this.isInAlertCooldown(tracked.globalId)) {
+          // Try to get LLM-enhanced description
+          const llmDescription = await this.getLlmDescription(
+            tracked,
+            lastSighting.cameraName,
+            sighting.cameraName,
+            sighting.cameraId
+          );
+
+          // Generate movement alert for cross-camera transition
+          await this.alertManager.checkAndAlert('movement', tracked, {
+            fromCameraId: lastSighting.cameraId,
+            fromCameraName: lastSighting.cameraName,
+            toCameraId: sighting.cameraId,
+            toCameraName: sighting.cameraName,
+            transitTime: transitDuration,
+            objectClass: sighting.detection.className,
+            objectLabel: llmDescription || sighting.detection.label,
+            detectionId: sighting.detectionId,
+          });
+
+          this.recordAlertTime(tracked.globalId);
+        }
       }
 
       // Add sighting to tracked object
@@ -253,14 +352,24 @@ export class TrackingEngine {
       );
 
       // Generate entry alert if this is an entry point
-      if (isEntryPoint) {
+      // Entry alerts also respect loitering threshold and cooldown
+      if (isEntryPoint && this.passesLoiteringThreshold(tracked) && !this.isInAlertCooldown(globalId)) {
+        const llmDescription = await this.getLlmDescription(
+          tracked,
+          'outside',
+          sighting.cameraName,
+          sighting.cameraId
+        );
+
         await this.alertManager.checkAndAlert('property_entry', tracked, {
           cameraId: sighting.cameraId,
           cameraName: sighting.cameraName,
           objectClass: sighting.detection.className,
-          objectLabel: sighting.detection.label,
+          objectLabel: llmDescription || sighting.detection.label,
           detectionId: sighting.detectionId,
         });
+
+        this.recordAlertTime(globalId);
       }
     }
   }

package/src/main.ts

@@ -85,6 +85,29 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       description: 'Use visual embeddings for object correlation (requires compatible detectors)',
       group: 'Tracking',
     },
+    loiteringThreshold: {
+      title: 'Loitering Threshold (seconds)',
+      type: 'number',
+      defaultValue: 3,
+      description: 'Object must be visible for this duration before triggering movement alerts',
+      group: 'Tracking',
+    },
+    objectAlertCooldown: {
+      title: 'Per-Object Alert Cooldown (seconds)',
+      type: 'number',
+      defaultValue: 30,
+      description: 'Minimum time between alerts for the same tracked object',
+      group: 'Tracking',
+    },
+
+    // LLM Integration
+    useLlmDescriptions: {
+      title: 'Use LLM for Rich Descriptions',
+      type: 'boolean',
+      defaultValue: true,
+      description: 'Use LLM plugin (if installed) to generate descriptive alerts like "Man walking from garage towards front door"',
+      group: 'Tracking',
+    },
 
     // MQTT Settings
     enableMqtt: {
@@ -123,10 +146,12 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       defaultValue: true,
       group: 'Alerts',
     },
-    defaultNotifier: {
-      title: 'Default Notifier',
+    defaultNotifiers: {
+      title: 'Notifiers',
       type: 'device',
+      multiple: true,
       deviceFilter: `interfaces.includes('${ScryptedInterface.Notifier}')`,
+      description: 'Select one or more notifiers to receive alerts',
       group: 'Alerts',
     },
 
@@ -242,6 +267,9 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       correlationThreshold: this.storageSettings.values.correlationThreshold as number || 0.6,
       lostTimeout: (this.storageSettings.values.lostTimeout as number || 300) * 1000,
       useVisualMatching: this.storageSettings.values.useVisualMatching as boolean ?? true,
+      loiteringThreshold: (this.storageSettings.values.loiteringThreshold as number || 3) * 1000,
+      objectAlertCooldown: (this.storageSettings.values.objectAlertCooldown as number || 30) * 1000,
+      useLlmDescriptions: this.storageSettings.values.useLlmDescriptions as boolean ?? true,
     };
 
     this.trackingEngine = new TrackingEngine(
@@ -515,7 +543,10 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       key === 'correlationWindow' ||
       key === 'correlationThreshold' ||
       key === 'lostTimeout' ||
-      key === 'useVisualMatching'
+      key === 'useVisualMatching' ||
+      key === 'loiteringThreshold' ||
+      key === 'objectAlertCooldown' ||
+      key === 'useLlmDescriptions'
     ) {
       const topologyJson = this.storage.getItem('topology');
       if (topologyJson) {