@blueharford/scrypted-spatial-awareness 0.1.16 → 0.2.0

package/src/core/tracking-engine.ts

@@ -14,7 +14,7 @@ import sdk, {
   Camera,
   MediaObject,
 } from '@scrypted/sdk';
-import { CameraTopology, findCamera, findConnection, findConnectionsFrom } from '../models/topology';
+import { CameraTopology, findCamera, findConnection, findConnectionsFrom, Landmark } from '../models/topology';
 import {
   TrackedObject,
   ObjectSighting,
@@ -25,6 +25,11 @@ import {
 import { TrackingState } from '../state/tracking-state';
 import { AlertManager } from '../alerts/alert-manager';
 import { ObjectCorrelator } from './object-correlator';
+import {
+  SpatialReasoningEngine,
+  SpatialReasoningConfig,
+  SpatialReasoningResult,
+} from './spatial-reasoning';
 
 const { systemManager } = sdk;
 
@@ -43,6 +48,10 @@ export interface TrackingEngineConfig {
   objectAlertCooldown: number;
   /** Use LLM for enhanced descriptions */
   useLlmDescriptions: boolean;
+  /** Enable landmark learning from AI */
+  enableLandmarkLearning?: boolean;
+  /** Minimum confidence for landmark suggestions */
+  landmarkConfidenceThreshold?: number;
 }
 
 export class TrackingEngine {
@@ -52,13 +61,14 @@ export class TrackingEngine {
   private config: TrackingEngineConfig;
   private console: Console;
   private correlator: ObjectCorrelator;
+  private spatialReasoning: SpatialReasoningEngine;
   private listeners: Map<string, EventListenerRegister> = new Map();
   private pendingTimers: Map<GlobalTrackingId, NodeJS.Timeout> = new Map();
   private lostCheckInterval: NodeJS.Timeout | null = null;
   /** Track last alert time per object to enforce cooldown */
   private objectLastAlertTime: Map<GlobalTrackingId, number> = new Map();
-  /** Cache for LLM device reference */
-  private llmDevice: ObjectDetection | null = null;
+  /** Callback for topology changes (e.g., landmark suggestions) */
+  private onTopologyChange?: (topology: CameraTopology) => void;
 
   constructor(
     topology: CameraTopology,
@@ -73,6 +83,21 @@ export class TrackingEngine {
     this.config = config;
     this.console = console;
     this.correlator = new ObjectCorrelator(topology, config);
+
+    // Initialize spatial reasoning engine
+    const spatialConfig: SpatialReasoningConfig = {
+      enableLlm: config.useLlmDescriptions,
+      enableLandmarkLearning: config.enableLandmarkLearning ?? true,
+      landmarkConfidenceThreshold: config.landmarkConfidenceThreshold ?? 0.7,
+      contextCacheTtl: 60000, // 1 minute cache
+    };
+    this.spatialReasoning = new SpatialReasoningEngine(spatialConfig, console);
+    this.spatialReasoning.updateTopology(topology);
+  }
+
+  /** Set callback for topology changes */
+  setTopologyChangeCallback(callback: (topology: CameraTopology) => void): void {
+    this.onTopologyChange = callback;
   }
 
   /** Start listening to all cameras in topology */
@@ -207,58 +232,69 @@ export class TrackingEngine {
     this.objectLastAlertTime.set(globalId, Date.now());
   }
 
-  /** Try to get LLM-enhanced description for movement */
-  private async getLlmDescription(
+  /** Get spatial reasoning result for movement (uses RAG + LLM) */
+  private async getSpatialDescription(
     tracked: TrackedObject,
-    fromCamera: string,
-    toCamera: string,
-    cameraId: string
-  ): Promise<string | null> {
-    if (!this.config.useLlmDescriptions) return null;
-
+    fromCameraId: string,
+    toCameraId: string,
+    transitTime: number,
+    currentCameraId: string
+  ): Promise<SpatialReasoningResult | null> {
     try {
-      // Find LLM plugin device if not cached
-      if (!this.llmDevice) {
-        for (const id of Object.keys(systemManager.getSystemState())) {
-          const device = systemManager.getDeviceById(id);
-          if (device?.interfaces?.includes(ScryptedInterface.ObjectDetection) &&
-              device.name?.toLowerCase().includes('llm')) {
-            this.llmDevice = device as unknown as ObjectDetection;
-            this.console.log(`Found LLM device: ${device.name}`);
-            break;
-          }
+      // Get snapshot from camera for LLM analysis (if LLM is enabled)
+      let mediaObject: MediaObject | undefined;
+      if (this.config.useLlmDescriptions) {
+        const camera = systemManager.getDeviceById<Camera>(currentCameraId);
+        if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
+          mediaObject = await camera.takePicture();
         }
       }
 
-      if (!this.llmDevice) return null;
-
-      // Get snapshot from camera for LLM analysis
-      const camera = systemManager.getDeviceById<Camera>(cameraId);
-      if (!camera?.interfaces?.includes(ScryptedInterface.Camera)) return null;
+      // Use spatial reasoning engine for rich context-aware description
+      const result = await this.spatialReasoning.generateMovementDescription(
+        tracked,
+        fromCameraId,
+        toCameraId,
+        transitTime,
+        mediaObject
+      );
 
-      const picture = await camera.takePicture();
-      if (!picture) return null;
+      // Optionally trigger landmark learning
+      if (this.config.enableLandmarkLearning && mediaObject) {
+        this.tryLearnLandmark(currentCameraId, mediaObject, tracked.className);
+      }
 
-      // Ask LLM to describe the movement
-      const prompt = `Describe this ${tracked.className} in one short sentence. ` +
-        `They are moving from the ${fromCamera} area towards the ${toCamera}. ` +
-        `Include details like: gender (man/woman), clothing color, vehicle color/type if applicable. ` +
-        `Example: "Man in blue jacket walking from garage towards front door" or ` +
-        `"Black SUV driving from driveway towards street"`;
+      return result;
+    } catch (e) {
+      this.console.warn('Spatial reasoning failed:', e);
+      return null;
+    }
+  }
 
-      const result = await this.llmDevice.detectObjects(picture, {
-        settings: { prompt }
-      } as any);
+  /** Try to learn new landmarks from detections (background task) */
+  private async tryLearnLandmark(
+    cameraId: string,
+    mediaObject: MediaObject,
+    objectClass: string
+  ): Promise<void> {
+    try {
+      // Position is approximate - could be improved with object position from detection
+      const position = { x: 50, y: 50 };
+      const suggestion = await this.spatialReasoning.suggestLandmark(
+        cameraId,
+        mediaObject,
+        objectClass,
+        position
+      );
 
-      // Extract description from LLM response
-      if (result.detections?.[0]?.label) {
-        return result.detections[0].label;
+      if (suggestion) {
+        this.console.log(
+          `AI suggested landmark: ${suggestion.landmark.name} ` +
+          `(${suggestion.landmark.type}, confidence: ${suggestion.landmark.aiConfidence?.toFixed(2)})`
+        );
       }
-
-      return null;
     } catch (e) {
-      this.console.warn('LLM description failed:', e);
-      return null;
+      // Landmark learning is best-effort, don't log errors
     }
   }
 
@@ -300,11 +336,12 @@ export class TrackingEngine {
 
         // Check loitering threshold and per-object cooldown before alerting
         if (this.passesLoiteringThreshold(tracked) && !this.isInAlertCooldown(tracked.globalId)) {
-          // Try to get LLM-enhanced description
-          const llmDescription = await this.getLlmDescription(
+          // Get spatial reasoning result with RAG context
+          const spatialResult = await this.getSpatialDescription(
             tracked,
-            lastSighting.cameraName,
-            sighting.cameraName,
+            lastSighting.cameraId,
+            sighting.cameraId,
+            transitDuration,
             sighting.cameraId
           );
 
@@ -316,8 +353,12 @@ export class TrackingEngine {
             toCameraName: sighting.cameraName,
             transitTime: transitDuration,
             objectClass: sighting.detection.className,
-            objectLabel: llmDescription || sighting.detection.label,
+            objectLabel: spatialResult?.description || sighting.detection.label,
             detectionId: sighting.detectionId,
+            // Include spatial context for enriched alerts
+            pathDescription: spatialResult?.pathDescription,
+            involvedLandmarks: spatialResult?.involvedLandmarks?.map(l => l.name),
+            usedLlm: spatialResult?.usedLlm,
           });
 
           this.recordAlertTime(tracked.globalId);
@@ -354,10 +395,12 @@ export class TrackingEngine {
       // Generate entry alert if this is an entry point
       // Entry alerts also respect loitering threshold and cooldown
       if (isEntryPoint && this.passesLoiteringThreshold(tracked) && !this.isInAlertCooldown(globalId)) {
-        const llmDescription = await this.getLlmDescription(
+        // Get spatial reasoning for entry event
+        const spatialResult = await this.getSpatialDescription(
           tracked,
-          'outside',
-          sighting.cameraName,
+          'outside', // Virtual "outside" location for entry
+          sighting.cameraId,
+          0,
           sighting.cameraId
         );
 
@@ -365,8 +408,10 @@ export class TrackingEngine {
           cameraId: sighting.cameraId,
           cameraName: sighting.cameraName,
           objectClass: sighting.detection.className,
-          objectLabel: llmDescription || sighting.detection.label,
+          objectLabel: spatialResult?.description || sighting.detection.label,
           detectionId: sighting.detectionId,
+          involvedLandmarks: spatialResult?.involvedLandmarks?.map(l => l.name),
+          usedLlm: spatialResult?.usedLlm,
         });
 
         this.recordAlertTime(globalId);
@@ -471,6 +516,45 @@ export class TrackingEngine {
   updateTopology(topology: CameraTopology): void {
     this.topology = topology;
     this.correlator = new ObjectCorrelator(topology, this.config);
+    this.spatialReasoning.updateTopology(topology);
+  }
+
+  /** Get pending landmark suggestions */
+  getPendingLandmarkSuggestions(): import('../models/topology').LandmarkSuggestion[] {
+    return this.spatialReasoning.getPendingSuggestions();
+  }
+
+  /** Accept a landmark suggestion, adding it to topology */
+  acceptLandmarkSuggestion(suggestionId: string): Landmark | null {
+    const landmark = this.spatialReasoning.acceptSuggestion(suggestionId);
+    if (landmark && this.topology) {
+      // Add the accepted landmark to topology
+      if (!this.topology.landmarks) {
+        this.topology.landmarks = [];
+      }
+      this.topology.landmarks.push(landmark);
+
+      // Notify about topology change
+      if (this.onTopologyChange) {
+        this.onTopologyChange(this.topology);
+      }
+    }
+    return landmark;
+  }
+
+  /** Reject a landmark suggestion */
+  rejectLandmarkSuggestion(suggestionId: string): boolean {
+    return this.spatialReasoning.rejectSuggestion(suggestionId);
+  }
+
+  /** Get landmark templates for UI */
+  getLandmarkTemplates(): typeof import('../models/topology').LANDMARK_TEMPLATES {
+    return this.spatialReasoning.getLandmarkTemplates();
+  }
+
+  /** Get the spatial reasoning engine for direct access */
+  getSpatialReasoningEngine(): SpatialReasoningEngine {
+    return this.spatialReasoning;
   }
 
   /** Get current topology */

package/src/main.ts

@@ -15,7 +15,14 @@ import sdk, {
   Readme,
 } from '@scrypted/sdk';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
-import { CameraTopology, createEmptyTopology } from './models/topology';
+import {
+  CameraTopology,
+  createEmptyTopology,
+  Landmark,
+  LandmarkSuggestion,
+  LANDMARK_TEMPLATES,
+  inferRelationships,
+} from './models/topology';
 import { TrackedObject } from './models/tracked-object';
 import { Alert, AlertRule, createDefaultRules } from './models/alert';
 import { TrackingState } from './state/tracking-state';
@@ -106,7 +113,21 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       type: 'boolean',
       defaultValue: true,
       description: 'Use LLM plugin (if installed) to generate descriptive alerts like "Man walking from garage towards front door"',
-      group: 'Tracking',
+      group: 'AI & Spatial Reasoning',
+    },
+    enableLandmarkLearning: {
+      title: 'Learn Landmarks from AI',
+      type: 'boolean',
+      defaultValue: true,
+      description: 'Allow AI to suggest new landmarks based on detected objects and camera context',
+      group: 'AI & Spatial Reasoning',
+    },
+    landmarkConfidenceThreshold: {
+      title: 'Landmark Suggestion Confidence',
+      type: 'number',
+      defaultValue: 0.7,
+      description: 'Minimum AI confidence (0-1) to suggest a landmark',
+      group: 'AI & Spatial Reasoning',
     },
 
     // MQTT Settings
@@ -270,6 +291,8 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       loiteringThreshold: (this.storageSettings.values.loiteringThreshold as number || 3) * 1000,
       objectAlertCooldown: (this.storageSettings.values.objectAlertCooldown as number || 30) * 1000,
       useLlmDescriptions: this.storageSettings.values.useLlmDescriptions as boolean ?? true,
+      enableLandmarkLearning: this.storageSettings.values.enableLandmarkLearning as boolean ?? true,
+      landmarkConfidenceThreshold: this.storageSettings.values.landmarkConfidenceThreshold as number ?? 0.7,
     };
 
     this.trackingEngine = new TrackingEngine(
@@ -280,6 +303,12 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       this.console
     );
 
+    // Set up callback to save topology changes (e.g., from accepted landmark suggestions)
+    this.trackingEngine.setTopologyChangeCallback((updatedTopology) => {
+      this.storage.setItem('topology', JSON.stringify(updatedTopology));
+      this.console.log('Topology auto-saved after change');
+    });
+
     await this.trackingEngine.startTracking();
     this.console.log('Tracking engine started');
   }
@@ -546,7 +575,9 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       key === 'useVisualMatching' ||
       key === 'loiteringThreshold' ||
       key === 'objectAlertCooldown' ||
-      key === 'useLlmDescriptions'
+      key === 'useLlmDescriptions' ||
+      key === 'enableLandmarkLearning' ||
+      key === 'landmarkConfidenceThreshold'
     ) {
       const topologyJson = this.storage.getItem('topology');
       if (topologyJson) {
@@ -609,6 +640,34 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
         return this.handleFloorPlanRequest(request, response);
       }
 
+      if (path.endsWith('/api/landmarks')) {
+        return this.handleLandmarksRequest(request, response);
+      }
+
+      if (path.match(/\/api\/landmarks\/[\w-]+$/)) {
+        const landmarkId = path.split('/').pop()!;
+        return this.handleLandmarkRequest(landmarkId, request, response);
+      }
+
+      if (path.endsWith('/api/landmark-suggestions')) {
+        return this.handleLandmarkSuggestionsRequest(request, response);
+      }
+
+      if (path.match(/\/api\/landmark-suggestions\/[\w-]+\/(accept|reject)$/)) {
+        const parts = path.split('/');
+        const action = parts.pop()!;
+        const suggestionId = parts.pop()!;
+        return this.handleSuggestionActionRequest(suggestionId, action, response);
+      }
+
+      if (path.endsWith('/api/landmark-templates')) {
+        return this.handleLandmarkTemplatesRequest(response);
+      }
+
+      if (path.endsWith('/api/infer-relationships')) {
+        return this.handleInferRelationshipsRequest(response);
+      }
+
       // UI Routes
       if (path.endsWith('/ui/editor') || path.endsWith('/ui/editor/')) {
         return this.serveEditorUI(response);
@@ -791,6 +850,210 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
     }
   }
 
+  private handleLandmarksRequest(request: HttpRequest, response: HttpResponse): void {
+    const topology = this.getTopology();
+    if (!topology) {
+      response.send(JSON.stringify({ landmarks: [] }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    if (request.method === 'GET') {
+      response.send(JSON.stringify({
+        landmarks: topology.landmarks || [],
+      }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+    } else if (request.method === 'POST') {
+      try {
+        const landmark = JSON.parse(request.body!) as Landmark;
+        if (!landmark.id) {
+          landmark.id = `landmark_${Date.now()}`;
+        }
+        if (!topology.landmarks) {
+          topology.landmarks = [];
+        }
+        topology.landmarks.push(landmark);
+        this.storage.setItem('topology', JSON.stringify(topology));
+        if (this.trackingEngine) {
+          this.trackingEngine.updateTopology(topology);
+        }
+        response.send(JSON.stringify({ success: true, landmark }), {
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } catch (e) {
+        response.send(JSON.stringify({ error: 'Invalid landmark data' }), {
+          code: 400,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    }
+  }
+
+  private handleLandmarkRequest(
+    landmarkId: string,
+    request: HttpRequest,
+    response: HttpResponse
+  ): void {
+    const topology = this.getTopology();
+    if (!topology) {
+      response.send(JSON.stringify({ error: 'No topology configured' }), {
+        code: 404,
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    const landmarkIndex = topology.landmarks?.findIndex(l => l.id === landmarkId) ?? -1;
+
+    if (request.method === 'GET') {
+      const landmark = topology.landmarks?.[landmarkIndex];
+      if (landmark) {
+        response.send(JSON.stringify(landmark), {
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } else {
+        response.send(JSON.stringify({ error: 'Landmark not found' }), {
+          code: 404,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    } else if (request.method === 'PUT') {
+      try {
+        const updates = JSON.parse(request.body!) as Partial<Landmark>;
+        if (landmarkIndex >= 0) {
+          topology.landmarks![landmarkIndex] = {
+            ...topology.landmarks![landmarkIndex],
+            ...updates,
+            id: landmarkId, // Preserve ID
+          };
+          this.storage.setItem('topology', JSON.stringify(topology));
+          if (this.trackingEngine) {
+            this.trackingEngine.updateTopology(topology);
+          }
+          response.send(JSON.stringify({ success: true, landmark: topology.landmarks![landmarkIndex] }), {
+            headers: { 'Content-Type': 'application/json' },
+          });
+        } else {
+          response.send(JSON.stringify({ error: 'Landmark not found' }), {
+            code: 404,
+            headers: { 'Content-Type': 'application/json' },
+          });
+        }
+      } catch (e) {
+        response.send(JSON.stringify({ error: 'Invalid landmark data' }), {
+          code: 400,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    } else if (request.method === 'DELETE') {
+      if (landmarkIndex >= 0) {
+        topology.landmarks!.splice(landmarkIndex, 1);
+        this.storage.setItem('topology', JSON.stringify(topology));
+        if (this.trackingEngine) {
+          this.trackingEngine.updateTopology(topology);
+        }
+        response.send(JSON.stringify({ success: true }), {
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } else {
+        response.send(JSON.stringify({ error: 'Landmark not found' }), {
+          code: 404,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    }
+  }
+
+  private handleLandmarkSuggestionsRequest(request: HttpRequest, response: HttpResponse): void {
+    if (!this.trackingEngine) {
+      response.send(JSON.stringify({ suggestions: [] }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    const suggestions = this.trackingEngine.getPendingLandmarkSuggestions();
+    response.send(JSON.stringify({
+      suggestions,
+      count: suggestions.length,
+    }), {
+      headers: { 'Content-Type': 'application/json' },
+    });
+  }
+
+  private handleSuggestionActionRequest(
+    suggestionId: string,
+    action: string,
+    response: HttpResponse
+  ): void {
+    if (!this.trackingEngine) {
+      response.send(JSON.stringify({ error: 'Tracking engine not running' }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    if (action === 'accept') {
+      const landmark = this.trackingEngine.acceptLandmarkSuggestion(suggestionId);
+      if (landmark) {
+        response.send(JSON.stringify({ success: true, landmark }), {
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } else {
+        response.send(JSON.stringify({ error: 'Suggestion not found' }), {
+          code: 404,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    } else if (action === 'reject') {
+      const success = this.trackingEngine.rejectLandmarkSuggestion(suggestionId);
+      if (success) {
+        response.send(JSON.stringify({ success: true }), {
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } else {
+        response.send(JSON.stringify({ error: 'Suggestion not found' }), {
+          code: 404,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    } else {
+      response.send(JSON.stringify({ error: 'Invalid action' }), {
+        code: 400,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+  }
+
+  private handleLandmarkTemplatesRequest(response: HttpResponse): void {
+    response.send(JSON.stringify({
+      templates: LANDMARK_TEMPLATES,
+    }), {
+      headers: { 'Content-Type': 'application/json' },
+    });
+  }
+
+  private handleInferRelationshipsRequest(response: HttpResponse): void {
+    const topology = this.getTopology();
+    if (!topology) {
+      response.send(JSON.stringify({ relationships: [] }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    const inferred = inferRelationships(topology);
+    response.send(JSON.stringify({
+      relationships: inferred,
+      count: inferred.length,
+    }), {
+      headers: { 'Content-Type': 'application/json' },
+    });
+  }
+
   private serveEditorUI(response: HttpResponse): void {
     response.send(EDITOR_HTML, {
       headers: { 'Content-Type': 'text/html' },

package/src/models/alert.ts

@@ -78,6 +78,13 @@ export interface AlertDetails {
   objectLabel?: string;
   /** Thumbnail image URL or data */
   thumbnailUrl?: string;
+  // --- Spatial Context from RAG ---
+  /** Description of the path taken (from spatial reasoning) */
+  pathDescription?: string;
+  /** Names of landmarks involved in the movement */
+  involvedLandmarks?: string[];
+  /** Whether LLM was used for description */
+  usedLlm?: boolean;
 }
 
 /** A condition for alert rules */
@@ -211,9 +218,22 @@ export function generateAlertMessage(
     case 'property_exit':
       return `${objectDesc} exited property via ${details.cameraName || 'unknown camera'}`;
     case 'movement':
+      // If we have a rich description from LLM/RAG, use it
+      if (details.objectLabel && details.usedLlm) {
+        const transitSecs = details.transitTime ? Math.round(details.transitTime / 1000) : 0;
+        const transitStr = transitSecs > 0 ? ` (${transitSecs}s)` : '';
+        // Include path/landmark context if available
+        const pathContext = details.pathDescription ? ` via ${details.pathDescription}` : '';
+        return `${details.objectLabel}${pathContext}${transitStr}`;
+      }
+      // Fallback to basic message with landmark info
       const transitSecs = details.transitTime ? Math.round(details.transitTime / 1000) : 0;
       const transitStr = transitSecs > 0 ? ` (${transitSecs}s transit)` : '';
-      return `${objectDesc} moving from ${details.fromCameraName || 'unknown'} towards ${details.toCameraName || 'unknown'}${transitStr}`;
+      let movementDesc = `${objectDesc} moving from ${details.fromCameraName || 'unknown'} towards ${details.toCameraName || 'unknown'}`;
+      if (details.involvedLandmarks && details.involvedLandmarks.length > 0) {
+        movementDesc += ` near ${details.involvedLandmarks.join(', ')}`;
+      }
+      return `${movementDesc}${transitStr}`;
     case 'unusual_path':
       return `${objectDesc} took unusual path: ${details.actualPath || 'unknown'}`;
     case 'dwell_time':