@blueharford/scrypted-spatial-awareness 0.4.7 → 0.5.0-beta

package/src/core/tracking-engine.ts

@@ -516,31 +516,61 @@ export class TrackingEngine {
         `(ID: ${globalId.slice(0, 8)})`
       );
 
-      // Generate entry alert if this is an entry point
-      // Entry alerts also respect loitering threshold and cooldown
-      if (isEntryPoint && this.passesLoiteringThreshold(tracked) && !this.isInAlertCooldown(globalId)) {
-        // Get spatial reasoning for entry event
-        const spatialResult = await this.getSpatialDescription(
-          tracked,
-          'outside', // Virtual "outside" location for entry
-          sighting.cameraId,
-          0,
-          sighting.cameraId
-        );
+      // Schedule loitering check - alert after object passes loitering threshold
+      // This ensures we don't miss alerts for brief appearances while still filtering noise
+      this.scheduleLoiteringAlert(globalId, sighting, isEntryPoint);
+    }
+  }
+
+  /** Schedule an alert after loitering threshold passes */
+  private scheduleLoiteringAlert(
+    globalId: GlobalTrackingId,
+    sighting: ObjectSighting,
+    isEntryPoint: boolean
+  ): void {
+    // Check after loitering threshold if object is still being tracked
+    setTimeout(async () => {
+      const tracked = this.state.getObject(globalId);
+      if (!tracked || tracked.state !== 'active') return;
+
+      // Check if we've already alerted for this object
+      if (this.isInAlertCooldown(globalId)) return;
+
+      // Generate spatial description
+      const spatialResult = this.spatialReasoning.generateEntryDescription(
+        tracked,
+        sighting.cameraId
+      );
 
+      if (isEntryPoint) {
+        // Entry point - generate property entry alert
         await this.alertManager.checkAndAlert('property_entry', tracked, {
           cameraId: sighting.cameraId,
           cameraName: sighting.cameraName,
           objectClass: sighting.detection.className,
-          objectLabel: spatialResult?.description || sighting.detection.label,
+          objectLabel: spatialResult.description,
           detectionId: sighting.detectionId,
-          involvedLandmarks: spatialResult?.involvedLandmarks?.map(l => l.name),
-          usedLlm: spatialResult?.usedLlm,
+          involvedLandmarks: spatialResult.involvedLandmarks?.map(l => l.name),
+          usedLlm: spatialResult.usedLlm,
+        });
+      } else {
+        // Non-entry point - still alert about activity using movement alert type
+        // This notifies about any activity around the property using topology context
+        await this.alertManager.checkAndAlert('movement', tracked, {
+          cameraId: sighting.cameraId,
+          cameraName: sighting.cameraName,
+          toCameraId: sighting.cameraId,
+          toCameraName: sighting.cameraName,
+          objectClass: sighting.detection.className,
+          objectLabel: spatialResult.description, // Use spatial reasoning description (topology-based)
+          detectionId: sighting.detectionId,
+          involvedLandmarks: spatialResult.involvedLandmarks?.map(l => l.name),
+          usedLlm: spatialResult.usedLlm,
         });
-
-        this.recordAlertTime(globalId);
       }
-    }
+
+      this.recordAlertTime(globalId);
+    }, this.config.loiteringThreshold);
   }
 
   /** Attempt to correlate a sighting with existing tracked objects */
@@ -596,15 +626,23 @@ export class TrackingEngine {
       if (current && current.state === 'pending') {
         this.state.markExited(tracked.globalId, sighting.cameraId, sighting.cameraName);
 
+        // Generate rich exit description using topology context
+        const spatialResult = this.spatialReasoning.generateExitDescription(
+          current,
+          sighting.cameraId
+        );
+
         this.console.log(
-          `Object ${tracked.globalId.slice(0, 8)} exited via ${sighting.cameraName}`
+          `Object ${tracked.globalId.slice(0, 8)} exited: ${spatialResult.description}`
         );
 
         await this.alertManager.checkAndAlert('property_exit', current, {
           cameraId: sighting.cameraId,
           cameraName: sighting.cameraName,
           objectClass: current.className,
-          objectLabel: current.label,
+          objectLabel: spatialResult.description,
+          involvedLandmarks: spatialResult.involvedLandmarks?.map(l => l.name),
+          usedLlm: spatialResult.usedLlm,
         });
       }
       this.pendingTimers.delete(tracked.globalId);

package/src/main.ts

@@ -36,6 +36,8 @@ import { MqttPublisher, MqttConfig } from './integrations/mqtt-publisher';
 import { EDITOR_HTML } from './ui/editor-html';
 import { TRAINING_HTML } from './ui/training-html';
 import { TrainingConfig, TrainingLandmark } from './models/training';
+import { TopologyDiscoveryEngine } from './core/topology-discovery';
+import { DiscoveryConfig, DiscoverySuggestion } from './models/discovery';
 
 const { deviceManager, systemManager, mediaManager } = sdk;
 
@@ -49,6 +51,7 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
   private trackingState: TrackingState;
   private alertManager: AlertManager;
   private mqttPublisher: MqttPublisher | null = null;
+  private discoveryEngine: TopologyDiscoveryEngine | null = null;
   private devices: Map<string, any> = new Map();
 
   storageSettings = new StorageSettings(this, {
@@ -169,6 +172,36 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       group: 'AI & Spatial Reasoning',
     },
 
+    // Auto-Topology Discovery Settings
+    discoveryIntervalHours: {
+      title: 'Auto-Discovery Interval (hours)',
+      type: 'number',
+      defaultValue: 0,
+      description: 'Automatically scan cameras to discover landmarks and connections. Set to 0 to disable. Uses vision LLM to analyze camera views.',
+      group: 'Auto-Topology Discovery',
+    },
+    minLandmarkConfidence: {
+      title: 'Min Landmark Confidence',
+      type: 'number',
+      defaultValue: 0.6,
+      description: 'Minimum confidence (0-1) for discovered landmarks',
+      group: 'Auto-Topology Discovery',
+    },
+    minConnectionConfidence: {
+      title: 'Min Connection Confidence',
+      type: 'number',
+      defaultValue: 0.5,
+      description: 'Minimum confidence (0-1) for suggested camera connections',
+      group: 'Auto-Topology Discovery',
+    },
+    autoAcceptThreshold: {
+      title: 'Auto-Accept Threshold',
+      type: 'number',
+      defaultValue: 0.85,
+      description: 'Suggestions above this confidence are automatically accepted (0-1)',
+      group: 'Auto-Topology Discovery',
+    },
+
     // MQTT Settings
     enableMqtt: {
       title: 'Enable MQTT',
@@ -355,6 +388,33 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
 
     await this.trackingEngine.startTracking();
     this.console.log('Tracking engine started');
+
+    // Initialize or update discovery engine
+    await this.initializeDiscoveryEngine(topology);
+  }
+
+  private async initializeDiscoveryEngine(topology: CameraTopology): Promise<void> {
+    const discoveryConfig: DiscoveryConfig = {
+      discoveryIntervalHours: this.storageSettings.values.discoveryIntervalHours as number ?? 0,
+      autoAcceptThreshold: this.storageSettings.values.autoAcceptThreshold as number ?? 0.85,
+      minLandmarkConfidence: this.storageSettings.values.minLandmarkConfidence as number ?? 0.6,
+      minConnectionConfidence: this.storageSettings.values.minConnectionConfidence as number ?? 0.5,
+    };
+
+    if (this.discoveryEngine) {
+      // Update existing engine
+      this.discoveryEngine.updateConfig(discoveryConfig);
+      this.discoveryEngine.updateTopology(topology);
+    } else {
+      // Create new engine
+      this.discoveryEngine = new TopologyDiscoveryEngine(discoveryConfig, this.console);
+      this.discoveryEngine.updateTopology(topology);
+
+      // Start periodic discovery if enabled
+      if (discoveryConfig.discoveryIntervalHours > 0) {
+        this.discoveryEngine.startPeriodicDiscovery();
+      }
+    }
   }
 
   // ==================== DeviceProvider Implementation ====================
@@ -874,6 +934,27 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
         return this.handleTrainingApplyRequest(response);
       }
 
+      // Discovery endpoints
+      if (path.endsWith('/api/discovery/scan')) {
+        return this.handleDiscoveryScanRequest(response);
+      }
+      if (path.endsWith('/api/discovery/status')) {
+        return this.handleDiscoveryStatusRequest(response);
+      }
+      if (path.endsWith('/api/discovery/suggestions')) {
+        return this.handleDiscoverySuggestionsRequest(response);
+      }
+      if (path.match(/\/api\/discovery\/suggestions\/[\w-]+\/(accept|reject)$/)) {
+        const parts = path.split('/');
+        const action = parts.pop()!;
+        const suggestionId = parts.pop()!;
+        return this.handleDiscoverySuggestionActionRequest(suggestionId, action, response);
+      }
+      if (path.match(/\/api\/discovery\/camera\/[\w-]+$/)) {
+        const cameraId = path.split('/').pop()!;
+        return this.handleDiscoveryCameraAnalysisRequest(cameraId, response);
+      }
+
       // UI Routes
       if (path.endsWith('/ui/editor') || path.endsWith('/ui/editor/')) {
         return this.serveEditorUI(response);
@@ -890,7 +971,7 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       // Default: return info page
       response.send(JSON.stringify({
         name: 'Spatial Awareness Plugin',
-        version: '0.4.0',
+        version: '0.5.0-beta',
         endpoints: {
           api: {
             trackedObjects: '/api/tracked-objects',
@@ -911,6 +992,12 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
               landmark: '/api/training/landmark',
               apply: '/api/training/apply',
             },
+            discovery: {
+              scan: '/api/discovery/scan',
+              status: '/api/discovery/status',
+              suggestions: '/api/discovery/suggestions',
+              camera: '/api/discovery/camera/{cameraId}',
+            },
           },
           ui: {
             editor: '/ui/editor',
@@ -1581,6 +1668,207 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
     });
   }
 
+  // ==================== Discovery Handlers ====================
+
+  private async handleDiscoveryScanRequest(response: HttpResponse): Promise<void> {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({ error: 'Discovery engine not initialized. Configure topology first.' }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    try {
+      this.console.log('[Discovery] Manual scan triggered via API');
+      const correlation = await this.discoveryEngine.runFullDiscovery();
+      const status = this.discoveryEngine.getStatus();
+      const suggestions = this.discoveryEngine.getPendingSuggestions();
+
+      response.send(JSON.stringify({
+        success: true,
+        status,
+        correlation,
+        suggestions,
+      }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+    } catch (e) {
+      this.console.error('[Discovery] Scan failed:', e);
+      response.send(JSON.stringify({ error: `Scan failed: ${(e as Error).message}` }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+  }
+
+  private handleDiscoveryStatusRequest(response: HttpResponse): void {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({
+        isRunning: false,
+        isScanning: false,
+        lastScanTime: null,
+        nextScanTime: null,
+        camerasAnalyzed: 0,
+        pendingSuggestions: 0,
+      }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    const status = this.discoveryEngine.getStatus();
+    response.send(JSON.stringify(status), {
+      headers: { 'Content-Type': 'application/json' },
+    });
+  }
+
+  private handleDiscoverySuggestionsRequest(response: HttpResponse): void {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({ suggestions: [] }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    const suggestions = this.discoveryEngine.getPendingSuggestions();
+    response.send(JSON.stringify({ suggestions }), {
+      headers: { 'Content-Type': 'application/json' },
+    });
+  }
+
+  private handleDiscoverySuggestionActionRequest(
+    suggestionId: string,
+    action: string,
+    response: HttpResponse
+  ): void {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({ error: 'Discovery engine not initialized' }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    if (action === 'accept') {
+      const suggestion = this.discoveryEngine.acceptSuggestion(suggestionId);
+      if (suggestion) {
+        // Apply accepted suggestion to topology
+        this.applyDiscoverySuggestion(suggestion);
+        response.send(JSON.stringify({ success: true, suggestion }), {
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } else {
+        response.send(JSON.stringify({ error: 'Suggestion not found' }), {
+          code: 404,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    } else if (action === 'reject') {
+      const success = this.discoveryEngine.rejectSuggestion(suggestionId);
+      if (success) {
+        response.send(JSON.stringify({ success: true }), {
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } else {
+        response.send(JSON.stringify({ error: 'Suggestion not found' }), {
+          code: 404,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    } else {
+      response.send(JSON.stringify({ error: 'Invalid action' }), {
+        code: 400,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+  }
+
+  private async handleDiscoveryCameraAnalysisRequest(
+    cameraId: string,
+    response: HttpResponse
+  ): Promise<void> {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({ error: 'Discovery engine not initialized' }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    try {
+      const analysis = await this.discoveryEngine.analyzeScene(cameraId);
+      response.send(JSON.stringify(analysis), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+    } catch (e) {
+      response.send(JSON.stringify({ error: `Analysis failed: ${(e as Error).message}` }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+  }
+
+  private applyDiscoverySuggestion(suggestion: DiscoverySuggestion): void {
+    if (!this.trackingEngine) return;
+
+    const topology = this.trackingEngine.getTopology();
+    let updated = false;
+
+    if (suggestion.type === 'landmark' && suggestion.landmark) {
+      // Add new landmark to topology
+      const landmark: Landmark = {
+        id: `landmark_${Date.now()}`,
+        name: suggestion.landmark.name!,
+        type: suggestion.landmark.type!,
+        position: suggestion.landmark.position || { x: 0, y: 0 },
+        description: suggestion.landmark.description,
+        visibleFromCameras: suggestion.landmark.visibleFromCameras,
+        aiSuggested: true,
+        aiConfidence: suggestion.confidence,
+      };
+
+      if (!topology.landmarks) {
+        topology.landmarks = [];
+      }
+      topology.landmarks.push(landmark);
+      updated = true;
+
+      this.console.log(`[Discovery] Added landmark: ${landmark.name}`);
+    }
+
+    if (suggestion.type === 'connection' && suggestion.connection) {
+      // Add new connection to topology
+      const conn = suggestion.connection;
+      const newConnection = {
+        id: `conn_${Date.now()}`,
+        fromCameraId: conn.fromCameraId,
+        toCameraId: conn.toCameraId,
+        bidirectional: conn.bidirectional,
+        // Default exit/entry zones covering full frame
+        exitZone: [[0, 0], [100, 0], [100, 100], [0, 100]] as [number, number][],
+        entryZone: [[0, 0], [100, 0], [100, 100], [0, 100]] as [number, number][],
+        transitTime: {
+          typical: conn.transitSeconds * 1000,
+          min: Math.max(1000, conn.transitSeconds * 500),
+          max: conn.transitSeconds * 2000,
+        },
+        name: conn.via ? `Via ${conn.via}` : undefined,
+      };
+
+      topology.connections.push(newConnection);
+      updated = true;
+
+      this.console.log(`[Discovery] Added connection: ${conn.fromCameraId} -> ${conn.toCameraId}`);
+    }
+
+    if (updated) {
+      // Save updated topology
+      this.storage.setItem('topology', JSON.stringify(topology));
+      this.trackingEngine.updateTopology(topology);
+    }
+  }
+
   private serveEditorUI(response: HttpResponse): void {
     response.send(EDITOR_HTML, {
       headers: { 'Content-Type': 'text/html' },

package/src/models/alert.ts

@@ -214,26 +214,53 @@ export function generateAlertMessage(
 
   switch (type) {
     case 'property_entry':
-      return `${objectDesc} arrived at ${details.cameraName || 'property'}`;
+      // Use the rich description from spatial reasoning if available
+      if (details.objectLabel && details.objectLabel !== details.objectClass) {
+        return details.objectLabel;
+      }
+      // Fallback to basic description
+      if (details.involvedLandmarks && details.involvedLandmarks.length > 0) {
+        return `${objectDesc} entered property near ${details.involvedLandmarks[0]}`;
+      }
+      return `${objectDesc} entered property at ${details.cameraName || 'entrance'}`;
     case 'property_exit':
-      return `${objectDesc} left from ${details.cameraName || 'property'}`;
+      // Use the rich description from spatial reasoning if available
+      if (details.objectLabel && details.objectLabel !== details.objectClass) {
+        return details.objectLabel;
+      }
+      // Fallback to basic description
+      if (details.involvedLandmarks && details.involvedLandmarks.length > 0) {
+        return `${objectDesc} left property via ${details.involvedLandmarks[0]}`;
+      }
+      return `${objectDesc} left property`;
     case 'movement':
-      // If we have a rich description from LLM/RAG, use it
-      if (details.objectLabel && details.usedLlm) {
+      // If objectLabel contains a full description, use it directly
+      if (details.objectLabel && details.objectLabel !== details.objectClass) {
+        // Check if this is a cross-camera movement or initial detection
+        if (details.fromCameraId && details.fromCameraId !== details.toCameraId && details.transitTime) {
+          const transitSecs = Math.round(details.transitTime / 1000);
+          const transitStr = transitSecs > 0 ? ` (${transitSecs}s)` : '';
+          const pathContext = details.pathDescription ? ` via ${details.pathDescription}` : '';
+          return `${details.objectLabel}${pathContext}${transitStr}`;
+        }
+        // Initial detection - use the label directly
+        return details.objectLabel;
+      }
+      // Cross-camera movement with basic info
+      if (details.fromCameraId && details.fromCameraId !== details.toCameraId) {
         const transitSecs = details.transitTime ? Math.round(details.transitTime / 1000) : 0;
-        const transitStr = transitSecs > 0 ? ` (${transitSecs}s)` : '';
-        // Include path/landmark context if available
-        const pathContext = details.pathDescription ? ` via ${details.pathDescription}` : '';
-        return `${details.objectLabel}${pathContext}${transitStr}`;
+        const transitStr = transitSecs > 0 ? ` (${transitSecs}s transit)` : '';
+        let movementDesc = `${objectDesc} moving from ${details.fromCameraName || 'unknown'} towards ${details.toCameraName || 'unknown'}`;
+        if (details.involvedLandmarks && details.involvedLandmarks.length > 0) {
+          movementDesc += ` near ${details.involvedLandmarks.join(', ')}`;
+        }
+        return `${movementDesc}${transitStr}`;
       }
-      // Fallback to basic message with landmark info
-      const transitSecs = details.transitTime ? Math.round(details.transitTime / 1000) : 0;
-      const transitStr = transitSecs > 0 ? ` (${transitSecs}s transit)` : '';
-      let movementDesc = `${objectDesc} moving from ${details.fromCameraName || 'unknown'} towards ${details.toCameraName || 'unknown'}`;
+      // Initial detection without full label
       if (details.involvedLandmarks && details.involvedLandmarks.length > 0) {
-        movementDesc += ` near ${details.involvedLandmarks.join(', ')}`;
+        return `${objectDesc} detected near ${details.involvedLandmarks[0]}`;
       }
-      return `${movementDesc}${transitStr}`;
+      return `${objectDesc} detected at ${details.cameraName || 'camera'}`;
     case 'unusual_path':
       return `${objectDesc} took unusual path: ${details.actualPath || 'unknown'}`;
     case 'dwell_time':