@blueharford/scrypted-spatial-awareness 0.4.8-beta.1 → 0.5.0-beta

package/src/main.ts

@@ -36,6 +36,8 @@ import { MqttPublisher, MqttConfig } from './integrations/mqtt-publisher';
 import { EDITOR_HTML } from './ui/editor-html';
 import { TRAINING_HTML } from './ui/training-html';
 import { TrainingConfig, TrainingLandmark } from './models/training';
+import { TopologyDiscoveryEngine } from './core/topology-discovery';
+import { DiscoveryConfig, DiscoverySuggestion } from './models/discovery';
 
 const { deviceManager, systemManager, mediaManager } = sdk;
 
@@ -49,6 +51,7 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
   private trackingState: TrackingState;
   private alertManager: AlertManager;
   private mqttPublisher: MqttPublisher | null = null;
+  private discoveryEngine: TopologyDiscoveryEngine | null = null;
   private devices: Map<string, any> = new Map();
 
   storageSettings = new StorageSettings(this, {
@@ -169,6 +172,36 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       group: 'AI & Spatial Reasoning',
     },
 
+    // Auto-Topology Discovery Settings
+    discoveryIntervalHours: {
+      title: 'Auto-Discovery Interval (hours)',
+      type: 'number',
+      defaultValue: 0,
+      description: 'Automatically scan cameras to discover landmarks and connections. Set to 0 to disable. Uses vision LLM to analyze camera views.',
+      group: 'Auto-Topology Discovery',
+    },
+    minLandmarkConfidence: {
+      title: 'Min Landmark Confidence',
+      type: 'number',
+      defaultValue: 0.6,
+      description: 'Minimum confidence (0-1) for discovered landmarks',
+      group: 'Auto-Topology Discovery',
+    },
+    minConnectionConfidence: {
+      title: 'Min Connection Confidence',
+      type: 'number',
+      defaultValue: 0.5,
+      description: 'Minimum confidence (0-1) for suggested camera connections',
+      group: 'Auto-Topology Discovery',
+    },
+    autoAcceptThreshold: {
+      title: 'Auto-Accept Threshold',
+      type: 'number',
+      defaultValue: 0.85,
+      description: 'Suggestions above this confidence are automatically accepted (0-1)',
+      group: 'Auto-Topology Discovery',
+    },
+
     // MQTT Settings
     enableMqtt: {
       title: 'Enable MQTT',
@@ -355,6 +388,33 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
 
     await this.trackingEngine.startTracking();
     this.console.log('Tracking engine started');
+
+    // Initialize or update discovery engine
+    await this.initializeDiscoveryEngine(topology);
+  }
+
+  private async initializeDiscoveryEngine(topology: CameraTopology): Promise<void> {
+    const discoveryConfig: DiscoveryConfig = {
+      discoveryIntervalHours: this.storageSettings.values.discoveryIntervalHours as number ?? 0,
+      autoAcceptThreshold: this.storageSettings.values.autoAcceptThreshold as number ?? 0.85,
+      minLandmarkConfidence: this.storageSettings.values.minLandmarkConfidence as number ?? 0.6,
+      minConnectionConfidence: this.storageSettings.values.minConnectionConfidence as number ?? 0.5,
+    };
+
+    if (this.discoveryEngine) {
+      // Update existing engine
+      this.discoveryEngine.updateConfig(discoveryConfig);
+      this.discoveryEngine.updateTopology(topology);
+    } else {
+      // Create new engine
+      this.discoveryEngine = new TopologyDiscoveryEngine(discoveryConfig, this.console);
+      this.discoveryEngine.updateTopology(topology);
+
+      // Start periodic discovery if enabled
+      if (discoveryConfig.discoveryIntervalHours > 0) {
+        this.discoveryEngine.startPeriodicDiscovery();
+      }
+    }
   }
 
   // ==================== DeviceProvider Implementation ====================
@@ -874,6 +934,27 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
         return this.handleTrainingApplyRequest(response);
       }
 
+      // Discovery endpoints
+      if (path.endsWith('/api/discovery/scan')) {
+        return this.handleDiscoveryScanRequest(response);
+      }
+      if (path.endsWith('/api/discovery/status')) {
+        return this.handleDiscoveryStatusRequest(response);
+      }
+      if (path.endsWith('/api/discovery/suggestions')) {
+        return this.handleDiscoverySuggestionsRequest(response);
+      }
+      if (path.match(/\/api\/discovery\/suggestions\/[\w-]+\/(accept|reject)$/)) {
+        const parts = path.split('/');
+        const action = parts.pop()!;
+        const suggestionId = parts.pop()!;
+        return this.handleDiscoverySuggestionActionRequest(suggestionId, action, response);
+      }
+      if (path.match(/\/api\/discovery\/camera\/[\w-]+$/)) {
+        const cameraId = path.split('/').pop()!;
+        return this.handleDiscoveryCameraAnalysisRequest(cameraId, response);
+      }
+
       // UI Routes
       if (path.endsWith('/ui/editor') || path.endsWith('/ui/editor/')) {
         return this.serveEditorUI(response);
@@ -890,7 +971,7 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       // Default: return info page
       response.send(JSON.stringify({
         name: 'Spatial Awareness Plugin',
-        version: '0.4.0',
+        version: '0.5.0-beta',
         endpoints: {
           api: {
             trackedObjects: '/api/tracked-objects',
@@ -911,6 +992,12 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
               landmark: '/api/training/landmark',
               apply: '/api/training/apply',
             },
+            discovery: {
+              scan: '/api/discovery/scan',
+              status: '/api/discovery/status',
+              suggestions: '/api/discovery/suggestions',
+              camera: '/api/discovery/camera/{cameraId}',
+            },
           },
           ui: {
             editor: '/ui/editor',
@@ -1581,6 +1668,207 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
     });
   }
 
+  // ==================== Discovery Handlers ====================
+
+  private async handleDiscoveryScanRequest(response: HttpResponse): Promise<void> {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({ error: 'Discovery engine not initialized. Configure topology first.' }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    try {
+      this.console.log('[Discovery] Manual scan triggered via API');
+      const correlation = await this.discoveryEngine.runFullDiscovery();
+      const status = this.discoveryEngine.getStatus();
+      const suggestions = this.discoveryEngine.getPendingSuggestions();
+
+      response.send(JSON.stringify({
+        success: true,
+        status,
+        correlation,
+        suggestions,
+      }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+    } catch (e) {
+      this.console.error('[Discovery] Scan failed:', e);
+      response.send(JSON.stringify({ error: `Scan failed: ${(e as Error).message}` }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+  }
+
+  private handleDiscoveryStatusRequest(response: HttpResponse): void {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({
+        isRunning: false,
+        isScanning: false,
+        lastScanTime: null,
+        nextScanTime: null,
+        camerasAnalyzed: 0,
+        pendingSuggestions: 0,
+      }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    const status = this.discoveryEngine.getStatus();
+    response.send(JSON.stringify(status), {
+      headers: { 'Content-Type': 'application/json' },
+    });
+  }
+
+  private handleDiscoverySuggestionsRequest(response: HttpResponse): void {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({ suggestions: [] }), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    const suggestions = this.discoveryEngine.getPendingSuggestions();
+    response.send(JSON.stringify({ suggestions }), {
+      headers: { 'Content-Type': 'application/json' },
+    });
+  }
+
+  private handleDiscoverySuggestionActionRequest(
+    suggestionId: string,
+    action: string,
+    response: HttpResponse
+  ): void {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({ error: 'Discovery engine not initialized' }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    if (action === 'accept') {
+      const suggestion = this.discoveryEngine.acceptSuggestion(suggestionId);
+      if (suggestion) {
+        // Apply accepted suggestion to topology
+        this.applyDiscoverySuggestion(suggestion);
+        response.send(JSON.stringify({ success: true, suggestion }), {
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } else {
+        response.send(JSON.stringify({ error: 'Suggestion not found' }), {
+          code: 404,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    } else if (action === 'reject') {
+      const success = this.discoveryEngine.rejectSuggestion(suggestionId);
+      if (success) {
+        response.send(JSON.stringify({ success: true }), {
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } else {
+        response.send(JSON.stringify({ error: 'Suggestion not found' }), {
+          code: 404,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    } else {
+      response.send(JSON.stringify({ error: 'Invalid action' }), {
+        code: 400,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+  }
+
+  private async handleDiscoveryCameraAnalysisRequest(
+    cameraId: string,
+    response: HttpResponse
+  ): Promise<void> {
+    if (!this.discoveryEngine) {
+      response.send(JSON.stringify({ error: 'Discovery engine not initialized' }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+      return;
+    }
+
+    try {
+      const analysis = await this.discoveryEngine.analyzeScene(cameraId);
+      response.send(JSON.stringify(analysis), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+    } catch (e) {
+      response.send(JSON.stringify({ error: `Analysis failed: ${(e as Error).message}` }), {
+        code: 500,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+  }
+
+  private applyDiscoverySuggestion(suggestion: DiscoverySuggestion): void {
+    if (!this.trackingEngine) return;
+
+    const topology = this.trackingEngine.getTopology();
+    let updated = false;
+
+    if (suggestion.type === 'landmark' && suggestion.landmark) {
+      // Add new landmark to topology
+      const landmark: Landmark = {
+        id: `landmark_${Date.now()}`,
+        name: suggestion.landmark.name!,
+        type: suggestion.landmark.type!,
+        position: suggestion.landmark.position || { x: 0, y: 0 },
+        description: suggestion.landmark.description,
+        visibleFromCameras: suggestion.landmark.visibleFromCameras,
+        aiSuggested: true,
+        aiConfidence: suggestion.confidence,
+      };
+
+      if (!topology.landmarks) {
+        topology.landmarks = [];
+      }
+      topology.landmarks.push(landmark);
+      updated = true;
+
+      this.console.log(`[Discovery] Added landmark: ${landmark.name}`);
+    }
+
+    if (suggestion.type === 'connection' && suggestion.connection) {
+      // Add new connection to topology
+      const conn = suggestion.connection;
+      const newConnection = {
+        id: `conn_${Date.now()}`,
+        fromCameraId: conn.fromCameraId,
+        toCameraId: conn.toCameraId,
+        bidirectional: conn.bidirectional,
+        // Default exit/entry zones covering full frame
+        exitZone: [[0, 0], [100, 0], [100, 100], [0, 100]] as [number, number][],
+        entryZone: [[0, 0], [100, 0], [100, 100], [0, 100]] as [number, number][],
+        transitTime: {
+          typical: conn.transitSeconds * 1000,
+          min: Math.max(1000, conn.transitSeconds * 500),
+          max: conn.transitSeconds * 2000,
+        },
+        name: conn.via ? `Via ${conn.via}` : undefined,
+      };
+
+      topology.connections.push(newConnection);
+      updated = true;
+
+      this.console.log(`[Discovery] Added connection: ${conn.fromCameraId} -> ${conn.toCameraId}`);
+    }
+
+    if (updated) {
+      // Save updated topology
+      this.storage.setItem('topology', JSON.stringify(topology));
+      this.trackingEngine.updateTopology(topology);
+    }
+  }
+
   private serveEditorUI(response: HttpResponse): void {
     response.send(EDITOR_HTML, {
       headers: { 'Content-Type': 'text/html' },

package/src/models/discovery.ts

@@ -0,0 +1,210 @@
+/**
+ * Auto-Topology Discovery Models
+ * Types for scene analysis and topology discovery via vision LLM
+ */
+
+import { LandmarkType, Landmark, CameraConnection } from './topology';
+
+// ==================== Discovery Configuration ====================
+
+/** Configuration for the topology discovery engine */
+export interface DiscoveryConfig {
+  /** Hours between automatic discovery scans (0 = disabled) */
+  discoveryIntervalHours: number;
+  /** Minimum confidence threshold for auto-accepting suggestions */
+  autoAcceptThreshold: number;
+  /** Minimum confidence for landmark suggestions */
+  minLandmarkConfidence: number;
+  /** Minimum confidence for connection suggestions */
+  minConnectionConfidence: number;
+}
+
+/** Default discovery configuration */
+export const DEFAULT_DISCOVERY_CONFIG: DiscoveryConfig = {
+  discoveryIntervalHours: 0, // Disabled by default
+  autoAcceptThreshold: 0.85,
+  minLandmarkConfidence: 0.6,
+  minConnectionConfidence: 0.5,
+};
+
+/** Rate limit warning thresholds (in hours) */
+export const RATE_LIMIT_WARNING_THRESHOLD = 1; // Warn if interval is less than 1 hour
+
+// ==================== Scene Analysis Types ====================
+
+/** Zone types that can be discovered in camera views */
+export type DiscoveredZoneType =
+  | 'yard'       // Front yard, back yard, side yard
+  | 'driveway'   // Driveway, parking area
+  | 'street'     // Street, road, sidewalk
+  | 'patio'      // Patio, deck
+  | 'walkway'    // Walkways, paths
+  | 'parking'    // Parking lot, parking space
+  | 'garden'     // Garden, landscaped area
+  | 'pool'       // Pool area
+  | 'unknown';   // Unidentified area
+
+/** A zone discovered in a camera view */
+export interface DiscoveredZone {
+  /** Name of the zone (e.g., "Front Yard", "Driveway") */
+  name: string;
+  /** Type classification */
+  type: DiscoveredZoneType;
+  /** Estimated percentage of frame this zone covers (0-1) */
+  coverage: number;
+  /** Description from LLM analysis */
+  description: string;
+  /** Bounding box in normalized coordinates [x, y, width, height] (0-1) */
+  boundingBox?: [number, number, number, number];
+}
+
+/** A landmark discovered in a camera view */
+export interface DiscoveredLandmark {
+  /** Name of the landmark */
+  name: string;
+  /** Type classification */
+  type: LandmarkType;
+  /** Confidence score from LLM (0-1) */
+  confidence: number;
+  /** Bounding box in normalized coordinates [x, y, width, height] (0-1) */
+  boundingBox?: [number, number, number, number];
+  /** Description from LLM analysis */
+  description: string;
+}
+
+/** Edge analysis - what's visible at frame boundaries */
+export interface EdgeAnalysis {
+  /** What's visible at the top edge */
+  top: string;
+  /** What's visible at the left edge */
+  left: string;
+  /** What's visible at the right edge */
+  right: string;
+  /** What's visible at the bottom edge */
+  bottom: string;
+}
+
+/** Complete scene analysis result for a single camera */
+export interface SceneAnalysis {
+  /** Camera device ID */
+  cameraId: string;
+  /** Camera name for reference */
+  cameraName: string;
+  /** When this analysis was performed */
+  timestamp: number;
+  /** Landmarks discovered in the scene */
+  landmarks: DiscoveredLandmark[];
+  /** Zones discovered in the scene */
+  zones: DiscoveredZone[];
+  /** Edge analysis for camera correlation */
+  edges: EdgeAnalysis;
+  /** Estimated camera facing direction */
+  orientation: 'north' | 'south' | 'east' | 'west' | 'northeast' | 'northwest' | 'southeast' | 'southwest' | 'unknown';
+  /** Camera IDs that may have overlapping views */
+  potentialOverlaps: string[];
+  /** Whether this analysis is still valid (not stale) */
+  isValid: boolean;
+  /** Error message if analysis failed */
+  error?: string;
+}
+
+// ==================== Correlation Types ====================
+
+/** A landmark that appears in multiple camera views */
+export interface SharedLandmark {
+  /** Suggested name for this landmark */
+  name: string;
+  /** Suggested type */
+  type: LandmarkType;
+  /** Camera IDs where this landmark is visible */
+  seenByCameras: string[];
+  /** Confidence in this correlation */
+  confidence: number;
+  /** Description of the shared landmark */
+  description?: string;
+}
+
+/** A suggested connection between cameras */
+export interface SuggestedConnection {
+  /** Source camera ID */
+  fromCameraId: string;
+  /** Destination camera ID */
+  toCameraId: string;
+  /** Estimated transit time in seconds */
+  transitSeconds: number;
+  /** Path description (e.g., "via driveway", "through front yard") */
+  via: string;
+  /** Confidence in this suggestion */
+  confidence: number;
+  /** Whether this is bidirectional */
+  bidirectional: boolean;
+}
+
+/** Result of correlating scenes across multiple cameras */
+export interface TopologyCorrelation {
+  /** Landmarks that appear in multiple camera views */
+  sharedLandmarks: SharedLandmark[];
+  /** Suggested connections between cameras */
+  suggestedConnections: SuggestedConnection[];
+  /** Overall description of property layout from LLM */
+  layoutDescription: string;
+  /** When this correlation was performed */
+  timestamp: number;
+}
+
+// ==================== Discovery Suggestions ====================
+
+/** Status of a discovery suggestion */
+export type SuggestionStatus = 'pending' | 'accepted' | 'rejected' | 'merged';
+
+/** A pending discovery suggestion for user review */
+export interface DiscoverySuggestion {
+  /** Unique ID for this suggestion */
+  id: string;
+  /** Type of suggestion */
+  type: 'landmark' | 'zone' | 'connection';
+  /** When this was discovered */
+  timestamp: number;
+  /** Cameras that contributed to this discovery */
+  sourceCameras: string[];
+  /** Confidence score */
+  confidence: number;
+  /** Current status */
+  status: SuggestionStatus;
+  /** The suggested landmark (if type is 'landmark') */
+  landmark?: Partial<Landmark>;
+  /** The suggested zone (if type is 'zone') */
+  zone?: DiscoveredZone;
+  /** The suggested connection (if type is 'connection') */
+  connection?: SuggestedConnection;
+}
+
+// ==================== Discovery Status ====================
+
+/** Current status of the discovery engine */
+export interface DiscoveryStatus {
+  /** Whether discovery is currently running */
+  isRunning: boolean;
+  /** Whether a scan is in progress */
+  isScanning: boolean;
+  /** Last scan timestamp */
+  lastScanTime: number | null;
+  /** Next scheduled scan timestamp */
+  nextScanTime: number | null;
+  /** Number of cameras analyzed */
+  camerasAnalyzed: number;
+  /** Number of pending suggestions */
+  pendingSuggestions: number;
+  /** Any error from last scan */
+  lastError?: string;
+}
+
+/** Default discovery status */
+export const DEFAULT_DISCOVERY_STATUS: DiscoveryStatus = {
+  isRunning: false,
+  isScanning: false,
+  lastScanTime: null,
+  nextScanTime: null,
+  camerasAnalyzed: 0,
+  pendingSuggestions: 0,
+};

package/src/models/topology.ts

@@ -198,6 +198,57 @@ export interface CameraZoneMapping {
   zone: ClipPath;
 }
 
+// ==================== Drawn Zones (Floor Plan) ====================
+
+/** Zone type for floor plan zones */
+export type DrawnZoneType =
+  | 'yard'        // Front yard, back yard, side yard
+  | 'driveway'    // Driveway, parking area
+  | 'street'      // Street, sidewalk
+  | 'patio'       // Patio, deck
+  | 'walkway'     // Walkways, paths
+  | 'parking'     // Parking lot, parking space
+  | 'garden'      // Garden, landscaped area
+  | 'pool'        // Pool area
+  | 'garage'      // Garage area
+  | 'entrance'    // Entry areas
+  | 'custom';     // Custom zone type
+
+/** Zone colors by type */
+export const ZONE_TYPE_COLORS: Record<DrawnZoneType, string> = {
+  yard: 'rgba(76, 175, 80, 0.3)',      // Green
+  driveway: 'rgba(158, 158, 158, 0.3)', // Gray
+  street: 'rgba(96, 96, 96, 0.3)',      // Dark gray
+  patio: 'rgba(255, 152, 0, 0.3)',      // Orange
+  walkway: 'rgba(121, 85, 72, 0.3)',    // Brown
+  parking: 'rgba(189, 189, 189, 0.3)',  // Light gray
+  garden: 'rgba(139, 195, 74, 0.3)',    // Light green
+  pool: 'rgba(33, 150, 243, 0.3)',      // Blue
+  garage: 'rgba(117, 117, 117, 0.3)',   // Medium gray
+  entrance: 'rgba(233, 30, 99, 0.3)',   // Pink
+  custom: 'rgba(156, 39, 176, 0.3)',    // Purple
+};
+
+/** A zone drawn on the floor plan */
+export interface DrawnZone {
+  /** Unique identifier */
+  id: string;
+  /** Display name */
+  name: string;
+  /** Zone type */
+  type: DrawnZoneType;
+  /** Polygon points on floor plan (x, y coordinates) */
+  polygon: Point[];
+  /** Custom color override (optional) */
+  color?: string;
+  /** Linked camera IDs that can see this zone */
+  linkedCameras?: string[];
+  /** Linked landmark IDs within this zone */
+  linkedLandmarks?: string[];
+  /** Description for context */
+  description?: string;
+}
+
 // ==================== Spatial Relationships ====================
 
 /** Types of spatial relationships between entities */
@@ -297,6 +348,8 @@ export interface CameraTopology {
   relationships: SpatialRelationship[];
   /** Named zones spanning multiple cameras */
   globalZones: GlobalZone[];
+  /** Zones drawn on the floor plan */
+  drawnZones?: DrawnZone[];
   /** Floor plan configuration (optional) */
   floorPlan?: FloorPlanConfig;
   /** Pending AI landmark suggestions */