@blueharford/scrypted-spatial-awareness 0.4.7 → 0.5.0-beta

package/src/core/spatial-reasoning.ts

@@ -9,6 +9,8 @@ import sdk, {
   ObjectDetection,
   Camera,
   MediaObject,
+  ScryptedDevice,
+  ScryptedMimeTypes,
 } from '@scrypted/sdk';
 import {
   CameraTopology,
@@ -25,7 +27,7 @@ import {
 } from '../models/topology';
 import { TrackedObject, ObjectSighting } from '../models/tracked-object';
 
-const { systemManager } = sdk;
+const { systemManager, mediaManager } = sdk;
 
 /** Configuration for the spatial reasoning engine */
 export interface SpatialReasoningConfig {
@@ -61,11 +63,40 @@ interface ContextChunk {
   metadata: Record<string, any>;
 }
 
+/** Interface for ChatCompletion devices (from @scrypted/llm plugin) */
+interface ChatCompletionDevice extends ScryptedDevice {
+  getChatCompletion?(params: any): Promise<any>;
+  streamChatCompletion?(params: any): AsyncGenerator<any>;
+}
+
+/**
+ * Convert a MediaObject to a base64 data URL for vision LLM consumption
+ * @param mediaObject - MediaObject from camera.takePicture()
+ * @returns Base64 data URL (data:image/jpeg;base64,...) or null if conversion fails
+ */
+export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<string | null> {
+  try {
+    // Convert MediaObject to Buffer using mediaManager
+    const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.Image);
+
+    // Convert buffer to base64
+    const base64 = buffer.toString('base64');
+
+    // Determine MIME type - default to JPEG for camera images
+    const mimeType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
+
+    return `data:${mimeType};base64,${base64}`;
+  } catch (e) {
+    console.warn('Failed to convert MediaObject to base64:', e);
+    return null;
+  }
+}
+
 export class SpatialReasoningEngine {
   private config: SpatialReasoningConfig;
   private console: Console;
   private topology: CameraTopology | null = null;
-  private llmDevice: ObjectDetection | null = null;
+  private llmDevice: ChatCompletionDevice | null = null;
   private contextChunks: ContextChunk[] = [];
   private topologyContextCache: string | null = null;
   private contextCacheTime: number = 0;
@@ -303,30 +334,213 @@ export class SpatialReasoningEngine {
     return relevant;
   }
 
-  /** Find or initialize LLM device */
-  private async findLlmDevice(): Promise<ObjectDetection | null> {
+  private llmSearched: boolean = false;
+  private llmProvider: string | null = null;
+
+  /** Find or initialize LLM device - looks for ChatCompletion interface from @scrypted/llm plugin */
+  private async findLlmDevice(): Promise<ChatCompletionDevice | null> {
     if (this.llmDevice) return this.llmDevice;
+    if (this.llmSearched) return null; // Already searched and found nothing
+
+    this.llmSearched = true;
 
     try {
+      // Look for devices with ChatCompletion interface (the correct interface for @scrypted/llm)
       for (const id of Object.keys(systemManager.getSystemState())) {
         const device = systemManager.getDeviceById(id);
-        if (device?.interfaces?.includes(ScryptedInterface.ObjectDetection)) {
-          const name = device.name?.toLowerCase() || '';
-          if (name.includes('llm') || name.includes('gpt') || name.includes('claude') ||
-              name.includes('ollama') || name.includes('gemini')) {
-            this.llmDevice = device as unknown as ObjectDetection;
-            this.console.log(`Found LLM device: ${device.name}`);
-            return this.llmDevice;
+        if (!device) continue;
+
+        // Check if this device has ChatCompletion interface
+        // The @scrypted/llm plugin exposes ChatCompletion, not ObjectDetection
+        if (device.interfaces?.includes('ChatCompletion')) {
+          const deviceName = device.name?.toLowerCase() || '';
+          const pluginId = (device as any).pluginId?.toLowerCase() || '';
+
+          // Identify the provider type for logging
+          let providerType = 'Unknown';
+          if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
+            providerType = 'Scrypted LLM';
+          }
+          if (deviceName.includes('openai') || deviceName.includes('gpt')) {
+            providerType = 'OpenAI';
+          } else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
+            providerType = 'Anthropic';
+          } else if (deviceName.includes('ollama')) {
+            providerType = 'Ollama';
+          } else if (deviceName.includes('gemini') || deviceName.includes('google')) {
+            providerType = 'Google';
+          } else if (deviceName.includes('llama')) {
+            providerType = 'llama.cpp';
           }
+
+          this.llmDevice = device as unknown as ChatCompletionDevice;
+          this.llmProvider = `${providerType} (${device.name})`;
+          this.console.log(`[LLM] Connected to ${providerType}: ${device.name}`);
+          this.console.log(`[LLM] Plugin: ${pluginId || 'N/A'}`);
+          this.console.log(`[LLM] Interfaces: ${device.interfaces?.join(', ')}`);
+          return this.llmDevice;
         }
       }
+
+      // If we get here, no LLM plugin found
+      this.console.warn('[LLM] No ChatCompletion device found. Install @scrypted/llm for enhanced descriptions.');
+      this.console.warn('[LLM] Falling back to rule-based descriptions using topology data.');
+
     } catch (e) {
-      this.console.warn('Error finding LLM device:', e);
+      this.console.error('[LLM] Error searching for LLM device:', e);
     }
 
     return null;
   }
 
+  /** Get the current LLM provider name */
+  getLlmProvider(): string | null {
+    return this.llmProvider;
+  }
+
+  /** Check if LLM is available */
+  isLlmAvailable(): boolean {
+    return this.llmDevice !== null;
+  }
+
+  /** Generate entry description when object enters property */
+  generateEntryDescription(
+    tracked: TrackedObject,
+    cameraId: string
+  ): SpatialReasoningResult {
+    if (!this.topology) {
+      return {
+        description: `${this.capitalizeFirst(tracked.className)} entered property`,
+        involvedLandmarks: [],
+        confidence: 0.5,
+        usedLlm: false,
+      };
+    }
+
+    const camera = findCamera(this.topology, cameraId);
+    if (!camera) {
+      return {
+        description: `${this.capitalizeFirst(tracked.className)} entered property`,
+        involvedLandmarks: [],
+        confidence: 0.5,
+        usedLlm: false,
+      };
+    }
+
+    const landmarks = getLandmarksVisibleFromCamera(this.topology, cameraId);
+    const objectType = this.capitalizeFirst(tracked.className);
+
+    // Build entry description using topology context
+    const location = this.describeLocation(camera, landmarks, 'to');
+
+    // Check if we can determine where they came from (e.g., street, neighbor)
+    const entryLandmark = landmarks.find(l => l.isEntryPoint);
+    const streetLandmark = landmarks.find(l => l.type === 'street');
+    const neighborLandmark = landmarks.find(l => l.type === 'neighbor');
+
+    let source = '';
+    if (streetLandmark) {
+      source = ` from ${streetLandmark.name}`;
+    } else if (neighborLandmark) {
+      source = ` from ${neighborLandmark.name}`;
+    }
+
+    return {
+      description: `${objectType} arrived at ${location}${source}`,
+      involvedLandmarks: landmarks,
+      confidence: 0.8,
+      usedLlm: false,
+    };
+  }
+
+  /** Generate exit description when object leaves property */
+  generateExitDescription(
+    tracked: TrackedObject,
+    cameraId: string
+  ): SpatialReasoningResult {
+    if (!this.topology) {
+      return {
+        description: `${this.capitalizeFirst(tracked.className)} left property`,
+        involvedLandmarks: [],
+        confidence: 0.5,
+        usedLlm: false,
+      };
+    }
+
+    const camera = findCamera(this.topology, cameraId);
+    if (!camera) {
+      return {
+        description: `${this.capitalizeFirst(tracked.className)} left property`,
+        involvedLandmarks: [],
+        confidence: 0.5,
+        usedLlm: false,
+      };
+    }
+
+    const landmarks = getLandmarksVisibleFromCamera(this.topology, cameraId);
+    const objectType = this.capitalizeFirst(tracked.className);
+
+    // Build exit description
+    const location = this.describeLocation(camera, landmarks, 'from');
+
+    // Check for exit point landmarks
+    const exitLandmark = landmarks.find(l => l.isExitPoint);
+    const streetLandmark = landmarks.find(l => l.type === 'street');
+
+    let destination = '';
+    if (streetLandmark) {
+      destination = ` towards ${streetLandmark.name}`;
+    } else if (exitLandmark) {
+      destination = ` via ${exitLandmark.name}`;
+    }
+
+    // Include time on property if available
+    const dwellTime = Math.round((tracked.lastSeen - tracked.firstSeen) / 1000);
+    let timeContext = '';
+    if (dwellTime > 60) {
+      timeContext = ` after ${Math.round(dwellTime / 60)}m on property`;
+    } else if (dwellTime > 10) {
+      timeContext = ` after ${dwellTime}s`;
+    }
+
+    // Summarize journey if they visited multiple cameras (use landmarks from topology)
+    let journeyContext = '';
+    if (tracked.journey.length > 0 && this.topology) {
+      const visitedLandmarks: string[] = [];
+
+      // Get landmarks from entry camera
+      if (tracked.entryCamera) {
+        const entryLandmarks = getLandmarksVisibleFromCamera(this.topology, tracked.entryCamera);
+        const entryLandmark = entryLandmarks.find(l => l.isEntryPoint || l.type === 'access') || entryLandmarks[0];
+        if (entryLandmark) {
+          visitedLandmarks.push(entryLandmark.name);
+        }
+      }
+
+      // Get landmarks from journey segments
+      for (const segment of tracked.journey) {
+        const segmentLandmarks = getLandmarksVisibleFromCamera(this.topology, segment.toCameraId);
+        const segmentLandmark = segmentLandmarks.find(l =>
+          !visitedLandmarks.includes(l.name) && (l.type === 'access' || l.type === 'zone' || l.type === 'structure')
+        );
+        if (segmentLandmark && !visitedLandmarks.includes(segmentLandmark.name)) {
+          visitedLandmarks.push(segmentLandmark.name);
+        }
+      }
+
+      if (visitedLandmarks.length > 1) {
+        journeyContext = ` — visited ${visitedLandmarks.join(' → ')}`;
+      }
+    }
+
+    return {
+      description: `${objectType} left ${location}${destination}${timeContext}${journeyContext}`,
+      involvedLandmarks: landmarks,
+      confidence: 0.8,
+      usedLlm: false,
+    };
+  }
+
   /** Generate rich movement description using LLM */
   async generateMovementDescription(
     tracked: TrackedObject,
@@ -415,28 +629,92 @@ export class SpatialReasoningEngine {
     const objectType = this.capitalizeFirst(tracked.className);
     const transitSecs = Math.round(transitTime / 1000);
 
-    // Build origin description
-    let origin = fromCamera.name;
-    if (fromLandmarks.length > 0) {
-      const nearLandmark = fromLandmarks[0];
-      origin = `near ${nearLandmark.name}`;
-    } else if (fromCamera.context?.coverageDescription) {
-      origin = fromCamera.context.coverageDescription.split('.')[0];
-    }
+    // Get connection for path context
+    const connection = this.topology ? findConnection(this.topology, fromCamera.deviceId, toCamera.deviceId) : null;
+
+    // Build origin description using landmarks, camera context, or camera name
+    let origin = this.describeLocation(fromCamera, fromLandmarks, 'from');
 
     // Build destination description
-    let destination = toCamera.name;
-    if (toLandmarks.length > 0) {
-      const nearLandmark = toLandmarks[0];
-      destination = `towards ${nearLandmark.name}`;
-    } else if (toCamera.context?.coverageDescription) {
-      destination = `towards ${toCamera.context.coverageDescription.split('.')[0]}`;
+    let destination = this.describeLocation(toCamera, toLandmarks, 'to');
+
+    // Check if we have a named path/connection
+    let pathContext = '';
+    if (connection?.name) {
+      pathContext = ` via ${connection.name}`;
+    } else if (connection?.pathLandmarks?.length && this.topology) {
+      const pathNames = connection.pathLandmarks
+        .map(id => findLandmark(this.topology!, id)?.name)
+        .filter(Boolean);
+      if (pathNames.length > 0) {
+        pathContext = ` past ${pathNames.join(' and ')}`;
+      }
     }
 
-    // Build transit string
-    const transitStr = transitSecs > 0 ? ` (${transitSecs}s)` : '';
+    // Include journey context if this is not the first camera
+    let journeyContext = '';
+    if (tracked.journey.length > 0) {
+      const totalTime = Math.round((Date.now() - tracked.firstSeen) / 1000);
+      if (totalTime > 60) {
+        journeyContext = ` (${Math.round(totalTime / 60)}m on property)`;
+      }
+    }
+
+    // Determine movement verb based on transit time and object type
+    const verb = this.getMovementVerb(tracked.className, transitSecs);
+
+    return `${objectType} ${verb} ${origin} heading ${destination}${pathContext}${journeyContext}`;
+  }
+
+  /** Describe a location using landmarks, camera context, or camera name */
+  private describeLocation(camera: CameraNode, landmarks: Landmark[], direction: 'from' | 'to'): string {
+    // Priority 1: Use entry/exit landmarks
+    const entryExitLandmark = landmarks.find(l =>
+      (direction === 'from' && l.isExitPoint) || (direction === 'to' && l.isEntryPoint)
+    );
+    if (entryExitLandmark) {
+      return direction === 'from' ? `the ${entryExitLandmark.name}` : `the ${entryExitLandmark.name}`;
+    }
+
+    // Priority 2: Use access landmarks (driveway, walkway, etc.)
+    const accessLandmark = landmarks.find(l => l.type === 'access');
+    if (accessLandmark) {
+      return `the ${accessLandmark.name}`;
+    }
+
+    // Priority 3: Use zone landmarks (front yard, back yard)
+    const zoneLandmark = landmarks.find(l => l.type === 'zone');
+    if (zoneLandmark) {
+      return `the ${zoneLandmark.name}`;
+    }
+
+    // Priority 4: Use any landmark
+    if (landmarks.length > 0) {
+      return `near ${landmarks[0].name}`;
+    }
+
+    // Priority 5: Use camera coverage description
+    if (camera.context?.coverageDescription) {
+      const desc = camera.context.coverageDescription.split('.')[0].toLowerCase();
+      return `the ${desc}`;
+    }
+
+    // Fallback: Generic description (no camera name inference - use topology for context)
+    return direction === 'from' ? 'property' : 'property';
+  }
 
-    return `${objectType} moving from ${origin} ${destination}${transitStr}`;
+  /** Get appropriate movement verb based on context */
+  private getMovementVerb(className: string, transitSecs: number): string {
+    if (className === 'car' || className === 'vehicle' || className === 'truck') {
+      return transitSecs < 10 ? 'driving from' : 'moved from';
+    }
+    if (transitSecs < 5) {
+      return 'walking from';
+    }
+    if (transitSecs < 30) {
+      return 'moved from';
+    }
+    return 'traveled from';
   }
 
   /** Build path description from connection */
@@ -458,7 +736,7 @@ export class SpatialReasoningEngine {
     return connection.name || undefined;
   }
 
-  /** Get LLM-enhanced description */
+  /** Get LLM-enhanced description using ChatCompletion interface with vision support */
   private async getLlmEnhancedDescription(
     tracked: TrackedObject,
     fromCamera: CameraNode,
@@ -469,9 +747,12 @@ export class SpatialReasoningEngine {
     mediaObject: MediaObject
   ): Promise<string | null> {
     const llm = await this.findLlmDevice();
-    if (!llm) return null;
+    if (!llm || !llm.getChatCompletion) return null;
 
     try {
+      // Convert image to base64 for vision LLM
+      const imageBase64 = await mediaObjectToBase64(mediaObject);
+
       // Retrieve relevant context for RAG
       const relevantChunks = this.retrieveRelevantContext(
         fromCamera.deviceId,
@@ -492,14 +773,35 @@ export class SpatialReasoningEngine {
         ragContext
       );
 
-      // Call LLM
-      const result = await llm.detectObjects(mediaObject, {
-        settings: { prompt }
-      } as any);
+      // Build message content - use multimodal format if we have an image
+      let messageContent: any;
+      if (imageBase64) {
+        // Vision-capable multimodal message format (OpenAI compatible)
+        messageContent = [
+          { type: 'text', text: prompt },
+          { type: 'image_url', image_url: { url: imageBase64 } },
+        ];
+      } else {
+        // Fallback to text-only if image conversion failed
+        messageContent = prompt;
+      }
+
+      // Call LLM using ChatCompletion interface
+      const result = await llm.getChatCompletion({
+        messages: [
+          {
+            role: 'user',
+            content: messageContent,
+          },
+        ],
+        max_tokens: 150,
+        temperature: 0.7,
+      });
 
-      // Extract description from result
-      if (result.detections?.[0]?.label) {
-        return result.detections[0].label;
+      // Extract description from ChatCompletion result
+      const content = result?.choices?.[0]?.message?.content;
+      if (content && typeof content === 'string') {
+        return content.trim();
       }
 
       return null;
@@ -547,7 +849,7 @@ Examples of good descriptions:
 Generate ONLY the description, nothing else:`;
   }
 
-  /** Suggest a new landmark based on AI analysis */
+  /** Suggest a new landmark based on AI analysis using ChatCompletion with vision */
   async suggestLandmark(
     cameraId: string,
     mediaObject: MediaObject,
@@ -557,9 +859,12 @@ Generate ONLY the description, nothing else:`;
     if (!this.config.enableLandmarkLearning) return null;
 
     const llm = await this.findLlmDevice();
-    if (!llm) return null;
+    if (!llm || !llm.getChatCompletion) return null;
 
     try {
+      // Convert image to base64 for vision LLM
+      const imageBase64 = await mediaObjectToBase64(mediaObject);
+
       const prompt = `Analyze this security camera image. A ${objectClass} was detected.
 
 Looking at the surroundings and environment, identify any notable landmarks or features visible that could help describe this location. Consider:
@@ -573,13 +878,35 @@ If you can identify a clear landmark feature, respond with ONLY a JSON object:
 
 If no clear landmark is identifiable, respond with: {"name": null}`;
 
-      const result = await llm.detectObjects(mediaObject, {
-        settings: { prompt }
-      } as any);
+      // Build message content - use multimodal format if we have an image
+      let messageContent: any;
+      if (imageBase64) {
+        // Vision-capable multimodal message format (OpenAI compatible)
+        messageContent = [
+          { type: 'text', text: prompt },
+          { type: 'image_url', image_url: { url: imageBase64 } },
+        ];
+      } else {
+        // Fallback to text-only if image conversion failed
+        messageContent = prompt;
+      }
+
+      // Call LLM using ChatCompletion interface
+      const result = await llm.getChatCompletion({
+        messages: [
+          {
+            role: 'user',
+            content: messageContent,
+          },
+        ],
+        max_tokens: 100,
+        temperature: 0.3,
+      });
 
-      if (result.detections?.[0]?.label) {
+      const content = result?.choices?.[0]?.message?.content;
+      if (content && typeof content === 'string') {
         try {
-          const parsed = JSON.parse(result.detections[0].label);
+          const parsed = JSON.parse(content.trim());
           if (parsed.name && parsed.type) {
             const suggestionId = `suggest_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;