@blueharford/scrypted-spatial-awareness 0.5.1 → 0.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -69,29 +69,54 @@ interface ChatCompletionDevice extends ScryptedDevice {
   streamChatCompletion?(params: any): AsyncGenerator<any>;
 }
 
+/** Image data for LLM vision APIs */
+export interface ImageData {
+  /** Raw base64 encoded image data (no data URL prefix) */
+  base64: string;
+  /** MIME type (e.g., 'image/jpeg') */
+  mediaType: string;
+}
+
 /**
- * Convert a MediaObject to a base64 data URL for vision LLM consumption
+ * Convert a MediaObject to base64 image data for vision LLM consumption
  * @param mediaObject - MediaObject from camera.takePicture()
- * @returns Base64 data URL (data:image/jpeg;base64,...) or null if conversion fails
+ * @returns ImageData with raw base64 and media type, or null if conversion fails
  */
-export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<string | null> {
+export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<ImageData | null> {
   try {
     // Convert MediaObject to Buffer using mediaManager
     const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.Image);
 
-    // Convert buffer to base64
+    // Convert buffer to base64 (raw, no data URL prefix)
     const base64 = buffer.toString('base64');
 
     // Determine MIME type - default to JPEG for camera images
-    const mimeType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
+    const mediaType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
 
-    return `data:${mimeType};base64,${base64}`;
+    return { base64, mediaType };
   } catch (e) {
     console.warn('Failed to convert MediaObject to base64:', e);
     return null;
   }
 }
 
+/**
+ * Build image content block for ChatCompletion API
+ * Compatible with both OpenAI and Anthropic formats via @scrypted/llm
+ */
+export function buildImageContent(imageData: ImageData): any {
+  // Use Anthropic's native format which @scrypted/llm should translate
+  // This format is more explicit about the base64 data
+  return {
+    type: 'image',
+    source: {
+      type: 'base64',
+      media_type: imageData.mediaType,
+      data: imageData.base64,
+    },
+  };
+}
+
 export class SpatialReasoningEngine {
   private config: SpatialReasoningConfig;
   private console: Console;
@@ -751,7 +776,7 @@ export class SpatialReasoningEngine {
 
     try {
       // Convert image to base64 for vision LLM
-      const imageBase64 = await mediaObjectToBase64(mediaObject);
+      const imageData = await mediaObjectToBase64(mediaObject);
 
       // Retrieve relevant context for RAG
       const relevantChunks = this.retrieveRelevantContext(
@@ -775,11 +800,11 @@ export class SpatialReasoningEngine {
 
       // Build message content - use multimodal format if we have an image
       let messageContent: any;
-      if (imageBase64) {
-        // Vision-capable multimodal message format (OpenAI compatible)
+      if (imageData) {
+        // Vision-capable multimodal message format (Anthropic native format)
         messageContent = [
           { type: 'text', text: prompt },
-          { type: 'image_url', image_url: { url: imageBase64 } },
+          buildImageContent(imageData),
         ];
       } else {
         // Fallback to text-only if image conversion failed
@@ -863,7 +888,7 @@ Generate ONLY the description, nothing else:`;
 
     try {
       // Convert image to base64 for vision LLM
-      const imageBase64 = await mediaObjectToBase64(mediaObject);
+      const imageData = await mediaObjectToBase64(mediaObject);
 
       const prompt = `Analyze this security camera image. A ${objectClass} was detected.
 
@@ -880,11 +905,11 @@ If no clear landmark is identifiable, respond with: {"name": null}`;
 
       // Build message content - use multimodal format if we have an image
       let messageContent: any;
-      if (imageBase64) {
-        // Vision-capable multimodal message format (OpenAI compatible)
+      if (imageData) {
+        // Vision-capable multimodal message format (Anthropic native format)
         messageContent = [
           { type: 'text', text: prompt },
-          { type: 'image_url', image_url: { url: imageBase64 } },
+          buildImageContent(imageData),
         ];
       } else {
         // Fallback to text-only if image conversion failed

package/src/core/topology-discovery.ts

@@ -30,7 +30,7 @@ import {
   Landmark,
   findCamera,
 } from '../models/topology';
-import { mediaObjectToBase64 } from './spatial-reasoning';
+import { mediaObjectToBase64, buildImageContent, ImageData } from './spatial-reasoning';
 
 const { systemManager } = sdk;
 
@@ -191,8 +191,8 @@ export class TopologyDiscoveryEngine {
     return null;
   }
 
-  /** Get camera snapshot as base64 */
-  private async getCameraSnapshot(cameraId: string): Promise<string | null> {
+  /** Get camera snapshot as ImageData */
+  private async getCameraSnapshot(cameraId: string): Promise<ImageData | null> {
     try {
       const camera = systemManager.getDeviceById<Camera>(cameraId);
       if (!camera?.interfaces?.includes(ScryptedInterface.Camera)) {
@@ -230,21 +230,21 @@ export class TopologyDiscoveryEngine {
       return analysis;
     }
 
-    const imageBase64 = await this.getCameraSnapshot(cameraId);
-    if (!imageBase64) {
+    const imageData = await this.getCameraSnapshot(cameraId);
+    if (!imageData) {
       analysis.error = 'Failed to capture camera snapshot';
       return analysis;
     }
 
     try {
-      // Build multimodal message
+      // Build multimodal message with Anthropic-native format
       const result = await llm.getChatCompletion({
         messages: [
           {
             role: 'user',
             content: [
               { type: 'text', text: SCENE_ANALYSIS_PROMPT },
-              { type: 'image_url', image_url: { url: imageBase64 } },
+              buildImageContent(imageData),
             ],
           },
         ],
@@ -387,6 +387,14 @@ export class TopologyDiscoveryEngine {
       this.status.camerasAnalyzed = analyses.length;
       this.console.log(`[Discovery] Analyzed ${analyses.length} cameras successfully`);
 
+      // Handle case where no cameras were successfully analyzed
+      if (analyses.length === 0) {
+        this.console.warn('[Discovery] No cameras were successfully analyzed');
+        this.status.lastError = 'No cameras were successfully analyzed - check LLM configuration';
+        this.status.lastScanTime = Date.now();
+        return null;
+      }
+
       // Correlate if we have multiple cameras
       let correlation: TopologyCorrelation | null = null;
       if (analyses.length >= 2) {
@@ -394,7 +402,7 @@ export class TopologyDiscoveryEngine {
         if (correlation) {
           this.generateSuggestionsFromCorrelation(correlation);
         }
-      } else {
+      } else if (analyses.length === 1) {
         // Single camera - generate suggestions from its analysis
         this.generateSuggestionsFromAnalysis(analyses[0]);
       }