@blueharford/scrypted-spatial-awareness 0.5.4 → 0.5.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.5.4",
+  "version": "0.5.6",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -84,35 +84,58 @@ export interface ImageData {
  */
 export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<ImageData | null> {
   try {
+    console.log(`[Image] Converting MediaObject, mimeType=${mediaObject?.mimeType}`);
+
     // Convert MediaObject to Buffer using mediaManager
     const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.Image);
 
+    if (!buffer) {
+      console.warn('[Image] convertMediaObjectToBuffer returned null/undefined');
+      return null;
+    }
+
+    console.log(`[Image] Buffer received: ${buffer.length} bytes`);
+
+    if (buffer.length === 0) {
+      console.warn('[Image] Buffer is empty (0 bytes)');
+      return null;
+    }
+
+    // Check if buffer is too small to be a valid image (< 1KB is suspicious)
+    if (buffer.length < 1000) {
+      // Log what the buffer contains - might be an error message
+      const bufferContent = buffer.toString('utf8').substring(0, 100);
+      console.warn(`[Image] Buffer too small (${buffer.length} bytes), content: ${bufferContent}`);
+      return null;
+    }
+
     // Convert buffer to base64 (raw, no data URL prefix)
     const base64 = buffer.toString('base64');
 
     // Determine MIME type - default to JPEG for camera images
     const mediaType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
 
+    console.log(`[Image] Converted to base64: ${base64.length} chars, type=${mediaType}`);
+
     return { base64, mediaType };
   } catch (e) {
-    console.warn('Failed to convert MediaObject to base64:', e);
+    console.warn('[Image] Failed to convert MediaObject to base64:', e);
     return null;
   }
 }
 
 /** LLM Provider type for image format selection */
-export type LlmProvider = 'openai' | 'anthropic' | 'unknown';
+export type LlmProvider = 'openai' | 'anthropic' | 'scrypted' | 'unknown';
 
 /**
  * Build image content block for ChatCompletion API
- * Supports both OpenAI and Anthropic formats
+ * Supports OpenAI, Anthropic, and @scrypted/llm formats
  * @param imageData - Image data with base64 and media type
- * @param provider - The LLM provider type (openai, anthropic, or unknown)
+ * @param provider - The LLM provider type
  */
 export function buildImageContent(imageData: ImageData, provider: LlmProvider = 'unknown'): any {
   if (provider === 'openai') {
     // OpenAI format: uses data URL with image_url wrapper
-    // Include detail parameter for compatibility
     return {
       type: 'image_url',
       image_url: {
@@ -121,7 +144,7 @@ export function buildImageContent(imageData: ImageData, provider: LlmProvider =
       },
     };
   } else if (provider === 'anthropic') {
-    // Anthropic format: uses separate base64 data and media_type
+    // Anthropic official format: uses 'data' key
     return {
       type: 'image',
       source: {
@@ -130,29 +153,43 @@ export function buildImageContent(imageData: ImageData, provider: LlmProvider =
         data: imageData.base64,
       },
     };
+  } else if (provider === 'scrypted') {
+    // @scrypted/llm format: uses 'base64' key (per error path .image.source.base64)
+    return {
+      type: 'image',
+      source: {
+        type: 'base64',
+        media_type: imageData.mediaType,
+        base64: imageData.base64,
+      },
+    };
   } else {
-    // Unknown provider: try Anthropic format first as it's more explicit
-    // Some plugins may translate this to OpenAI format internally
+    // Unknown provider: try @scrypted/llm format first
     return {
       type: 'image',
       source: {
         type: 'base64',
         media_type: imageData.mediaType,
-        data: imageData.base64,
+        base64: imageData.base64,
       },
     };
   }
 }
 
-/** Check if an error indicates vision/multimodal content is not supported */
-export function isVisionNotSupportedError(error: any): boolean {
+/** Check if an error indicates vision/multimodal content format issue (should try alternate format) */
+export function isVisionFormatError(error: any): boolean {
   const errorStr = String(error);
   return (
     errorStr.includes('content.str') ||
     errorStr.includes('should be a valid string') ||
     errorStr.includes('Invalid content type') ||
     errorStr.includes('does not support vision') ||
-    errorStr.includes('image_url') && errorStr.includes('not supported')
+    errorStr.includes('invalid base64') ||
+    errorStr.includes('Invalid base64') ||
+    errorStr.includes('.image.source') ||
+    errorStr.includes('.image_url') ||
+    (errorStr.includes('image_url') && errorStr.includes('not supported')) ||
+    (errorStr.includes('400') && errorStr.includes('content'))
   );
 }
 

package/src/core/topology-discovery.ts

@@ -30,7 +30,7 @@ import {
   Landmark,
   findCamera,
 } from '../models/topology';
-import { mediaObjectToBase64, buildImageContent, ImageData, LlmProvider, isVisionNotSupportedError } from './spatial-reasoning';
+import { mediaObjectToBase64, buildImageContent, ImageData, LlmProvider, isVisionFormatError } from './spatial-reasoning';
 
 const { systemManager } = sdk;
 
@@ -213,11 +213,27 @@ export class TopologyDiscoveryEngine {
     try {
       const camera = systemManager.getDeviceById<Camera>(cameraId);
       if (!camera?.interfaces?.includes(ScryptedInterface.Camera)) {
+        this.console.warn(`[Discovery] Camera ${cameraId} doesn't have Camera interface`);
         return null;
       }
 
+      this.console.log(`[Discovery] Taking snapshot from camera: ${camera.name || cameraId}`);
       const mediaObject = await camera.takePicture();
-      return mediaObjectToBase64(mediaObject);
+
+      if (!mediaObject) {
+        this.console.warn(`[Discovery] takePicture() returned null for ${camera.name}`);
+        return null;
+      }
+
+      this.console.log(`[Discovery] MediaObject received: mimeType=${mediaObject.mimeType}`);
+
+      const imageData = await mediaObjectToBase64(mediaObject);
+
+      if (!imageData) {
+        this.console.warn(`[Discovery] Failed to convert MediaObject to base64 for ${camera.name}`);
+      }
+
+      return imageData;
     } catch (e) {
       this.console.warn(`[Discovery] Failed to get snapshot from camera ${cameraId}:`, e);
       return null;
@@ -253,17 +269,23 @@ export class TopologyDiscoveryEngine {
       return analysis;
     }
 
-    // Try with detected provider format first, then fallback to alternate format
-    const formatsToTry: LlmProvider[] = [this.llmProviderType];
+    // Try with detected provider format first, then fallback to alternates
+    // The order matters: try the most likely formats first
+    const formatsToTry: LlmProvider[] = [];
+
+    // Start with detected format
+    formatsToTry.push(this.llmProviderType);
 
-    // Add fallback format
+    // Add fallbacks based on detected provider
     if (this.llmProviderType === 'openai') {
-      formatsToTry.push('anthropic');
+      formatsToTry.push('scrypted', 'anthropic');
     } else if (this.llmProviderType === 'anthropic') {
-      formatsToTry.push('openai');
+      formatsToTry.push('scrypted', 'openai');
+    } else if (this.llmProviderType === 'scrypted') {
+      formatsToTry.push('anthropic', 'openai');
     } else {
-      // Unknown - try both
-      formatsToTry.push('openai');
+      // Unknown - try all formats
+      formatsToTry.push('scrypted', 'anthropic', 'openai');
     }
 
     let lastError: any = null;
@@ -353,8 +375,8 @@ export class TopologyDiscoveryEngine {
         lastError = e;
 
         // Check if this is a vision/multimodal format error
-        if (isVisionNotSupportedError(e)) {
-          this.console.warn(`[Discovery] ${formatType} format not supported, trying fallback...`);
+        if (isVisionFormatError(e)) {
+          this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
           continue; // Try next format
         }
 
@@ -367,8 +389,8 @@ export class TopologyDiscoveryEngine {
     // All formats failed
     if (lastError) {
       const errorStr = String(lastError);
-      if (isVisionNotSupportedError(lastError)) {
-        analysis.error = 'Vision/image analysis not supported by configured LLM. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured.';
+      if (isVisionFormatError(lastError)) {
+        analysis.error = 'Vision/image analysis failed with all formats. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured and the @scrypted/llm plugin supports vision.';
       } else {
         analysis.error = `Analysis failed: ${errorStr}`;
       }