@blueharford/scrypted-spatial-awareness 0.5.4 → 0.5.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.5.4",
+  "version": "0.5.5",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -87,12 +87,25 @@ export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<Ima
     // Convert MediaObject to Buffer using mediaManager
     const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.Image);
 
+    if (!buffer || buffer.length === 0) {
+      console.warn('Failed to convert MediaObject: empty buffer');
+      return null;
+    }
+
     // Convert buffer to base64 (raw, no data URL prefix)
     const base64 = buffer.toString('base64');
 
+    // Validate base64 - check it's not empty and looks valid
+    if (!base64 || base64.length < 100) {
+      console.warn(`Invalid base64: length=${base64?.length || 0}`);
+      return null;
+    }
+
     // Determine MIME type - default to JPEG for camera images
     const mediaType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
 
+    console.log(`[Image] Converted to base64: ${base64.length} chars, type=${mediaType}`);
+
     return { base64, mediaType };
   } catch (e) {
     console.warn('Failed to convert MediaObject to base64:', e);
@@ -101,18 +114,17 @@ export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<Ima
 }
 
 /** LLM Provider type for image format selection */
-export type LlmProvider = 'openai' | 'anthropic' | 'unknown';
+export type LlmProvider = 'openai' | 'anthropic' | 'scrypted' | 'unknown';
 
 /**
  * Build image content block for ChatCompletion API
- * Supports both OpenAI and Anthropic formats
+ * Supports OpenAI, Anthropic, and @scrypted/llm formats
  * @param imageData - Image data with base64 and media type
- * @param provider - The LLM provider type (openai, anthropic, or unknown)
+ * @param provider - The LLM provider type
  */
 export function buildImageContent(imageData: ImageData, provider: LlmProvider = 'unknown'): any {
   if (provider === 'openai') {
     // OpenAI format: uses data URL with image_url wrapper
-    // Include detail parameter for compatibility
     return {
       type: 'image_url',
       image_url: {
@@ -121,7 +133,7 @@ export function buildImageContent(imageData: ImageData, provider: LlmProvider =
       },
     };
   } else if (provider === 'anthropic') {
-    // Anthropic format: uses separate base64 data and media_type
+    // Anthropic official format: uses 'data' key
     return {
       type: 'image',
       source: {
@@ -130,29 +142,43 @@ export function buildImageContent(imageData: ImageData, provider: LlmProvider =
         data: imageData.base64,
       },
     };
+  } else if (provider === 'scrypted') {
+    // @scrypted/llm format: uses 'base64' key (per error path .image.source.base64)
+    return {
+      type: 'image',
+      source: {
+        type: 'base64',
+        media_type: imageData.mediaType,
+        base64: imageData.base64,
+      },
+    };
   } else {
-    // Unknown provider: try Anthropic format first as it's more explicit
-    // Some plugins may translate this to OpenAI format internally
+    // Unknown provider: try @scrypted/llm format first
     return {
       type: 'image',
       source: {
         type: 'base64',
         media_type: imageData.mediaType,
-        data: imageData.base64,
+        base64: imageData.base64,
       },
     };
   }
 }
 
-/** Check if an error indicates vision/multimodal content is not supported */
-export function isVisionNotSupportedError(error: any): boolean {
+/** Check if an error indicates vision/multimodal content format issue (should try alternate format) */
+export function isVisionFormatError(error: any): boolean {
   const errorStr = String(error);
   return (
     errorStr.includes('content.str') ||
     errorStr.includes('should be a valid string') ||
     errorStr.includes('Invalid content type') ||
     errorStr.includes('does not support vision') ||
-    errorStr.includes('image_url') && errorStr.includes('not supported')
+    errorStr.includes('invalid base64') ||
+    errorStr.includes('Invalid base64') ||
+    errorStr.includes('.image.source') ||
+    errorStr.includes('.image_url') ||
+    (errorStr.includes('image_url') && errorStr.includes('not supported')) ||
+    (errorStr.includes('400') && errorStr.includes('content'))
   );
 }
 

package/src/core/topology-discovery.ts

@@ -30,7 +30,7 @@ import {
   Landmark,
   findCamera,
 } from '../models/topology';
-import { mediaObjectToBase64, buildImageContent, ImageData, LlmProvider, isVisionNotSupportedError } from './spatial-reasoning';
+import { mediaObjectToBase64, buildImageContent, ImageData, LlmProvider, isVisionFormatError } from './spatial-reasoning';
 
 const { systemManager } = sdk;
 
@@ -253,17 +253,23 @@ export class TopologyDiscoveryEngine {
       return analysis;
     }
 
-    // Try with detected provider format first, then fallback to alternate format
-    const formatsToTry: LlmProvider[] = [this.llmProviderType];
+    // Try with detected provider format first, then fallback to alternates
+    // The order matters: try the most likely formats first
+    const formatsToTry: LlmProvider[] = [];
 
-    // Add fallback format
+    // Start with detected format
+    formatsToTry.push(this.llmProviderType);
+
+    // Add fallbacks based on detected provider
     if (this.llmProviderType === 'openai') {
-      formatsToTry.push('anthropic');
+      formatsToTry.push('scrypted', 'anthropic');
     } else if (this.llmProviderType === 'anthropic') {
-      formatsToTry.push('openai');
+      formatsToTry.push('scrypted', 'openai');
+    } else if (this.llmProviderType === 'scrypted') {
+      formatsToTry.push('anthropic', 'openai');
     } else {
-      // Unknown - try both
-      formatsToTry.push('openai');
+      // Unknown - try all formats
+      formatsToTry.push('scrypted', 'anthropic', 'openai');
     }
 
     let lastError: any = null;
@@ -353,8 +359,8 @@ export class TopologyDiscoveryEngine {
         lastError = e;
 
         // Check if this is a vision/multimodal format error
-        if (isVisionNotSupportedError(e)) {
-          this.console.warn(`[Discovery] ${formatType} format not supported, trying fallback...`);
+        if (isVisionFormatError(e)) {
+          this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
           continue; // Try next format
         }
 
@@ -367,8 +373,8 @@ export class TopologyDiscoveryEngine {
     // All formats failed
     if (lastError) {
       const errorStr = String(lastError);
-      if (isVisionNotSupportedError(lastError)) {
-        analysis.error = 'Vision/image analysis not supported by configured LLM. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured.';
+      if (isVisionFormatError(lastError)) {
+        analysis.error = 'Vision/image analysis failed with all formats. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured and the @scrypted/llm plugin supports vision.';
       } else {
         analysis.error = `Analysis failed: ${errorStr}`;
       }