@blueharford/scrypted-spatial-awareness 0.5.3 → 0.5.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.5.3",
+  "version": "0.5.5",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -87,12 +87,25 @@ export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<Ima
     // Convert MediaObject to Buffer using mediaManager
     const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, ScryptedMimeTypes.Image);
 
+    if (!buffer || buffer.length === 0) {
+      console.warn('Failed to convert MediaObject: empty buffer');
+      return null;
+    }
+
     // Convert buffer to base64 (raw, no data URL prefix)
     const base64 = buffer.toString('base64');
 
+    // Validate base64 - check it's not empty and looks valid
+    if (!base64 || base64.length < 100) {
+      console.warn(`Invalid base64: length=${base64?.length || 0}`);
+      return null;
+    }
+
     // Determine MIME type - default to JPEG for camera images
     const mediaType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
 
+    console.log(`[Image] Converted to base64: ${base64.length} chars, type=${mediaType}`);
+
     return { base64, mediaType };
   } catch (e) {
     console.warn('Failed to convert MediaObject to base64:', e);
@@ -101,13 +114,13 @@ export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<Ima
 }
 
 /** LLM Provider type for image format selection */
-export type LlmProvider = 'openai' | 'anthropic' | 'unknown';
+export type LlmProvider = 'openai' | 'anthropic' | 'scrypted' | 'unknown';
 
 /**
  * Build image content block for ChatCompletion API
- * Supports both OpenAI and Anthropic formats
+ * Supports OpenAI, Anthropic, and @scrypted/llm formats
  * @param imageData - Image data with base64 and media type
- * @param provider - The LLM provider type (openai, anthropic, or unknown)
+ * @param provider - The LLM provider type
  */
 export function buildImageContent(imageData: ImageData, provider: LlmProvider = 'unknown'): any {
   if (provider === 'openai') {
@@ -116,10 +129,11 @@ export function buildImageContent(imageData: ImageData, provider: LlmProvider =
       type: 'image_url',
       image_url: {
         url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+        detail: 'auto',
       },
     };
   } else if (provider === 'anthropic') {
-    // Anthropic format: uses separate base64 data and media_type
+    // Anthropic official format: uses 'data' key
     return {
       type: 'image',
       source: {
@@ -128,18 +142,46 @@ export function buildImageContent(imageData: ImageData, provider: LlmProvider =
         data: imageData.base64,
       },
     };
+  } else if (provider === 'scrypted') {
+    // @scrypted/llm format: uses 'base64' key (per error path .image.source.base64)
+    return {
+      type: 'image',
+      source: {
+        type: 'base64',
+        media_type: imageData.mediaType,
+        base64: imageData.base64,
+      },
+    };
   } else {
-    // Unknown provider: try OpenAI format as it's more commonly supported
-    // Most LLM wrappers (including @scrypted/llm) understand the OpenAI format
+    // Unknown provider: try @scrypted/llm format first
     return {
-      type: 'image_url',
-      image_url: {
-        url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+      type: 'image',
+      source: {
+        type: 'base64',
+        media_type: imageData.mediaType,
+        base64: imageData.base64,
       },
     };
   }
 }
 
+/** Check if an error indicates vision/multimodal content format issue (should try alternate format) */
+export function isVisionFormatError(error: any): boolean {
+  const errorStr = String(error);
+  return (
+    errorStr.includes('content.str') ||
+    errorStr.includes('should be a valid string') ||
+    errorStr.includes('Invalid content type') ||
+    errorStr.includes('does not support vision') ||
+    errorStr.includes('invalid base64') ||
+    errorStr.includes('Invalid base64') ||
+    errorStr.includes('.image.source') ||
+    errorStr.includes('.image_url') ||
+    (errorStr.includes('image_url') && errorStr.includes('not supported')) ||
+    (errorStr.includes('400') && errorStr.includes('content'))
+  );
+}
+
 export class SpatialReasoningEngine {
   private config: SpatialReasoningConfig;
   private console: Console;

package/src/core/topology-discovery.ts

@@ -30,7 +30,7 @@ import {
   Landmark,
   findCamera,
 } from '../models/topology';
-import { mediaObjectToBase64, buildImageContent, ImageData, LlmProvider } from './spatial-reasoning';
+import { mediaObjectToBase64, buildImageContent, ImageData, LlmProvider, isVisionFormatError } from './spatial-reasoning';
 
 const { systemManager } = sdk;
 
@@ -253,77 +253,131 @@ export class TopologyDiscoveryEngine {
       return analysis;
     }
 
-    try {
-      // Build multimodal message with provider-specific image format
-      const result = await llm.getChatCompletion({
-        messages: [
-          {
-            role: 'user',
-            content: [
-              { type: 'text', text: SCENE_ANALYSIS_PROMPT },
-              buildImageContent(imageData, this.llmProviderType),
-            ],
-          },
-        ],
-        max_tokens: 500,
-        temperature: 0.3,
-      });
-
-      const content = result?.choices?.[0]?.message?.content;
-      if (content && typeof content === 'string') {
-        try {
-          // Extract JSON from response (handle markdown code blocks)
-          let jsonStr = content.trim();
-          if (jsonStr.startsWith('```')) {
-            jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
-          }
-
-          const parsed = JSON.parse(jsonStr);
-
-          // Map parsed data to our types
-          if (Array.isArray(parsed.landmarks)) {
-            analysis.landmarks = parsed.landmarks.map((l: any) => ({
-              name: l.name || 'Unknown',
-              type: this.mapLandmarkType(l.type),
-              confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
-              description: l.description || '',
-              boundingBox: l.boundingBox,
-            }));
-          }
+    // Try with detected provider format first, then fallback to alternates
+    // The order matters: try the most likely formats first
+    const formatsToTry: LlmProvider[] = [];
+
+    // Start with detected format
+    formatsToTry.push(this.llmProviderType);
+
+    // Add fallbacks based on detected provider
+    if (this.llmProviderType === 'openai') {
+      formatsToTry.push('scrypted', 'anthropic');
+    } else if (this.llmProviderType === 'anthropic') {
+      formatsToTry.push('scrypted', 'openai');
+    } else if (this.llmProviderType === 'scrypted') {
+      formatsToTry.push('anthropic', 'openai');
+    } else {
+      // Unknown - try all formats
+      formatsToTry.push('scrypted', 'anthropic', 'openai');
+    }
 
-          if (Array.isArray(parsed.zones)) {
-            analysis.zones = parsed.zones.map((z: any) => ({
-              name: z.name || 'Unknown',
-              type: this.mapZoneType(z.type),
-              coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
-              description: z.description || '',
-              boundingBox: z.boundingBox,
-            }));
+    let lastError: any = null;
+
+    for (const formatType of formatsToTry) {
+      try {
+        this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
+
+        // Build multimodal message with provider-specific image format
+        const result = await llm.getChatCompletion({
+          messages: [
+            {
+              role: 'user',
+              content: [
+                { type: 'text', text: SCENE_ANALYSIS_PROMPT },
+                buildImageContent(imageData, formatType),
+              ],
+            },
+          ],
+          max_tokens: 500,
+          temperature: 0.3,
+        });
+
+        const content = result?.choices?.[0]?.message?.content;
+        if (content && typeof content === 'string') {
+          try {
+            // Extract JSON from response (handle markdown code blocks)
+            let jsonStr = content.trim();
+            if (jsonStr.startsWith('```')) {
+              jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
+            }
+
+            const parsed = JSON.parse(jsonStr);
+
+            // Map parsed data to our types
+            if (Array.isArray(parsed.landmarks)) {
+              analysis.landmarks = parsed.landmarks.map((l: any) => ({
+                name: l.name || 'Unknown',
+                type: this.mapLandmarkType(l.type),
+                confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
+                description: l.description || '',
+                boundingBox: l.boundingBox,
+              }));
+            }
+
+            if (Array.isArray(parsed.zones)) {
+              analysis.zones = parsed.zones.map((z: any) => ({
+                name: z.name || 'Unknown',
+                type: this.mapZoneType(z.type),
+                coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
+                description: z.description || '',
+                boundingBox: z.boundingBox,
+              }));
+            }
+
+            if (parsed.edges && typeof parsed.edges === 'object') {
+              analysis.edges = {
+                top: parsed.edges.top || '',
+                left: parsed.edges.left || '',
+                right: parsed.edges.right || '',
+                bottom: parsed.edges.bottom || '',
+              };
+            }
+
+            if (parsed.orientation) {
+              analysis.orientation = this.mapOrientation(parsed.orientation);
+            }
+
+            analysis.isValid = true;
+            this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
+
+            // Update the preferred format for future requests
+            if (formatType !== this.llmProviderType) {
+              this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
+              this.llmProviderType = formatType;
+            }
+
+            // Success - exit the retry loop
+            return analysis;
+          } catch (parseError) {
+            this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
+            analysis.error = 'Failed to parse LLM response';
+            return analysis;
           }
+        }
+      } catch (e) {
+        lastError = e;
 
-          if (parsed.edges && typeof parsed.edges === 'object') {
-            analysis.edges = {
-              top: parsed.edges.top || '',
-              left: parsed.edges.left || '',
-              right: parsed.edges.right || '',
-              bottom: parsed.edges.bottom || '',
-            };
-          }
+        // Check if this is a vision/multimodal format error
+        if (isVisionFormatError(e)) {
+          this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
+          continue; // Try next format
+        }
 
-          if (parsed.orientation) {
-            analysis.orientation = this.mapOrientation(parsed.orientation);
-          }
+        // Not a format error - don't retry
+        this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
+        break;
+      }
+    }
 
-          analysis.isValid = true;
-          this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones`);
-        } catch (parseError) {
-          this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
-          analysis.error = 'Failed to parse LLM response';
-        }
+    // All formats failed
+    if (lastError) {
+      const errorStr = String(lastError);
+      if (isVisionFormatError(lastError)) {
+        analysis.error = 'Vision/image analysis failed with all formats. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured and the @scrypted/llm plugin supports vision.';
+      } else {
+        analysis.error = `Analysis failed: ${errorStr}`;
       }
-    } catch (e) {
-      this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
-      analysis.error = `Analysis failed: ${e}`;
     }
 
     // Cache the analysis