@blueharford/scrypted-spatial-awareness 0.5.2 → 0.5.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.5.2",
+  "version": "0.5.4",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -100,21 +100,60 @@ export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<Ima
   }
 }
 
+/** LLM Provider type for image format selection */
+export type LlmProvider = 'openai' | 'anthropic' | 'unknown';
+
 /**
  * Build image content block for ChatCompletion API
- * Compatible with both OpenAI and Anthropic formats via @scrypted/llm
+ * Supports both OpenAI and Anthropic formats
+ * @param imageData - Image data with base64 and media type
+ * @param provider - The LLM provider type (openai, anthropic, or unknown)
  */
-export function buildImageContent(imageData: ImageData): any {
-  // Use Anthropic's native format which @scrypted/llm should translate
-  // This format is more explicit about the base64 data
-  return {
-    type: 'image',
-    source: {
-      type: 'base64',
-      media_type: imageData.mediaType,
-      data: imageData.base64,
-    },
-  };
+export function buildImageContent(imageData: ImageData, provider: LlmProvider = 'unknown'): any {
+  if (provider === 'openai') {
+    // OpenAI format: uses data URL with image_url wrapper
+    // Include detail parameter for compatibility
+    return {
+      type: 'image_url',
+      image_url: {
+        url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+        detail: 'auto',
+      },
+    };
+  } else if (provider === 'anthropic') {
+    // Anthropic format: uses separate base64 data and media_type
+    return {
+      type: 'image',
+      source: {
+        type: 'base64',
+        media_type: imageData.mediaType,
+        data: imageData.base64,
+      },
+    };
+  } else {
+    // Unknown provider: try Anthropic format first as it's more explicit
+    // Some plugins may translate this to OpenAI format internally
+    return {
+      type: 'image',
+      source: {
+        type: 'base64',
+        media_type: imageData.mediaType,
+        data: imageData.base64,
+      },
+    };
+  }
+}
+
+/** Check if an error indicates vision/multimodal content is not supported */
+export function isVisionNotSupportedError(error: any): boolean {
+  const errorStr = String(error);
+  return (
+    errorStr.includes('content.str') ||
+    errorStr.includes('should be a valid string') ||
+    errorStr.includes('Invalid content type') ||
+    errorStr.includes('does not support vision') ||
+    errorStr.includes('image_url') && errorStr.includes('not supported')
+  );
 }
 
 export class SpatialReasoningEngine {
@@ -361,6 +400,7 @@ export class SpatialReasoningEngine {
 
   private llmSearched: boolean = false;
   private llmProvider: string | null = null;
+  private llmProviderType: LlmProvider = 'unknown';
 
   /** Find or initialize LLM device - looks for ChatCompletion interface from @scrypted/llm plugin */
   private async findLlmDevice(): Promise<ChatCompletionDevice | null> {
@@ -381,27 +421,36 @@ export class SpatialReasoningEngine {
           const deviceName = device.name?.toLowerCase() || '';
           const pluginId = (device as any).pluginId?.toLowerCase() || '';
 
-          // Identify the provider type for logging
+          // Identify the provider type for logging and image format selection
           let providerType = 'Unknown';
-          if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
-            providerType = 'Scrypted LLM';
-          }
+          let providerTypeEnum: LlmProvider = 'unknown';
+
           if (deviceName.includes('openai') || deviceName.includes('gpt')) {
             providerType = 'OpenAI';
+            providerTypeEnum = 'openai';
           } else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
             providerType = 'Anthropic';
+            providerTypeEnum = 'anthropic';
           } else if (deviceName.includes('ollama')) {
             providerType = 'Ollama';
+            providerTypeEnum = 'openai'; // Ollama uses OpenAI-compatible format
           } else if (deviceName.includes('gemini') || deviceName.includes('google')) {
             providerType = 'Google';
+            providerTypeEnum = 'openai'; // Google uses OpenAI-compatible format
           } else if (deviceName.includes('llama')) {
             providerType = 'llama.cpp';
+            providerTypeEnum = 'openai'; // llama.cpp uses OpenAI-compatible format
+          } else if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
+            providerType = 'Scrypted LLM';
+            providerTypeEnum = 'unknown';
           }
 
           this.llmDevice = device as unknown as ChatCompletionDevice;
           this.llmProvider = `${providerType} (${device.name})`;
+          this.llmProviderType = providerTypeEnum;
           this.console.log(`[LLM] Connected to ${providerType}: ${device.name}`);
           this.console.log(`[LLM] Plugin: ${pluginId || 'N/A'}`);
+          this.console.log(`[LLM] Image format: ${providerTypeEnum}`);
           this.console.log(`[LLM] Interfaces: ${device.interfaces?.join(', ')}`);
           return this.llmDevice;
         }
@@ -423,6 +472,11 @@ export class SpatialReasoningEngine {
     return this.llmProvider;
   }
 
+  /** Get the current LLM provider type for image format selection */
+  getLlmProviderType(): LlmProvider {
+    return this.llmProviderType;
+  }
+
   /** Check if LLM is available */
   isLlmAvailable(): boolean {
     return this.llmDevice !== null;
@@ -801,10 +855,10 @@ export class SpatialReasoningEngine {
       // Build message content - use multimodal format if we have an image
       let messageContent: any;
       if (imageData) {
-        // Vision-capable multimodal message format (Anthropic native format)
+        // Vision-capable multimodal message format (provider-specific)
         messageContent = [
           { type: 'text', text: prompt },
-          buildImageContent(imageData),
+          buildImageContent(imageData, this.llmProviderType),
         ];
       } else {
         // Fallback to text-only if image conversion failed
@@ -906,10 +960,10 @@ If no clear landmark is identifiable, respond with: {"name": null}`;
       // Build message content - use multimodal format if we have an image
       let messageContent: any;
       if (imageData) {
-        // Vision-capable multimodal message format (Anthropic native format)
+        // Vision-capable multimodal message format (provider-specific)
         messageContent = [
           { type: 'text', text: prompt },
-          buildImageContent(imageData),
+          buildImageContent(imageData, this.llmProviderType),
         ];
       } else {
         // Fallback to text-only if image conversion failed

package/src/core/topology-discovery.ts

@@ -30,7 +30,7 @@ import {
   Landmark,
   findCamera,
 } from '../models/topology';
-import { mediaObjectToBase64, buildImageContent, ImageData } from './spatial-reasoning';
+import { mediaObjectToBase64, buildImageContent, ImageData, LlmProvider, isVisionNotSupportedError } from './spatial-reasoning';
 
 const { systemManager } = sdk;
 
@@ -100,6 +100,7 @@ export class TopologyDiscoveryEngine {
   private topology: CameraTopology | null = null;
   private llmDevice: ChatCompletionDevice | null = null;
   private llmSearched: boolean = false;
+  private llmProviderType: LlmProvider = 'unknown';
 
   // Scene analysis cache (camera ID -> analysis)
   private sceneCache: Map<string, SceneAnalysis> = new Map();
@@ -177,8 +178,24 @@ export class TopologyDiscoveryEngine {
         if (!device) continue;
 
         if (device.interfaces?.includes('ChatCompletion')) {
+          const deviceName = device.name?.toLowerCase() || '';
+
+          // Detect provider type for image format selection
+          if (deviceName.includes('openai') || deviceName.includes('gpt')) {
+            this.llmProviderType = 'openai';
+          } else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
+            this.llmProviderType = 'anthropic';
+          } else if (deviceName.includes('ollama') || deviceName.includes('gemini') ||
+                     deviceName.includes('google') || deviceName.includes('llama')) {
+            // These providers use OpenAI-compatible format
+            this.llmProviderType = 'openai';
+          } else {
+            this.llmProviderType = 'unknown';
+          }
+
           this.llmDevice = device as unknown as ChatCompletionDevice;
           this.console.log(`[Discovery] Connected to LLM: ${device.name}`);
+          this.console.log(`[Discovery] Image format: ${this.llmProviderType}`);
           return this.llmDevice;
         }
       }
@@ -236,77 +253,125 @@ export class TopologyDiscoveryEngine {
       return analysis;
     }
 
-    try {
-      // Build multimodal message with Anthropic-native format
-      const result = await llm.getChatCompletion({
-        messages: [
-          {
-            role: 'user',
-            content: [
-              { type: 'text', text: SCENE_ANALYSIS_PROMPT },
-              buildImageContent(imageData),
-            ],
-          },
-        ],
-        max_tokens: 500,
-        temperature: 0.3,
-      });
-
-      const content = result?.choices?.[0]?.message?.content;
-      if (content && typeof content === 'string') {
-        try {
-          // Extract JSON from response (handle markdown code blocks)
-          let jsonStr = content.trim();
-          if (jsonStr.startsWith('```')) {
-            jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
-          }
-
-          const parsed = JSON.parse(jsonStr);
-
-          // Map parsed data to our types
-          if (Array.isArray(parsed.landmarks)) {
-            analysis.landmarks = parsed.landmarks.map((l: any) => ({
-              name: l.name || 'Unknown',
-              type: this.mapLandmarkType(l.type),
-              confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
-              description: l.description || '',
-              boundingBox: l.boundingBox,
-            }));
-          }
+    // Try with detected provider format first, then fallback to alternate format
+    const formatsToTry: LlmProvider[] = [this.llmProviderType];
+
+    // Add fallback format
+    if (this.llmProviderType === 'openai') {
+      formatsToTry.push('anthropic');
+    } else if (this.llmProviderType === 'anthropic') {
+      formatsToTry.push('openai');
+    } else {
+      // Unknown - try both
+      formatsToTry.push('openai');
+    }
 
-          if (Array.isArray(parsed.zones)) {
-            analysis.zones = parsed.zones.map((z: any) => ({
-              name: z.name || 'Unknown',
-              type: this.mapZoneType(z.type),
-              coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
-              description: z.description || '',
-              boundingBox: z.boundingBox,
-            }));
+    let lastError: any = null;
+
+    for (const formatType of formatsToTry) {
+      try {
+        this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
+
+        // Build multimodal message with provider-specific image format
+        const result = await llm.getChatCompletion({
+          messages: [
+            {
+              role: 'user',
+              content: [
+                { type: 'text', text: SCENE_ANALYSIS_PROMPT },
+                buildImageContent(imageData, formatType),
+              ],
+            },
+          ],
+          max_tokens: 500,
+          temperature: 0.3,
+        });
+
+        const content = result?.choices?.[0]?.message?.content;
+        if (content && typeof content === 'string') {
+          try {
+            // Extract JSON from response (handle markdown code blocks)
+            let jsonStr = content.trim();
+            if (jsonStr.startsWith('```')) {
+              jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
+            }
+
+            const parsed = JSON.parse(jsonStr);
+
+            // Map parsed data to our types
+            if (Array.isArray(parsed.landmarks)) {
+              analysis.landmarks = parsed.landmarks.map((l: any) => ({
+                name: l.name || 'Unknown',
+                type: this.mapLandmarkType(l.type),
+                confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
+                description: l.description || '',
+                boundingBox: l.boundingBox,
+              }));
+            }
+
+            if (Array.isArray(parsed.zones)) {
+              analysis.zones = parsed.zones.map((z: any) => ({
+                name: z.name || 'Unknown',
+                type: this.mapZoneType(z.type),
+                coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
+                description: z.description || '',
+                boundingBox: z.boundingBox,
+              }));
+            }
+
+            if (parsed.edges && typeof parsed.edges === 'object') {
+              analysis.edges = {
+                top: parsed.edges.top || '',
+                left: parsed.edges.left || '',
+                right: parsed.edges.right || '',
+                bottom: parsed.edges.bottom || '',
+              };
+            }
+
+            if (parsed.orientation) {
+              analysis.orientation = this.mapOrientation(parsed.orientation);
+            }
+
+            analysis.isValid = true;
+            this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
+
+            // Update the preferred format for future requests
+            if (formatType !== this.llmProviderType) {
+              this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
+              this.llmProviderType = formatType;
+            }
+
+            // Success - exit the retry loop
+            return analysis;
+          } catch (parseError) {
+            this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
+            analysis.error = 'Failed to parse LLM response';
+            return analysis;
           }
+        }
+      } catch (e) {
+        lastError = e;
 
-          if (parsed.edges && typeof parsed.edges === 'object') {
-            analysis.edges = {
-              top: parsed.edges.top || '',
-              left: parsed.edges.left || '',
-              right: parsed.edges.right || '',
-              bottom: parsed.edges.bottom || '',
-            };
-          }
+        // Check if this is a vision/multimodal format error
+        if (isVisionNotSupportedError(e)) {
+          this.console.warn(`[Discovery] ${formatType} format not supported, trying fallback...`);
+          continue; // Try next format
+        }
 
-          if (parsed.orientation) {
-            analysis.orientation = this.mapOrientation(parsed.orientation);
-          }
+        // Not a format error - don't retry
+        this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
+        break;
+      }
+    }
 
-          analysis.isValid = true;
-          this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones`);
-        } catch (parseError) {
-          this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
-          analysis.error = 'Failed to parse LLM response';
-        }
+    // All formats failed
+    if (lastError) {
+      const errorStr = String(lastError);
+      if (isVisionNotSupportedError(lastError)) {
+        analysis.error = 'Vision/image analysis not supported by configured LLM. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured.';
+      } else {
+        analysis.error = `Analysis failed: ${errorStr}`;
       }
-    } catch (e) {
-      this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
-      analysis.error = `Analysis failed: ${e}`;
     }
 
     // Cache the analysis