@blueharford/scrypted-spatial-awareness 0.5.2 → 0.5.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -100,21 +100,44 @@ export async function mediaObjectToBase64(mediaObject: MediaObject): Promise<Ima
   }
 }
 
+/** LLM Provider type for image format selection */
+export type LlmProvider = 'openai' | 'anthropic' | 'unknown';
+
 /**
  * Build image content block for ChatCompletion API
- * Compatible with both OpenAI and Anthropic formats via @scrypted/llm
+ * Supports both OpenAI and Anthropic formats
+ * @param imageData - Image data with base64 and media type
+ * @param provider - The LLM provider type (openai, anthropic, or unknown)
  */
-export function buildImageContent(imageData: ImageData): any {
-  // Use Anthropic's native format which @scrypted/llm should translate
-  // This format is more explicit about the base64 data
-  return {
-    type: 'image',
-    source: {
-      type: 'base64',
-      media_type: imageData.mediaType,
-      data: imageData.base64,
-    },
-  };
+export function buildImageContent(imageData: ImageData, provider: LlmProvider = 'unknown'): any {
+  if (provider === 'openai') {
+    // OpenAI format: uses data URL with image_url wrapper
+    return {
+      type: 'image_url',
+      image_url: {
+        url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+      },
+    };
+  } else if (provider === 'anthropic') {
+    // Anthropic format: uses separate base64 data and media_type
+    return {
+      type: 'image',
+      source: {
+        type: 'base64',
+        media_type: imageData.mediaType,
+        data: imageData.base64,
+      },
+    };
+  } else {
+    // Unknown provider: try OpenAI format as it's more commonly supported
+    // Most LLM wrappers (including @scrypted/llm) understand the OpenAI format
+    return {
+      type: 'image_url',
+      image_url: {
+        url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+      },
+    };
+  }
 }
 
 export class SpatialReasoningEngine {
@@ -361,6 +384,7 @@ export class SpatialReasoningEngine {
 
   private llmSearched: boolean = false;
   private llmProvider: string | null = null;
+  private llmProviderType: LlmProvider = 'unknown';
 
   /** Find or initialize LLM device - looks for ChatCompletion interface from @scrypted/llm plugin */
   private async findLlmDevice(): Promise<ChatCompletionDevice | null> {
@@ -381,27 +405,36 @@ export class SpatialReasoningEngine {
           const deviceName = device.name?.toLowerCase() || '';
           const pluginId = (device as any).pluginId?.toLowerCase() || '';
 
-          // Identify the provider type for logging
+          // Identify the provider type for logging and image format selection
           let providerType = 'Unknown';
-          if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
-            providerType = 'Scrypted LLM';
-          }
+          let providerTypeEnum: LlmProvider = 'unknown';
+
           if (deviceName.includes('openai') || deviceName.includes('gpt')) {
             providerType = 'OpenAI';
+            providerTypeEnum = 'openai';
           } else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
             providerType = 'Anthropic';
+            providerTypeEnum = 'anthropic';
           } else if (deviceName.includes('ollama')) {
             providerType = 'Ollama';
+            providerTypeEnum = 'openai'; // Ollama uses OpenAI-compatible format
           } else if (deviceName.includes('gemini') || deviceName.includes('google')) {
             providerType = 'Google';
+            providerTypeEnum = 'openai'; // Google uses OpenAI-compatible format
           } else if (deviceName.includes('llama')) {
             providerType = 'llama.cpp';
+            providerTypeEnum = 'openai'; // llama.cpp uses OpenAI-compatible format
+          } else if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
+            providerType = 'Scrypted LLM';
+            providerTypeEnum = 'unknown';
           }
 
           this.llmDevice = device as unknown as ChatCompletionDevice;
           this.llmProvider = `${providerType} (${device.name})`;
+          this.llmProviderType = providerTypeEnum;
           this.console.log(`[LLM] Connected to ${providerType}: ${device.name}`);
           this.console.log(`[LLM] Plugin: ${pluginId || 'N/A'}`);
+          this.console.log(`[LLM] Image format: ${providerTypeEnum}`);
           this.console.log(`[LLM] Interfaces: ${device.interfaces?.join(', ')}`);
           return this.llmDevice;
         }
@@ -423,6 +456,11 @@ export class SpatialReasoningEngine {
     return this.llmProvider;
   }
 
+  /** Get the current LLM provider type for image format selection */
+  getLlmProviderType(): LlmProvider {
+    return this.llmProviderType;
+  }
+
   /** Check if LLM is available */
   isLlmAvailable(): boolean {
     return this.llmDevice !== null;
@@ -801,10 +839,10 @@ export class SpatialReasoningEngine {
       // Build message content - use multimodal format if we have an image
       let messageContent: any;
       if (imageData) {
-        // Vision-capable multimodal message format (Anthropic native format)
+        // Vision-capable multimodal message format (provider-specific)
         messageContent = [
           { type: 'text', text: prompt },
-          buildImageContent(imageData),
+          buildImageContent(imageData, this.llmProviderType),
         ];
       } else {
         // Fallback to text-only if image conversion failed
@@ -906,10 +944,10 @@ If no clear landmark is identifiable, respond with: {"name": null}`;
       // Build message content - use multimodal format if we have an image
       let messageContent: any;
       if (imageData) {
-        // Vision-capable multimodal message format (Anthropic native format)
+        // Vision-capable multimodal message format (provider-specific)
         messageContent = [
           { type: 'text', text: prompt },
-          buildImageContent(imageData),
+          buildImageContent(imageData, this.llmProviderType),
         ];
       } else {
         // Fallback to text-only if image conversion failed

package/src/core/topology-discovery.ts

@@ -30,7 +30,7 @@ import {
   Landmark,
   findCamera,
 } from '../models/topology';
-import { mediaObjectToBase64, buildImageContent, ImageData } from './spatial-reasoning';
+import { mediaObjectToBase64, buildImageContent, ImageData, LlmProvider } from './spatial-reasoning';
 
 const { systemManager } = sdk;
 
@@ -100,6 +100,7 @@ export class TopologyDiscoveryEngine {
   private topology: CameraTopology | null = null;
   private llmDevice: ChatCompletionDevice | null = null;
   private llmSearched: boolean = false;
+  private llmProviderType: LlmProvider = 'unknown';
 
   // Scene analysis cache (camera ID -> analysis)
   private sceneCache: Map<string, SceneAnalysis> = new Map();
@@ -177,8 +178,24 @@ export class TopologyDiscoveryEngine {
         if (!device) continue;
 
         if (device.interfaces?.includes('ChatCompletion')) {
+          const deviceName = device.name?.toLowerCase() || '';
+
+          // Detect provider type for image format selection
+          if (deviceName.includes('openai') || deviceName.includes('gpt')) {
+            this.llmProviderType = 'openai';
+          } else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
+            this.llmProviderType = 'anthropic';
+          } else if (deviceName.includes('ollama') || deviceName.includes('gemini') ||
+                     deviceName.includes('google') || deviceName.includes('llama')) {
+            // These providers use OpenAI-compatible format
+            this.llmProviderType = 'openai';
+          } else {
+            this.llmProviderType = 'unknown';
+          }
+
           this.llmDevice = device as unknown as ChatCompletionDevice;
           this.console.log(`[Discovery] Connected to LLM: ${device.name}`);
+          this.console.log(`[Discovery] Image format: ${this.llmProviderType}`);
           return this.llmDevice;
         }
       }
@@ -237,14 +254,14 @@ export class TopologyDiscoveryEngine {
     }
 
     try {
-      // Build multimodal message with Anthropic-native format
+      // Build multimodal message with provider-specific image format
       const result = await llm.getChatCompletion({
         messages: [
           {
             role: 'user',
             content: [
               { type: 'text', text: SCENE_ANALYSIS_PROMPT },
-              buildImageContent(imageData),
+              buildImageContent(imageData, this.llmProviderType),
             ],
           },
         ],