@blueharford/scrypted-spatial-awareness 0.5.2 → 0.5.3

package/out/main.nodejs.js

@@ -35120,19 +35120,41 @@ async function mediaObjectToBase64(mediaObject) {
 }
 /**
  * Build image content block for ChatCompletion API
- * Compatible with both OpenAI and Anthropic formats via @scrypted/llm
+ * Supports both OpenAI and Anthropic formats
+ * @param imageData - Image data with base64 and media type
+ * @param provider - The LLM provider type (openai, anthropic, or unknown)
  */
-function buildImageContent(imageData) {
-    // Use Anthropic's native format which @scrypted/llm should translate
-    // This format is more explicit about the base64 data
-    return {
-        type: 'image',
-        source: {
-            type: 'base64',
-            media_type: imageData.mediaType,
-            data: imageData.base64,
-        },
-    };
+function buildImageContent(imageData, provider = 'unknown') {
+    if (provider === 'openai') {
+        // OpenAI format: uses data URL with image_url wrapper
+        return {
+            type: 'image_url',
+            image_url: {
+                url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+            },
+        };
+    }
+    else if (provider === 'anthropic') {
+        // Anthropic format: uses separate base64 data and media_type
+        return {
+            type: 'image',
+            source: {
+                type: 'base64',
+                media_type: imageData.mediaType,
+                data: imageData.base64,
+            },
+        };
+    }
+    else {
+        // Unknown provider: try OpenAI format as it's more commonly supported
+        // Most LLM wrappers (including @scrypted/llm) understand the OpenAI format
+        return {
+            type: 'image_url',
+            image_url: {
+                url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+            },
+        };
+    }
 }
 class SpatialReasoningEngine {
     config;
@@ -35353,6 +35375,7 @@ class SpatialReasoningEngine {
     }
     llmSearched = false;
     llmProvider = null;
+    llmProviderType = 'unknown';
     /** Find or initialize LLM device - looks for ChatCompletion interface from @scrypted/llm plugin */
     async findLlmDevice() {
         if (this.llmDevice)
@@ -35371,30 +35394,39 @@ class SpatialReasoningEngine {
                 if (device.interfaces?.includes('ChatCompletion')) {
                     const deviceName = device.name?.toLowerCase() || '';
                     const pluginId = device.pluginId?.toLowerCase() || '';
-                    // Identify the provider type for logging
+                    // Identify the provider type for logging and image format selection
                     let providerType = 'Unknown';
-                    if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
-                        providerType = 'Scrypted LLM';
-                    }
+                    let providerTypeEnum = 'unknown';
                     if (deviceName.includes('openai') || deviceName.includes('gpt')) {
                         providerType = 'OpenAI';
+                        providerTypeEnum = 'openai';
                     }
                     else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
                         providerType = 'Anthropic';
+                        providerTypeEnum = 'anthropic';
                     }
                     else if (deviceName.includes('ollama')) {
                         providerType = 'Ollama';
+                        providerTypeEnum = 'openai'; // Ollama uses OpenAI-compatible format
                     }
                     else if (deviceName.includes('gemini') || deviceName.includes('google')) {
                         providerType = 'Google';
+                        providerTypeEnum = 'openai'; // Google uses OpenAI-compatible format
                     }
                     else if (deviceName.includes('llama')) {
                         providerType = 'llama.cpp';
+                        providerTypeEnum = 'openai'; // llama.cpp uses OpenAI-compatible format
+                    }
+                    else if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
+                        providerType = 'Scrypted LLM';
+                        providerTypeEnum = 'unknown';
                     }
                     this.llmDevice = device;
                     this.llmProvider = `${providerType} (${device.name})`;
+                    this.llmProviderType = providerTypeEnum;
                     this.console.log(`[LLM] Connected to ${providerType}: ${device.name}`);
                     this.console.log(`[LLM] Plugin: ${pluginId || 'N/A'}`);
+                    this.console.log(`[LLM] Image format: ${providerTypeEnum}`);
                     this.console.log(`[LLM] Interfaces: ${device.interfaces?.join(', ')}`);
                     return this.llmDevice;
                 }
@@ -35412,6 +35444,10 @@ class SpatialReasoningEngine {
     getLlmProvider() {
         return this.llmProvider;
     }
+    /** Get the current LLM provider type for image format selection */
+    getLlmProviderType() {
+        return this.llmProviderType;
+    }
     /** Check if LLM is available */
     isLlmAvailable() {
         return this.llmDevice !== null;
@@ -35688,10 +35724,10 @@ class SpatialReasoningEngine {
             // Build message content - use multimodal format if we have an image
             let messageContent;
             if (imageData) {
-                // Vision-capable multimodal message format (Anthropic native format)
+                // Vision-capable multimodal message format (provider-specific)
                 messageContent = [
                     { type: 'text', text: prompt },
-                    buildImageContent(imageData),
+                    buildImageContent(imageData, this.llmProviderType),
                 ];
             }
             else {
@@ -35774,10 +35810,10 @@ If no clear landmark is identifiable, respond with: {"name": null}`;
             // Build message content - use multimodal format if we have an image
             let messageContent;
             if (imageData) {
-                // Vision-capable multimodal message format (Anthropic native format)
+                // Vision-capable multimodal message format (provider-specific)
                 messageContent = [
                     { type: 'text', text: prompt },
-                    buildImageContent(imageData),
+                    buildImageContent(imageData, this.llmProviderType),
                 ];
             }
             else {
@@ -36017,6 +36053,7 @@ class TopologyDiscoveryEngine {
     topology = null;
     llmDevice = null;
     llmSearched = false;
+    llmProviderType = 'unknown';
     // Scene analysis cache (camera ID -> analysis)
     sceneCache = new Map();
     // Pending suggestions for user review
@@ -36080,8 +36117,25 @@ class TopologyDiscoveryEngine {
                 if (!device)
                     continue;
                 if (device.interfaces?.includes('ChatCompletion')) {
+                    const deviceName = device.name?.toLowerCase() || '';
+                    // Detect provider type for image format selection
+                    if (deviceName.includes('openai') || deviceName.includes('gpt')) {
+                        this.llmProviderType = 'openai';
+                    }
+                    else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
+                        this.llmProviderType = 'anthropic';
+                    }
+                    else if (deviceName.includes('ollama') || deviceName.includes('gemini') ||
+                        deviceName.includes('google') || deviceName.includes('llama')) {
+                        // These providers use OpenAI-compatible format
+                        this.llmProviderType = 'openai';
+                    }
+                    else {
+                        this.llmProviderType = 'unknown';
+                    }
                     this.llmDevice = device;
                     this.console.log(`[Discovery] Connected to LLM: ${device.name}`);
+                    this.console.log(`[Discovery] Image format: ${this.llmProviderType}`);
                     return this.llmDevice;
                 }
             }
@@ -36133,14 +36187,14 @@ class TopologyDiscoveryEngine {
             return analysis;
         }
         try {
-            // Build multimodal message with Anthropic-native format
+            // Build multimodal message with provider-specific image format
             const result = await llm.getChatCompletion({
                 messages: [
                     {
                         role: 'user',
                         content: [
                             { type: 'text', text: SCENE_ANALYSIS_PROMPT },
-                            (0, spatial_reasoning_1.buildImageContent)(imageData),
+                            (0, spatial_reasoning_1.buildImageContent)(imageData, this.llmProviderType),
                         ],
                     },
                 ],