@blueharford/scrypted-spatial-awareness 0.6.9 → 0.6.11

package/out/main.nodejs.js

@@ -35137,54 +35137,23 @@ async function mediaObjectToBase64(mediaObject) {
 }
 /**
  * Build image content block for ChatCompletion API
- * Supports OpenAI, Anthropic, and @scrypted/llm formats
+ *
+ * IMPORTANT: @scrypted/llm uses OpenAI-compatible format for ALL providers.
+ * The plugin internally converts this format to the appropriate provider format.
+ * So we ALWAYS use the OpenAI image_url format with data URI.
+ *
  * @param imageData - Image data with base64 and media type
- * @param provider - The LLM provider type
+ * @param provider - The LLM provider type (currently unused, kept for logging)
  */
 function buildImageContent(imageData, provider = 'unknown') {
-    if (provider === 'openai') {
-        // OpenAI format: uses data URL with image_url wrapper
-        return {
-            type: 'image_url',
-            image_url: {
-                url: `data:${imageData.mediaType};base64,${imageData.base64}`,
-                detail: 'auto',
-            },
-        };
-    }
-    else if (provider === 'anthropic') {
-        // Anthropic official format: uses 'data' key
-        return {
-            type: 'image',
-            source: {
-                type: 'base64',
-                media_type: imageData.mediaType,
-                data: imageData.base64,
-            },
-        };
-    }
-    else if (provider === 'scrypted') {
-        // @scrypted/llm format: uses 'base64' key (per error path .image.source.base64)
-        return {
-            type: 'image',
-            source: {
-                type: 'base64',
-                media_type: imageData.mediaType,
-                base64: imageData.base64,
-            },
-        };
-    }
-    else {
-        // Unknown provider: try @scrypted/llm format first
-        return {
-            type: 'image',
-            source: {
-                type: 'base64',
-                media_type: imageData.mediaType,
-                base64: imageData.base64,
-            },
-        };
-    }
+    // @scrypted/llm uses OpenAI-compatible format for ALL providers
+    // The plugin handles internal conversion to Anthropic/other formats
+    return {
+        type: 'image_url',
+        image_url: {
+            url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+        },
+    };
 }
 /** Check if an error indicates vision/multimodal content format issue (should try alternate format) */
 function isVisionFormatError(error) {
@@ -35905,32 +35874,56 @@ Examples of good descriptions:
 - "Landscaper with leaf blower heading to work truck"
 
 Generate ONLY the description, nothing else:`;
-            // Build message content - use multimodal format if we have an image
-            let messageContent;
+            // Try multimodal format first, fall back to text-only if it fails
+            let result;
+            let usedVision = false;
             if (imageData) {
-                messageContent = [
-                    { type: 'text', text: prompt },
-                    buildImageContent(imageData, this.llmProviderType),
-                ];
+                // First attempt: Try multimodal with image
+                try {
+                    this.console.log(`[LLM] Attempting multimodal ${eventType} call with image...`);
+                    const multimodalContent = [
+                        { type: 'text', text: prompt },
+                        buildImageContent(imageData, this.llmProviderType),
+                    ];
+                    result = await llm.getChatCompletion({
+                        messages: [
+                            {
+                                role: 'user',
+                                content: multimodalContent,
+                            },
+                        ],
+                        max_tokens: 100,
+                        temperature: 0.7,
+                    });
+                    usedVision = true;
+                }
+                catch (visionError) {
+                    // If vision format fails, try text-only
+                    if (isVisionFormatError(visionError)) {
+                        this.console.warn(`[LLM] Vision format not supported, falling back to text-only: ${visionError.message || visionError}`);
+                    }
+                    else {
+                        this.console.warn(`[LLM] Multimodal call failed, trying text-only: ${visionError.message || visionError}`);
+                    }
+                }
             }
-            else {
-                messageContent = prompt;
+            // If no result yet, try text-only
+            if (!result) {
+                this.console.log(`[LLM] Calling text-only getChatCompletion for ${eventType}...`);
+                result = await llm.getChatCompletion({
+                    messages: [
+                        {
+                            role: 'user',
+                            content: prompt,
+                        },
+                    ],
+                    max_tokens: 100,
+                    temperature: 0.7,
+                });
             }
-            // Call LLM using ChatCompletion interface
-            this.console.log(`[LLM] Calling getChatCompletion for ${eventType}...`);
-            const result = await llm.getChatCompletion({
-                messages: [
-                    {
-                        role: 'user',
-                        content: messageContent,
-                    },
-                ],
-                max_tokens: 100,
-                temperature: 0.7,
-            });
             const content = result?.choices?.[0]?.message?.content;
             if (content && typeof content === 'string') {
-                this.console.log(`[LLM] Got ${eventType} description: ${content.trim().substring(0, 50)}...`);
+                this.console.log(`[LLM] Got ${eventType} description (vision=${usedVision}): ${content.trim().substring(0, 50)}...`);
                 return content.trim();
             }
             this.console.warn(`[LLM] No content in response for ${eventType}`);