@blueharford/scrypted-spatial-awareness 0.5.4 → 0.5.5

package/out/main.nodejs.js

@@ -35095,7 +35095,7 @@ Object.defineProperty(exports, "__esModule", ({ value: true }));
 exports.SpatialReasoningEngine = void 0;
 exports.mediaObjectToBase64 = mediaObjectToBase64;
 exports.buildImageContent = buildImageContent;
-exports.isVisionNotSupportedError = isVisionNotSupportedError;
+exports.isVisionFormatError = isVisionFormatError;
 const sdk_1 = __importStar(__webpack_require__(/*! @scrypted/sdk */ "./node_modules/@scrypted/sdk/dist/src/index.js"));
 const topology_1 = __webpack_require__(/*! ../models/topology */ "./src/models/topology.ts");
 const { systemManager, mediaManager } = sdk_1.default;
@@ -35108,10 +35108,20 @@ async function mediaObjectToBase64(mediaObject) {
     try {
         // Convert MediaObject to Buffer using mediaManager
         const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, sdk_1.ScryptedMimeTypes.Image);
+        if (!buffer || buffer.length === 0) {
+            console.warn('Failed to convert MediaObject: empty buffer');
+            return null;
+        }
         // Convert buffer to base64 (raw, no data URL prefix)
         const base64 = buffer.toString('base64');
+        // Validate base64 - check it's not empty and looks valid
+        if (!base64 || base64.length < 100) {
+            console.warn(`Invalid base64: length=${base64?.length || 0}`);
+            return null;
+        }
         // Determine MIME type - default to JPEG for camera images
         const mediaType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
+        console.log(`[Image] Converted to base64: ${base64.length} chars, type=${mediaType}`);
         return { base64, mediaType };
     }
     catch (e) {
@@ -35121,14 +35131,13 @@ async function mediaObjectToBase64(mediaObject) {
 }
 /**
  * Build image content block for ChatCompletion API
- * Supports both OpenAI and Anthropic formats
+ * Supports OpenAI, Anthropic, and @scrypted/llm formats
  * @param imageData - Image data with base64 and media type
- * @param provider - The LLM provider type (openai, anthropic, or unknown)
+ * @param provider - The LLM provider type
  */
 function buildImageContent(imageData, provider = 'unknown') {
     if (provider === 'openai') {
         // OpenAI format: uses data URL with image_url wrapper
-        // Include detail parameter for compatibility
         return {
             type: 'image_url',
             image_url: {
@@ -35138,7 +35147,7 @@ function buildImageContent(imageData, provider = 'unknown') {
         };
     }
     else if (provider === 'anthropic') {
-        // Anthropic format: uses separate base64 data and media_type
+        // Anthropic official format: uses 'data' key
         return {
             type: 'image',
             source: {
@@ -35148,27 +35157,42 @@ function buildImageContent(imageData, provider = 'unknown') {
             },
         };
     }
+    else if (provider === 'scrypted') {
+        // @scrypted/llm format: uses 'base64' key (per error path .image.source.base64)
+        return {
+            type: 'image',
+            source: {
+                type: 'base64',
+                media_type: imageData.mediaType,
+                base64: imageData.base64,
+            },
+        };
+    }
     else {
-        // Unknown provider: try Anthropic format first as it's more explicit
-        // Some plugins may translate this to OpenAI format internally
+        // Unknown provider: try @scrypted/llm format first
         return {
             type: 'image',
             source: {
                 type: 'base64',
                 media_type: imageData.mediaType,
-                data: imageData.base64,
+                base64: imageData.base64,
             },
         };
     }
 }
-/** Check if an error indicates vision/multimodal content is not supported */
-function isVisionNotSupportedError(error) {
+/** Check if an error indicates vision/multimodal content format issue (should try alternate format) */
+function isVisionFormatError(error) {
     const errorStr = String(error);
     return (errorStr.includes('content.str') ||
         errorStr.includes('should be a valid string') ||
         errorStr.includes('Invalid content type') ||
         errorStr.includes('does not support vision') ||
-        errorStr.includes('image_url') && errorStr.includes('not supported'));
+        errorStr.includes('invalid base64') ||
+        errorStr.includes('Invalid base64') ||
+        errorStr.includes('.image.source') ||
+        errorStr.includes('.image_url') ||
+        (errorStr.includes('image_url') && errorStr.includes('not supported')) ||
+        (errorStr.includes('400') && errorStr.includes('content')));
 }
 class SpatialReasoningEngine {
     config;
@@ -36200,18 +36224,24 @@ class TopologyDiscoveryEngine {
             analysis.error = 'Failed to capture camera snapshot';
             return analysis;
         }
-        // Try with detected provider format first, then fallback to alternate format
-        const formatsToTry = [this.llmProviderType];
-        // Add fallback format
+        // Try with detected provider format first, then fallback to alternates
+        // The order matters: try the most likely formats first
+        const formatsToTry = [];
+        // Start with detected format
+        formatsToTry.push(this.llmProviderType);
+        // Add fallbacks based on detected provider
         if (this.llmProviderType === 'openai') {
-            formatsToTry.push('anthropic');
+            formatsToTry.push('scrypted', 'anthropic');
         }
         else if (this.llmProviderType === 'anthropic') {
-            formatsToTry.push('openai');
+            formatsToTry.push('scrypted', 'openai');
+        }
+        else if (this.llmProviderType === 'scrypted') {
+            formatsToTry.push('anthropic', 'openai');
         }
         else {
-            // Unknown - try both
-            formatsToTry.push('openai');
+            // Unknown - try all formats
+            formatsToTry.push('scrypted', 'anthropic', 'openai');
         }
         let lastError = null;
         for (const formatType of formatsToTry) {
@@ -36290,8 +36320,8 @@ class TopologyDiscoveryEngine {
             catch (e) {
                 lastError = e;
                 // Check if this is a vision/multimodal format error
-                if ((0, spatial_reasoning_1.isVisionNotSupportedError)(e)) {
-                    this.console.warn(`[Discovery] ${formatType} format not supported, trying fallback...`);
+                if ((0, spatial_reasoning_1.isVisionFormatError)(e)) {
+                    this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
                     continue; // Try next format
                 }
                 // Not a format error - don't retry
@@ -36302,8 +36332,8 @@ class TopologyDiscoveryEngine {
         // All formats failed
         if (lastError) {
             const errorStr = String(lastError);
-            if ((0, spatial_reasoning_1.isVisionNotSupportedError)(lastError)) {
-                analysis.error = 'Vision/image analysis not supported by configured LLM. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured.';
+            if ((0, spatial_reasoning_1.isVisionFormatError)(lastError)) {
+                analysis.error = 'Vision/image analysis failed with all formats. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured and the @scrypted/llm plugin supports vision.';
             }
             else {
                 analysis.error = `Analysis failed: ${errorStr}`;