@blueharford/scrypted-spatial-awareness 0.5.4 → 0.5.6

package/out/main.nodejs.js

@@ -35095,7 +35095,7 @@ Object.defineProperty(exports, "__esModule", ({ value: true }));
 exports.SpatialReasoningEngine = void 0;
 exports.mediaObjectToBase64 = mediaObjectToBase64;
 exports.buildImageContent = buildImageContent;
-exports.isVisionNotSupportedError = isVisionNotSupportedError;
+exports.isVisionFormatError = isVisionFormatError;
 const sdk_1 = __importStar(__webpack_require__(/*! @scrypted/sdk */ "./node_modules/@scrypted/sdk/dist/src/index.js"));
 const topology_1 = __webpack_require__(/*! ../models/topology */ "./src/models/topology.ts");
 const { systemManager, mediaManager } = sdk_1.default;
@@ -35106,29 +35106,46 @@ const { systemManager, mediaManager } = sdk_1.default;
  */
 async function mediaObjectToBase64(mediaObject) {
     try {
+        console.log(`[Image] Converting MediaObject, mimeType=${mediaObject?.mimeType}`);
         // Convert MediaObject to Buffer using mediaManager
         const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, sdk_1.ScryptedMimeTypes.Image);
+        if (!buffer) {
+            console.warn('[Image] convertMediaObjectToBuffer returned null/undefined');
+            return null;
+        }
+        console.log(`[Image] Buffer received: ${buffer.length} bytes`);
+        if (buffer.length === 0) {
+            console.warn('[Image] Buffer is empty (0 bytes)');
+            return null;
+        }
+        // Check if buffer is too small to be a valid image (< 1KB is suspicious)
+        if (buffer.length < 1000) {
+            // Log what the buffer contains - might be an error message
+            const bufferContent = buffer.toString('utf8').substring(0, 100);
+            console.warn(`[Image] Buffer too small (${buffer.length} bytes), content: ${bufferContent}`);
+            return null;
+        }
         // Convert buffer to base64 (raw, no data URL prefix)
         const base64 = buffer.toString('base64');
         // Determine MIME type - default to JPEG for camera images
         const mediaType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
+        console.log(`[Image] Converted to base64: ${base64.length} chars, type=${mediaType}`);
         return { base64, mediaType };
     }
     catch (e) {
-        console.warn('Failed to convert MediaObject to base64:', e);
+        console.warn('[Image] Failed to convert MediaObject to base64:', e);
         return null;
     }
 }
 /**
  * Build image content block for ChatCompletion API
- * Supports both OpenAI and Anthropic formats
+ * Supports OpenAI, Anthropic, and @scrypted/llm formats
  * @param imageData - Image data with base64 and media type
- * @param provider - The LLM provider type (openai, anthropic, or unknown)
+ * @param provider - The LLM provider type
  */
 function buildImageContent(imageData, provider = 'unknown') {
     if (provider === 'openai') {
         // OpenAI format: uses data URL with image_url wrapper
-        // Include detail parameter for compatibility
         return {
             type: 'image_url',
             image_url: {
@@ -35138,7 +35155,7 @@ function buildImageContent(imageData, provider = 'unknown') {
         };
     }
     else if (provider === 'anthropic') {
-        // Anthropic format: uses separate base64 data and media_type
+        // Anthropic official format: uses 'data' key
         return {
             type: 'image',
             source: {
@@ -35148,27 +35165,42 @@ function buildImageContent(imageData, provider = 'unknown') {
             },
         };
     }
+    else if (provider === 'scrypted') {
+        // @scrypted/llm format: uses 'base64' key (per error path .image.source.base64)
+        return {
+            type: 'image',
+            source: {
+                type: 'base64',
+                media_type: imageData.mediaType,
+                base64: imageData.base64,
+            },
+        };
+    }
     else {
-        // Unknown provider: try Anthropic format first as it's more explicit
-        // Some plugins may translate this to OpenAI format internally
+        // Unknown provider: try @scrypted/llm format first
         return {
             type: 'image',
             source: {
                 type: 'base64',
                 media_type: imageData.mediaType,
-                data: imageData.base64,
+                base64: imageData.base64,
             },
         };
     }
 }
-/** Check if an error indicates vision/multimodal content is not supported */
-function isVisionNotSupportedError(error) {
+/** Check if an error indicates vision/multimodal content format issue (should try alternate format) */
+function isVisionFormatError(error) {
     const errorStr = String(error);
     return (errorStr.includes('content.str') ||
         errorStr.includes('should be a valid string') ||
         errorStr.includes('Invalid content type') ||
         errorStr.includes('does not support vision') ||
-        errorStr.includes('image_url') && errorStr.includes('not supported'));
+        errorStr.includes('invalid base64') ||
+        errorStr.includes('Invalid base64') ||
+        errorStr.includes('.image.source') ||
+        errorStr.includes('.image_url') ||
+        (errorStr.includes('image_url') && errorStr.includes('not supported')) ||
+        (errorStr.includes('400') && errorStr.includes('content')));
 }
 class SpatialReasoningEngine {
     config;
@@ -36165,10 +36197,21 @@ class TopologyDiscoveryEngine {
         try {
             const camera = systemManager.getDeviceById(cameraId);
             if (!camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)) {
+                this.console.warn(`[Discovery] Camera ${cameraId} doesn't have Camera interface`);
                 return null;
             }
+            this.console.log(`[Discovery] Taking snapshot from camera: ${camera.name || cameraId}`);
             const mediaObject = await camera.takePicture();
-            return (0, spatial_reasoning_1.mediaObjectToBase64)(mediaObject);
+            if (!mediaObject) {
+                this.console.warn(`[Discovery] takePicture() returned null for ${camera.name}`);
+                return null;
+            }
+            this.console.log(`[Discovery] MediaObject received: mimeType=${mediaObject.mimeType}`);
+            const imageData = await (0, spatial_reasoning_1.mediaObjectToBase64)(mediaObject);
+            if (!imageData) {
+                this.console.warn(`[Discovery] Failed to convert MediaObject to base64 for ${camera.name}`);
+            }
+            return imageData;
         }
         catch (e) {
             this.console.warn(`[Discovery] Failed to get snapshot from camera ${cameraId}:`, e);
@@ -36200,18 +36243,24 @@ class TopologyDiscoveryEngine {
             analysis.error = 'Failed to capture camera snapshot';
             return analysis;
         }
-        // Try with detected provider format first, then fallback to alternate format
-        const formatsToTry = [this.llmProviderType];
-        // Add fallback format
+        // Try with detected provider format first, then fallback to alternates
+        // The order matters: try the most likely formats first
+        const formatsToTry = [];
+        // Start with detected format
+        formatsToTry.push(this.llmProviderType);
+        // Add fallbacks based on detected provider
         if (this.llmProviderType === 'openai') {
-            formatsToTry.push('anthropic');
+            formatsToTry.push('scrypted', 'anthropic');
         }
         else if (this.llmProviderType === 'anthropic') {
-            formatsToTry.push('openai');
+            formatsToTry.push('scrypted', 'openai');
+        }
+        else if (this.llmProviderType === 'scrypted') {
+            formatsToTry.push('anthropic', 'openai');
         }
         else {
-            // Unknown - try both
-            formatsToTry.push('openai');
+            // Unknown - try all formats
+            formatsToTry.push('scrypted', 'anthropic', 'openai');
         }
         let lastError = null;
         for (const formatType of formatsToTry) {
@@ -36290,8 +36339,8 @@ class TopologyDiscoveryEngine {
             catch (e) {
                 lastError = e;
                 // Check if this is a vision/multimodal format error
-                if ((0, spatial_reasoning_1.isVisionNotSupportedError)(e)) {
-                    this.console.warn(`[Discovery] ${formatType} format not supported, trying fallback...`);
+                if ((0, spatial_reasoning_1.isVisionFormatError)(e)) {
+                    this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
                     continue; // Try next format
                 }
                 // Not a format error - don't retry
@@ -36302,8 +36351,8 @@ class TopologyDiscoveryEngine {
         // All formats failed
         if (lastError) {
             const errorStr = String(lastError);
-            if ((0, spatial_reasoning_1.isVisionNotSupportedError)(lastError)) {
-                analysis.error = 'Vision/image analysis not supported by configured LLM. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured.';
+            if ((0, spatial_reasoning_1.isVisionFormatError)(lastError)) {
+                analysis.error = 'Vision/image analysis failed with all formats. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured and the @scrypted/llm plugin supports vision.';
             }
             else {
                 analysis.error = `Analysis failed: ${errorStr}`;