@blueharford/scrypted-spatial-awareness 0.5.3 → 0.5.5

package/out/main.nodejs.js

@@ -35095,6 +35095,7 @@ Object.defineProperty(exports, "__esModule", ({ value: true }));
 exports.SpatialReasoningEngine = void 0;
 exports.mediaObjectToBase64 = mediaObjectToBase64;
 exports.buildImageContent = buildImageContent;
+exports.isVisionFormatError = isVisionFormatError;
 const sdk_1 = __importStar(__webpack_require__(/*! @scrypted/sdk */ "./node_modules/@scrypted/sdk/dist/src/index.js"));
 const topology_1 = __webpack_require__(/*! ../models/topology */ "./src/models/topology.ts");
 const { systemManager, mediaManager } = sdk_1.default;
@@ -35107,10 +35108,20 @@ async function mediaObjectToBase64(mediaObject) {
     try {
         // Convert MediaObject to Buffer using mediaManager
         const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, sdk_1.ScryptedMimeTypes.Image);
+        if (!buffer || buffer.length === 0) {
+            console.warn('Failed to convert MediaObject: empty buffer');
+            return null;
+        }
         // Convert buffer to base64 (raw, no data URL prefix)
         const base64 = buffer.toString('base64');
+        // Validate base64 - check it's not empty and looks valid
+        if (!base64 || base64.length < 100) {
+            console.warn(`Invalid base64: length=${base64?.length || 0}`);
+            return null;
+        }
         // Determine MIME type - default to JPEG for camera images
         const mediaType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
+        console.log(`[Image] Converted to base64: ${base64.length} chars, type=${mediaType}`);
         return { base64, mediaType };
     }
     catch (e) {
@@ -35120,9 +35131,9 @@ async function mediaObjectToBase64(mediaObject) {
 }
 /**
  * Build image content block for ChatCompletion API
- * Supports both OpenAI and Anthropic formats
+ * Supports OpenAI, Anthropic, and @scrypted/llm formats
  * @param imageData - Image data with base64 and media type
- * @param provider - The LLM provider type (openai, anthropic, or unknown)
+ * @param provider - The LLM provider type
  */
 function buildImageContent(imageData, provider = 'unknown') {
     if (provider === 'openai') {
@@ -35131,11 +35142,12 @@ function buildImageContent(imageData, provider = 'unknown') {
             type: 'image_url',
             image_url: {
                 url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+                detail: 'auto',
             },
         };
     }
     else if (provider === 'anthropic') {
-        // Anthropic format: uses separate base64 data and media_type
+        // Anthropic official format: uses 'data' key
         return {
             type: 'image',
             source: {
@@ -35145,17 +35157,43 @@ function buildImageContent(imageData, provider = 'unknown') {
             },
         };
     }
+    else if (provider === 'scrypted') {
+        // @scrypted/llm format: uses 'base64' key (per error path .image.source.base64)
+        return {
+            type: 'image',
+            source: {
+                type: 'base64',
+                media_type: imageData.mediaType,
+                base64: imageData.base64,
+            },
+        };
+    }
     else {
-        // Unknown provider: try OpenAI format as it's more commonly supported
-        // Most LLM wrappers (including @scrypted/llm) understand the OpenAI format
+        // Unknown provider: try @scrypted/llm format first
         return {
-            type: 'image_url',
-            image_url: {
-                url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+            type: 'image',
+            source: {
+                type: 'base64',
+                media_type: imageData.mediaType,
+                base64: imageData.base64,
             },
         };
     }
 }
+/** Check if an error indicates vision/multimodal content format issue (should try alternate format) */
+function isVisionFormatError(error) {
+    const errorStr = String(error);
+    return (errorStr.includes('content.str') ||
+        errorStr.includes('should be a valid string') ||
+        errorStr.includes('Invalid content type') ||
+        errorStr.includes('does not support vision') ||
+        errorStr.includes('invalid base64') ||
+        errorStr.includes('Invalid base64') ||
+        errorStr.includes('.image.source') ||
+        errorStr.includes('.image_url') ||
+        (errorStr.includes('image_url') && errorStr.includes('not supported')) ||
+        (errorStr.includes('400') && errorStr.includes('content')));
+}
 class SpatialReasoningEngine {
     config;
     console;
@@ -36186,72 +36224,120 @@ class TopologyDiscoveryEngine {
             analysis.error = 'Failed to capture camera snapshot';
             return analysis;
         }
-        try {
-            // Build multimodal message with provider-specific image format
-            const result = await llm.getChatCompletion({
-                messages: [
-                    {
-                        role: 'user',
-                        content: [
-                            { type: 'text', text: SCENE_ANALYSIS_PROMPT },
-                            (0, spatial_reasoning_1.buildImageContent)(imageData, this.llmProviderType),
-                        ],
-                    },
-                ],
-                max_tokens: 500,
-                temperature: 0.3,
-            });
-            const content = result?.choices?.[0]?.message?.content;
-            if (content && typeof content === 'string') {
-                try {
-                    // Extract JSON from response (handle markdown code blocks)
-                    let jsonStr = content.trim();
-                    if (jsonStr.startsWith('```')) {
-                        jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
-                    }
-                    const parsed = JSON.parse(jsonStr);
-                    // Map parsed data to our types
-                    if (Array.isArray(parsed.landmarks)) {
-                        analysis.landmarks = parsed.landmarks.map((l) => ({
-                            name: l.name || 'Unknown',
-                            type: this.mapLandmarkType(l.type),
-                            confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
-                            description: l.description || '',
-                            boundingBox: l.boundingBox,
-                        }));
-                    }
-                    if (Array.isArray(parsed.zones)) {
-                        analysis.zones = parsed.zones.map((z) => ({
-                            name: z.name || 'Unknown',
-                            type: this.mapZoneType(z.type),
-                            coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
-                            description: z.description || '',
-                            boundingBox: z.boundingBox,
-                        }));
-                    }
-                    if (parsed.edges && typeof parsed.edges === 'object') {
-                        analysis.edges = {
-                            top: parsed.edges.top || '',
-                            left: parsed.edges.left || '',
-                            right: parsed.edges.right || '',
-                            bottom: parsed.edges.bottom || '',
-                        };
+        // Try with detected provider format first, then fallback to alternates
+        // The order matters: try the most likely formats first
+        const formatsToTry = [];
+        // Start with detected format
+        formatsToTry.push(this.llmProviderType);
+        // Add fallbacks based on detected provider
+        if (this.llmProviderType === 'openai') {
+            formatsToTry.push('scrypted', 'anthropic');
+        }
+        else if (this.llmProviderType === 'anthropic') {
+            formatsToTry.push('scrypted', 'openai');
+        }
+        else if (this.llmProviderType === 'scrypted') {
+            formatsToTry.push('anthropic', 'openai');
+        }
+        else {
+            // Unknown - try all formats
+            formatsToTry.push('scrypted', 'anthropic', 'openai');
+        }
+        let lastError = null;
+        for (const formatType of formatsToTry) {
+            try {
+                this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
+                // Build multimodal message with provider-specific image format
+                const result = await llm.getChatCompletion({
+                    messages: [
+                        {
+                            role: 'user',
+                            content: [
+                                { type: 'text', text: SCENE_ANALYSIS_PROMPT },
+                                (0, spatial_reasoning_1.buildImageContent)(imageData, formatType),
+                            ],
+                        },
+                    ],
+                    max_tokens: 500,
+                    temperature: 0.3,
+                });
+                const content = result?.choices?.[0]?.message?.content;
+                if (content && typeof content === 'string') {
+                    try {
+                        // Extract JSON from response (handle markdown code blocks)
+                        let jsonStr = content.trim();
+                        if (jsonStr.startsWith('```')) {
+                            jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
+                        }
+                        const parsed = JSON.parse(jsonStr);
+                        // Map parsed data to our types
+                        if (Array.isArray(parsed.landmarks)) {
+                            analysis.landmarks = parsed.landmarks.map((l) => ({
+                                name: l.name || 'Unknown',
+                                type: this.mapLandmarkType(l.type),
+                                confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
+                                description: l.description || '',
+                                boundingBox: l.boundingBox,
+                            }));
+                        }
+                        if (Array.isArray(parsed.zones)) {
+                            analysis.zones = parsed.zones.map((z) => ({
+                                name: z.name || 'Unknown',
+                                type: this.mapZoneType(z.type),
+                                coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
+                                description: z.description || '',
+                                boundingBox: z.boundingBox,
+                            }));
+                        }
+                        if (parsed.edges && typeof parsed.edges === 'object') {
+                            analysis.edges = {
+                                top: parsed.edges.top || '',
+                                left: parsed.edges.left || '',
+                                right: parsed.edges.right || '',
+                                bottom: parsed.edges.bottom || '',
+                            };
+                        }
+                        if (parsed.orientation) {
+                            analysis.orientation = this.mapOrientation(parsed.orientation);
+                        }
+                        analysis.isValid = true;
+                        this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
+                        // Update the preferred format for future requests
+                        if (formatType !== this.llmProviderType) {
+                            this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
+                            this.llmProviderType = formatType;
+                        }
+                        // Success - exit the retry loop
+                        return analysis;
                     }
-                    if (parsed.orientation) {
-                        analysis.orientation = this.mapOrientation(parsed.orientation);
+                    catch (parseError) {
+                        this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
+                        analysis.error = 'Failed to parse LLM response';
+                        return analysis;
                     }
-                    analysis.isValid = true;
-                    this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones`);
                 }
-                catch (parseError) {
-                    this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
-                    analysis.error = 'Failed to parse LLM response';
+            }
+            catch (e) {
+                lastError = e;
+                // Check if this is a vision/multimodal format error
+                if ((0, spatial_reasoning_1.isVisionFormatError)(e)) {
+                    this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
+                    continue; // Try next format
                 }
+                // Not a format error - don't retry
+                this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
+                break;
             }
         }
-        catch (e) {
-            this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
-            analysis.error = `Analysis failed: ${e}`;
+        // All formats failed
+        if (lastError) {
+            const errorStr = String(lastError);
+            if ((0, spatial_reasoning_1.isVisionFormatError)(lastError)) {
+                analysis.error = 'Vision/image analysis failed with all formats. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured and the @scrypted/llm plugin supports vision.';
+            }
+            else {
+                analysis.error = `Analysis failed: ${errorStr}`;
+            }
         }
         // Cache the analysis
         this.sceneCache.set(cameraId, analysis);