@blueharford/scrypted-spatial-awareness 0.5.3 → 0.5.4

package/out/main.nodejs.js

@@ -35095,6 +35095,7 @@ Object.defineProperty(exports, "__esModule", ({ value: true }));
 exports.SpatialReasoningEngine = void 0;
 exports.mediaObjectToBase64 = mediaObjectToBase64;
 exports.buildImageContent = buildImageContent;
+exports.isVisionNotSupportedError = isVisionNotSupportedError;
 const sdk_1 = __importStar(__webpack_require__(/*! @scrypted/sdk */ "./node_modules/@scrypted/sdk/dist/src/index.js"));
 const topology_1 = __webpack_require__(/*! ../models/topology */ "./src/models/topology.ts");
 const { systemManager, mediaManager } = sdk_1.default;
@@ -35127,10 +35128,12 @@ async function mediaObjectToBase64(mediaObject) {
 function buildImageContent(imageData, provider = 'unknown') {
     if (provider === 'openai') {
         // OpenAI format: uses data URL with image_url wrapper
+        // Include detail parameter for compatibility
         return {
             type: 'image_url',
             image_url: {
                 url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+                detail: 'auto',
             },
         };
     }
@@ -35146,16 +35149,27 @@ function buildImageContent(imageData, provider = 'unknown') {
         };
     }
     else {
-        // Unknown provider: try OpenAI format as it's more commonly supported
-        // Most LLM wrappers (including @scrypted/llm) understand the OpenAI format
+        // Unknown provider: try Anthropic format first as it's more explicit
+        // Some plugins may translate this to OpenAI format internally
         return {
-            type: 'image_url',
-            image_url: {
-                url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+            type: 'image',
+            source: {
+                type: 'base64',
+                media_type: imageData.mediaType,
+                data: imageData.base64,
             },
         };
     }
 }
+/** Check if an error indicates vision/multimodal content is not supported */
+function isVisionNotSupportedError(error) {
+    const errorStr = String(error);
+    return (errorStr.includes('content.str') ||
+        errorStr.includes('should be a valid string') ||
+        errorStr.includes('Invalid content type') ||
+        errorStr.includes('does not support vision') ||
+        errorStr.includes('image_url') && errorStr.includes('not supported'));
+}
 class SpatialReasoningEngine {
     config;
     console;
@@ -36186,72 +36200,114 @@ class TopologyDiscoveryEngine {
             analysis.error = 'Failed to capture camera snapshot';
             return analysis;
         }
-        try {
-            // Build multimodal message with provider-specific image format
-            const result = await llm.getChatCompletion({
-                messages: [
-                    {
-                        role: 'user',
-                        content: [
-                            { type: 'text', text: SCENE_ANALYSIS_PROMPT },
-                            (0, spatial_reasoning_1.buildImageContent)(imageData, this.llmProviderType),
-                        ],
-                    },
-                ],
-                max_tokens: 500,
-                temperature: 0.3,
-            });
-            const content = result?.choices?.[0]?.message?.content;
-            if (content && typeof content === 'string') {
-                try {
-                    // Extract JSON from response (handle markdown code blocks)
-                    let jsonStr = content.trim();
-                    if (jsonStr.startsWith('```')) {
-                        jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
-                    }
-                    const parsed = JSON.parse(jsonStr);
-                    // Map parsed data to our types
-                    if (Array.isArray(parsed.landmarks)) {
-                        analysis.landmarks = parsed.landmarks.map((l) => ({
-                            name: l.name || 'Unknown',
-                            type: this.mapLandmarkType(l.type),
-                            confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
-                            description: l.description || '',
-                            boundingBox: l.boundingBox,
-                        }));
-                    }
-                    if (Array.isArray(parsed.zones)) {
-                        analysis.zones = parsed.zones.map((z) => ({
-                            name: z.name || 'Unknown',
-                            type: this.mapZoneType(z.type),
-                            coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
-                            description: z.description || '',
-                            boundingBox: z.boundingBox,
-                        }));
-                    }
-                    if (parsed.edges && typeof parsed.edges === 'object') {
-                        analysis.edges = {
-                            top: parsed.edges.top || '',
-                            left: parsed.edges.left || '',
-                            right: parsed.edges.right || '',
-                            bottom: parsed.edges.bottom || '',
-                        };
+        // Try with detected provider format first, then fallback to alternate format
+        const formatsToTry = [this.llmProviderType];
+        // Add fallback format
+        if (this.llmProviderType === 'openai') {
+            formatsToTry.push('anthropic');
+        }
+        else if (this.llmProviderType === 'anthropic') {
+            formatsToTry.push('openai');
+        }
+        else {
+            // Unknown - try both
+            formatsToTry.push('openai');
+        }
+        let lastError = null;
+        for (const formatType of formatsToTry) {
+            try {
+                this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
+                // Build multimodal message with provider-specific image format
+                const result = await llm.getChatCompletion({
+                    messages: [
+                        {
+                            role: 'user',
+                            content: [
+                                { type: 'text', text: SCENE_ANALYSIS_PROMPT },
+                                (0, spatial_reasoning_1.buildImageContent)(imageData, formatType),
+                            ],
+                        },
+                    ],
+                    max_tokens: 500,
+                    temperature: 0.3,
+                });
+                const content = result?.choices?.[0]?.message?.content;
+                if (content && typeof content === 'string') {
+                    try {
+                        // Extract JSON from response (handle markdown code blocks)
+                        let jsonStr = content.trim();
+                        if (jsonStr.startsWith('```')) {
+                            jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
+                        }
+                        const parsed = JSON.parse(jsonStr);
+                        // Map parsed data to our types
+                        if (Array.isArray(parsed.landmarks)) {
+                            analysis.landmarks = parsed.landmarks.map((l) => ({
+                                name: l.name || 'Unknown',
+                                type: this.mapLandmarkType(l.type),
+                                confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
+                                description: l.description || '',
+                                boundingBox: l.boundingBox,
+                            }));
+                        }
+                        if (Array.isArray(parsed.zones)) {
+                            analysis.zones = parsed.zones.map((z) => ({
+                                name: z.name || 'Unknown',
+                                type: this.mapZoneType(z.type),
+                                coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
+                                description: z.description || '',
+                                boundingBox: z.boundingBox,
+                            }));
+                        }
+                        if (parsed.edges && typeof parsed.edges === 'object') {
+                            analysis.edges = {
+                                top: parsed.edges.top || '',
+                                left: parsed.edges.left || '',
+                                right: parsed.edges.right || '',
+                                bottom: parsed.edges.bottom || '',
+                            };
+                        }
+                        if (parsed.orientation) {
+                            analysis.orientation = this.mapOrientation(parsed.orientation);
+                        }
+                        analysis.isValid = true;
+                        this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
+                        // Update the preferred format for future requests
+                        if (formatType !== this.llmProviderType) {
+                            this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
+                            this.llmProviderType = formatType;
+                        }
+                        // Success - exit the retry loop
+                        return analysis;
                     }
-                    if (parsed.orientation) {
-                        analysis.orientation = this.mapOrientation(parsed.orientation);
+                    catch (parseError) {
+                        this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
+                        analysis.error = 'Failed to parse LLM response';
+                        return analysis;
                     }
-                    analysis.isValid = true;
-                    this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones`);
                 }
-                catch (parseError) {
-                    this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
-                    analysis.error = 'Failed to parse LLM response';
+            }
+            catch (e) {
+                lastError = e;
+                // Check if this is a vision/multimodal format error
+                if ((0, spatial_reasoning_1.isVisionNotSupportedError)(e)) {
+                    this.console.warn(`[Discovery] ${formatType} format not supported, trying fallback...`);
+                    continue; // Try next format
                 }
+                // Not a format error - don't retry
+                this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
+                break;
             }
         }
-        catch (e) {
-            this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
-            analysis.error = `Analysis failed: ${e}`;
+        // All formats failed
+        if (lastError) {
+            const errorStr = String(lastError);
+            if ((0, spatial_reasoning_1.isVisionNotSupportedError)(lastError)) {
+                analysis.error = 'Vision/image analysis not supported by configured LLM. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured.';
+            }
+            else {
+                analysis.error = `Analysis failed: ${errorStr}`;
+            }
         }
         // Cache the analysis
         this.sceneCache.set(cameraId, analysis);