@blueharford/scrypted-spatial-awareness 0.5.1 → 0.5.3

package/out/main.nodejs.js

@@ -35094,29 +35094,68 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", ({ value: true }));
 exports.SpatialReasoningEngine = void 0;
 exports.mediaObjectToBase64 = mediaObjectToBase64;
+exports.buildImageContent = buildImageContent;
 const sdk_1 = __importStar(__webpack_require__(/*! @scrypted/sdk */ "./node_modules/@scrypted/sdk/dist/src/index.js"));
 const topology_1 = __webpack_require__(/*! ../models/topology */ "./src/models/topology.ts");
 const { systemManager, mediaManager } = sdk_1.default;
 /**
- * Convert a MediaObject to a base64 data URL for vision LLM consumption
+ * Convert a MediaObject to base64 image data for vision LLM consumption
  * @param mediaObject - MediaObject from camera.takePicture()
- * @returns Base64 data URL (data:image/jpeg;base64,...) or null if conversion fails
+ * @returns ImageData with raw base64 and media type, or null if conversion fails
  */
 async function mediaObjectToBase64(mediaObject) {
     try {
         // Convert MediaObject to Buffer using mediaManager
         const buffer = await mediaManager.convertMediaObjectToBuffer(mediaObject, sdk_1.ScryptedMimeTypes.Image);
-        // Convert buffer to base64
+        // Convert buffer to base64 (raw, no data URL prefix)
         const base64 = buffer.toString('base64');
         // Determine MIME type - default to JPEG for camera images
-        const mimeType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
-        return `data:${mimeType};base64,${base64}`;
+        const mediaType = mediaObject.mimeType?.split(';')[0] || 'image/jpeg';
+        return { base64, mediaType };
     }
     catch (e) {
         console.warn('Failed to convert MediaObject to base64:', e);
         return null;
     }
 }
+/**
+ * Build image content block for ChatCompletion API
+ * Supports both OpenAI and Anthropic formats
+ * @param imageData - Image data with base64 and media type
+ * @param provider - The LLM provider type (openai, anthropic, or unknown)
+ */
+function buildImageContent(imageData, provider = 'unknown') {
+    if (provider === 'openai') {
+        // OpenAI format: uses data URL with image_url wrapper
+        return {
+            type: 'image_url',
+            image_url: {
+                url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+            },
+        };
+    }
+    else if (provider === 'anthropic') {
+        // Anthropic format: uses separate base64 data and media_type
+        return {
+            type: 'image',
+            source: {
+                type: 'base64',
+                media_type: imageData.mediaType,
+                data: imageData.base64,
+            },
+        };
+    }
+    else {
+        // Unknown provider: try OpenAI format as it's more commonly supported
+        // Most LLM wrappers (including @scrypted/llm) understand the OpenAI format
+        return {
+            type: 'image_url',
+            image_url: {
+                url: `data:${imageData.mediaType};base64,${imageData.base64}`,
+            },
+        };
+    }
+}
 class SpatialReasoningEngine {
     config;
     console;
@@ -35336,6 +35375,7 @@ class SpatialReasoningEngine {
     }
     llmSearched = false;
     llmProvider = null;
+    llmProviderType = 'unknown';
     /** Find or initialize LLM device - looks for ChatCompletion interface from @scrypted/llm plugin */
     async findLlmDevice() {
         if (this.llmDevice)
@@ -35354,30 +35394,39 @@ class SpatialReasoningEngine {
                 if (device.interfaces?.includes('ChatCompletion')) {
                     const deviceName = device.name?.toLowerCase() || '';
                     const pluginId = device.pluginId?.toLowerCase() || '';
-                    // Identify the provider type for logging
+                    // Identify the provider type for logging and image format selection
                     let providerType = 'Unknown';
-                    if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
-                        providerType = 'Scrypted LLM';
-                    }
+                    let providerTypeEnum = 'unknown';
                     if (deviceName.includes('openai') || deviceName.includes('gpt')) {
                         providerType = 'OpenAI';
+                        providerTypeEnum = 'openai';
                     }
                     else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
                         providerType = 'Anthropic';
+                        providerTypeEnum = 'anthropic';
                     }
                     else if (deviceName.includes('ollama')) {
                         providerType = 'Ollama';
+                        providerTypeEnum = 'openai'; // Ollama uses OpenAI-compatible format
                     }
                     else if (deviceName.includes('gemini') || deviceName.includes('google')) {
                         providerType = 'Google';
+                        providerTypeEnum = 'openai'; // Google uses OpenAI-compatible format
                     }
                     else if (deviceName.includes('llama')) {
                         providerType = 'llama.cpp';
+                        providerTypeEnum = 'openai'; // llama.cpp uses OpenAI-compatible format
+                    }
+                    else if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
+                        providerType = 'Scrypted LLM';
+                        providerTypeEnum = 'unknown';
                     }
                     this.llmDevice = device;
                     this.llmProvider = `${providerType} (${device.name})`;
+                    this.llmProviderType = providerTypeEnum;
                     this.console.log(`[LLM] Connected to ${providerType}: ${device.name}`);
                     this.console.log(`[LLM] Plugin: ${pluginId || 'N/A'}`);
+                    this.console.log(`[LLM] Image format: ${providerTypeEnum}`);
                     this.console.log(`[LLM] Interfaces: ${device.interfaces?.join(', ')}`);
                     return this.llmDevice;
                 }
@@ -35395,6 +35444,10 @@ class SpatialReasoningEngine {
     getLlmProvider() {
         return this.llmProvider;
     }
+    /** Get the current LLM provider type for image format selection */
+    getLlmProviderType() {
+        return this.llmProviderType;
+    }
     /** Check if LLM is available */
     isLlmAvailable() {
         return this.llmDevice !== null;
@@ -35661,7 +35714,7 @@ class SpatialReasoningEngine {
             return null;
         try {
             // Convert image to base64 for vision LLM
-            const imageBase64 = await mediaObjectToBase64(mediaObject);
+            const imageData = await mediaObjectToBase64(mediaObject);
             // Retrieve relevant context for RAG
             const relevantChunks = this.retrieveRelevantContext(fromCamera.deviceId, toCamera.deviceId);
             // Build RAG context
@@ -35670,11 +35723,11 @@ class SpatialReasoningEngine {
             const prompt = this.buildLlmPrompt(tracked, fromCamera, toCamera, transitTime, fromLandmarks, toLandmarks, ragContext);
             // Build message content - use multimodal format if we have an image
             let messageContent;
-            if (imageBase64) {
-                // Vision-capable multimodal message format (OpenAI compatible)
+            if (imageData) {
+                // Vision-capable multimodal message format (provider-specific)
                 messageContent = [
                     { type: 'text', text: prompt },
-                    { type: 'image_url', image_url: { url: imageBase64 } },
+                    buildImageContent(imageData, this.llmProviderType),
                 ];
             }
             else {
@@ -35741,7 +35794,7 @@ Generate ONLY the description, nothing else:`;
             return null;
         try {
             // Convert image to base64 for vision LLM
-            const imageBase64 = await mediaObjectToBase64(mediaObject);
+            const imageData = await mediaObjectToBase64(mediaObject);
             const prompt = `Analyze this security camera image. A ${objectClass} was detected.
 
 Looking at the surroundings and environment, identify any notable landmarks or features visible that could help describe this location. Consider:
@@ -35756,11 +35809,11 @@ If you can identify a clear landmark feature, respond with ONLY a JSON object:
 If no clear landmark is identifiable, respond with: {"name": null}`;
             // Build message content - use multimodal format if we have an image
             let messageContent;
-            if (imageBase64) {
-                // Vision-capable multimodal message format (OpenAI compatible)
+            if (imageData) {
+                // Vision-capable multimodal message format (provider-specific)
                 messageContent = [
                     { type: 'text', text: prompt },
-                    { type: 'image_url', image_url: { url: imageBase64 } },
+                    buildImageContent(imageData, this.llmProviderType),
                 ];
             }
             else {
@@ -36000,6 +36053,7 @@ class TopologyDiscoveryEngine {
     topology = null;
     llmDevice = null;
     llmSearched = false;
+    llmProviderType = 'unknown';
     // Scene analysis cache (camera ID -> analysis)
     sceneCache = new Map();
     // Pending suggestions for user review
@@ -36063,8 +36117,25 @@ class TopologyDiscoveryEngine {
                 if (!device)
                     continue;
                 if (device.interfaces?.includes('ChatCompletion')) {
+                    const deviceName = device.name?.toLowerCase() || '';
+                    // Detect provider type for image format selection
+                    if (deviceName.includes('openai') || deviceName.includes('gpt')) {
+                        this.llmProviderType = 'openai';
+                    }
+                    else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
+                        this.llmProviderType = 'anthropic';
+                    }
+                    else if (deviceName.includes('ollama') || deviceName.includes('gemini') ||
+                        deviceName.includes('google') || deviceName.includes('llama')) {
+                        // These providers use OpenAI-compatible format
+                        this.llmProviderType = 'openai';
+                    }
+                    else {
+                        this.llmProviderType = 'unknown';
+                    }
                     this.llmDevice = device;
                     this.console.log(`[Discovery] Connected to LLM: ${device.name}`);
+                    this.console.log(`[Discovery] Image format: ${this.llmProviderType}`);
                     return this.llmDevice;
                 }
             }
@@ -36075,7 +36146,7 @@ class TopologyDiscoveryEngine {
         }
         return null;
     }
-    /** Get camera snapshot as base64 */
+    /** Get camera snapshot as ImageData */
     async getCameraSnapshot(cameraId) {
         try {
             const camera = systemManager.getDeviceById(cameraId);
@@ -36110,20 +36181,20 @@ class TopologyDiscoveryEngine {
             analysis.error = 'No LLM device available';
             return analysis;
         }
-        const imageBase64 = await this.getCameraSnapshot(cameraId);
-        if (!imageBase64) {
+        const imageData = await this.getCameraSnapshot(cameraId);
+        if (!imageData) {
             analysis.error = 'Failed to capture camera snapshot';
             return analysis;
         }
         try {
-            // Build multimodal message
+            // Build multimodal message with provider-specific image format
             const result = await llm.getChatCompletion({
                 messages: [
                     {
                         role: 'user',
                         content: [
                             { type: 'text', text: SCENE_ANALYSIS_PROMPT },
-                            { type: 'image_url', image_url: { url: imageBase64 } },
+                            (0, spatial_reasoning_1.buildImageContent)(imageData, this.llmProviderType),
                         ],
                     },
                 ],
@@ -36258,6 +36329,13 @@ class TopologyDiscoveryEngine {
             }
             this.status.camerasAnalyzed = analyses.length;
             this.console.log(`[Discovery] Analyzed ${analyses.length} cameras successfully`);
+            // Handle case where no cameras were successfully analyzed
+            if (analyses.length === 0) {
+                this.console.warn('[Discovery] No cameras were successfully analyzed');
+                this.status.lastError = 'No cameras were successfully analyzed - check LLM configuration';
+                this.status.lastScanTime = Date.now();
+                return null;
+            }
             // Correlate if we have multiple cameras
             let correlation = null;
             if (analyses.length >= 2) {
@@ -36266,7 +36344,7 @@ class TopologyDiscoveryEngine {
                     this.generateSuggestionsFromCorrelation(correlation);
                 }
             }
-            else {
+            else if (analyses.length === 1) {
                 // Single camera - generate suggestions from its analysis
                 this.generateSuggestionsFromAnalysis(analyses[0]);
             }