@blueharford/scrypted-spatial-awareness 0.6.33 → 0.6.34

package/out/main.nodejs.js

@@ -36597,6 +36597,12 @@ class TopologyDiscoveryEngine {
     getStatus() {
         return { ...this.status };
     }
+    /** Get list of LLMs excluded for lack of vision support */
+    getExcludedVisionLlmNames() {
+        return this.llmDevices
+            .filter(l => !l.visionCapable)
+            .map(l => l.name || l.id);
+    }
     /** Get pending suggestions */
     getPendingSuggestions() {
         return Array.from(this.suggestions.values())
@@ -36648,6 +36654,7 @@ class TopologyDiscoveryEngine {
                         providerType,
                         lastUsed: 0,
                         errorCount: 0,
+                        visionCapable: true,
                     });
                     this.console.log(`[Discovery] Found LLM: ${device.name}`);
                 }
@@ -36694,6 +36701,42 @@ class TopologyDiscoveryEngine {
         this.console.log(`[Discovery] Selected LLM: ${selected.name}`);
         return selected.device;
     }
+    /** Select an LLM device, excluding any IDs if provided */
+    async selectLlmDevice(excludeIds) {
+        await this.findAllLlmDevices();
+        if (this.llmDevices.length === 0)
+            return null;
+        let bestIndex = -1;
+        let bestScore = Infinity;
+        for (let i = 0; i < this.llmDevices.length; i++) {
+            const llm = this.llmDevices[i];
+            if (excludeIds.has(llm.id))
+                continue;
+            if (!llm.visionCapable)
+                continue;
+            const score = llm.lastUsed + (llm.errorCount * 60000);
+            if (score < bestScore) {
+                bestScore = score;
+                bestIndex = i;
+            }
+        }
+        if (bestIndex === -1)
+            return null;
+        const selected = this.llmDevices[bestIndex];
+        this.llmDevice = selected.device;
+        this.llmProviderType = selected.providerType;
+        selected.lastUsed = Date.now();
+        this.console.log(`[Discovery] Selected LLM: ${selected.name}`);
+        return selected.device;
+    }
+    isRetryableLlmError(error) {
+        const errorStr = String(error).toLowerCase();
+        return (errorStr.includes('404') ||
+            errorStr.includes('not found') ||
+            errorStr.includes('no such model') ||
+            errorStr.includes('model not found') ||
+            errorStr.includes('endpoint'));
+    }
     /** Mark an LLM as having an error */
     markLlmError(device) {
         const llm = this.llmDevices.find(l => l.device === device);
@@ -36743,45 +36786,51 @@ class TopologyDiscoveryEngine {
             potentialOverlaps: [],
             isValid: false,
         };
-        const llm = await this.findLlmDevice();
-        if (!llm?.getChatCompletion) {
-            analysis.error = 'No LLM device available';
-            return analysis;
-        }
         const imageData = await this.getCameraSnapshot(cameraId);
         if (!imageData) {
             analysis.error = 'Failed to capture camera snapshot';
             return analysis;
         }
-        // Try with detected provider format first, then fallback to alternates
-        // The order matters: try the most likely formats first
-        const formatsToTry = [];
-        // Start with detected format
-        formatsToTry.push(this.llmProviderType);
-        // Add fallbacks based on detected provider
-        if (this.llmProviderType === 'openai') {
-            formatsToTry.push('scrypted', 'anthropic');
-        }
-        else if (this.llmProviderType === 'anthropic') {
-            formatsToTry.push('scrypted', 'openai');
-        }
-        else if (this.llmProviderType === 'scrypted') {
-            formatsToTry.push('anthropic', 'openai');
-        }
-        else {
-            // Unknown - try all formats
-            formatsToTry.push('scrypted', 'anthropic', 'openai');
-        }
+        await this.findAllLlmDevices();
+        const excludeIds = new Set();
         let lastError = null;
-        for (const formatType of formatsToTry) {
-            try {
-                this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
-                // Build prompt with camera context (height)
-                const cameraNode = this.topology ? (0, topology_1.findCamera)(this.topology, cameraId) : null;
-                const mountHeight = cameraNode?.context?.mountHeight || 8;
-                const cameraRange = cameraNode?.fov?.range || 80;
-                // Add camera-specific context to the prompt
-                const contextPrefix = `CAMERA INFORMATION:
+        const maxAttempts = Math.max(1, this.llmDevices.length || 1);
+        for (let attempt = 0; attempt < maxAttempts; attempt++) {
+            const llm = await this.selectLlmDevice(excludeIds);
+            if (!llm?.getChatCompletion) {
+                analysis.error = 'No LLM device available';
+                return analysis;
+            }
+            let allFormatsVisionError = false;
+            // Try with detected provider format first, then fallback to alternates
+            // The order matters: try the most likely formats first
+            const formatsToTry = [];
+            // Start with detected format
+            formatsToTry.push(this.llmProviderType);
+            // Add fallbacks based on detected provider
+            if (this.llmProviderType === 'openai') {
+                formatsToTry.push('scrypted', 'anthropic');
+            }
+            else if (this.llmProviderType === 'anthropic') {
+                formatsToTry.push('scrypted', 'openai');
+            }
+            else if (this.llmProviderType === 'scrypted') {
+                formatsToTry.push('anthropic', 'openai');
+            }
+            else {
+                // Unknown - try all formats
+                formatsToTry.push('scrypted', 'anthropic', 'openai');
+            }
+            let visionFormatFailures = 0;
+            for (const formatType of formatsToTry) {
+                try {
+                    this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
+                    // Build prompt with camera context (height)
+                    const cameraNode = this.topology ? (0, topology_1.findCamera)(this.topology, cameraId) : null;
+                    const mountHeight = cameraNode?.context?.mountHeight || 8;
+                    const cameraRange = cameraNode?.fov?.range || 80;
+                    // Add camera-specific context to the prompt
+                    const contextPrefix = `CAMERA INFORMATION:
 - Camera Name: ${cameraName}
 - Mount Height: ${mountHeight} feet above ground
 - Approximate viewing range: ${cameraRange} feet
@@ -36789,96 +36838,122 @@ class TopologyDiscoveryEngine {
 Use the mount height to help estimate distances - objects at ground level will appear at different angles depending on distance from a camera mounted at ${mountHeight} feet.
 
 `;
-                // Build multimodal message with provider-specific image format
-                const result = await llm.getChatCompletion({
-                    messages: [
-                        {
-                            role: 'user',
-                            content: [
-                                { type: 'text', text: contextPrefix + SCENE_ANALYSIS_PROMPT },
-                                (0, spatial_reasoning_1.buildImageContent)(imageData, formatType),
-                            ],
-                        },
-                    ],
-                    max_tokens: 4000, // Increased for detailed scene analysis
-                    temperature: 0.3,
-                });
-                const content = result?.choices?.[0]?.message?.content;
-                if (content && typeof content === 'string') {
-                    try {
-                        // Extract JSON from response (handle markdown code blocks)
-                        let jsonStr = content.trim();
-                        if (jsonStr.startsWith('```')) {
-                            jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
-                        }
-                        // Try to recover truncated JSON
-                        const parsed = this.parseJsonWithRecovery(jsonStr, cameraName);
-                        // Map parsed data to our types
-                        if (Array.isArray(parsed.landmarks)) {
-                            analysis.landmarks = parsed.landmarks.map((l) => ({
-                                name: l.name || 'Unknown',
-                                type: this.mapLandmarkType(l.type),
-                                confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
-                                distance: this.mapDistance(l.distance),
-                                description: l.description || '',
-                                boundingBox: l.boundingBox,
-                            }));
-                        }
-                        if (Array.isArray(parsed.zones)) {
-                            analysis.zones = parsed.zones.map((z) => ({
-                                name: z.name || 'Unknown',
-                                type: this.mapZoneType(z.type),
-                                coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
-                                description: z.description || '',
-                                boundingBox: z.boundingBox,
-                                distance: this.mapDistance(z.distance), // Parse distance for zones too
-                            }));
-                        }
-                        if (parsed.edges && typeof parsed.edges === 'object') {
-                            analysis.edges = {
-                                top: parsed.edges.top || '',
-                                left: parsed.edges.left || '',
-                                right: parsed.edges.right || '',
-                                bottom: parsed.edges.bottom || '',
-                            };
-                        }
-                        if (parsed.orientation) {
-                            analysis.orientation = this.mapOrientation(parsed.orientation);
+                    // Build multimodal message with provider-specific image format
+                    const result = await llm.getChatCompletion({
+                        messages: [
+                            {
+                                role: 'user',
+                                content: [
+                                    { type: 'text', text: contextPrefix + SCENE_ANALYSIS_PROMPT },
+                                    (0, spatial_reasoning_1.buildImageContent)(imageData, formatType),
+                                ],
+                            },
+                        ],
+                        max_tokens: 4000, // Increased for detailed scene analysis
+                        temperature: 0.3,
+                    });
+                    const content = result?.choices?.[0]?.message?.content;
+                    if (content && typeof content === 'string') {
+                        try {
+                            // Extract JSON from response (handle markdown code blocks)
+                            let jsonStr = content.trim();
+                            if (jsonStr.startsWith('```')) {
+                                jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
+                            }
+                            // Try to recover truncated JSON
+                            const parsed = this.parseJsonWithRecovery(jsonStr, cameraName);
+                            // Map parsed data to our types
+                            if (Array.isArray(parsed.landmarks)) {
+                                analysis.landmarks = parsed.landmarks.map((l) => ({
+                                    name: l.name || 'Unknown',
+                                    type: this.mapLandmarkType(l.type),
+                                    confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
+                                    distance: this.mapDistance(l.distance),
+                                    description: l.description || '',
+                                    boundingBox: l.boundingBox,
+                                }));
+                            }
+                            if (Array.isArray(parsed.zones)) {
+                                analysis.zones = parsed.zones.map((z) => ({
+                                    name: z.name || 'Unknown',
+                                    type: this.mapZoneType(z.type),
+                                    coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
+                                    description: z.description || '',
+                                    boundingBox: z.boundingBox,
+                                    distance: this.mapDistance(z.distance), // Parse distance for zones too
+                                }));
+                            }
+                            if (parsed.edges && typeof parsed.edges === 'object') {
+                                analysis.edges = {
+                                    top: parsed.edges.top || '',
+                                    left: parsed.edges.left || '',
+                                    right: parsed.edges.right || '',
+                                    bottom: parsed.edges.bottom || '',
+                                };
+                            }
+                            if (parsed.orientation) {
+                                analysis.orientation = this.mapOrientation(parsed.orientation);
+                            }
+                            analysis.isValid = true;
+                            this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
+                            // Update the preferred format for future requests
+                            if (formatType !== this.llmProviderType) {
+                                this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
+                                this.llmProviderType = formatType;
+                            }
+                            // Success - exit the retry loop
+                            return analysis;
                         }
-                        analysis.isValid = true;
-                        this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
-                        // Update the preferred format for future requests
-                        if (formatType !== this.llmProviderType) {
-                            this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
-                            this.llmProviderType = formatType;
+                        catch (parseError) {
+                            this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
+                            analysis.error = 'Failed to parse LLM response';
+                            return analysis;
                         }
-                        // Success - exit the retry loop
-                        return analysis;
                     }
-                    catch (parseError) {
-                        this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
-                        analysis.error = 'Failed to parse LLM response';
-                        return analysis;
+                }
+                catch (e) {
+                    lastError = e;
+                    // Check if this is a vision/multimodal format error
+                    if ((0, spatial_reasoning_1.isVisionFormatError)(e)) {
+                        this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
+                        visionFormatFailures++;
+                        continue; // Try next format
                     }
+                    // Retry with a different LLM if error indicates bad endpoint/model
+                    if (this.isRetryableLlmError(e)) {
+                        this.console.warn(`[Discovery] LLM error for ${cameraName}, trying another provider...`);
+                        this.markLlmError(llm);
+                        const llmEntry = this.llmDevices.find(d => d.device === llm);
+                        if (llmEntry) {
+                            excludeIds.add(llmEntry.id);
+                        }
+                        break;
+                    }
+                    // Not a format error - don't retry
+                    this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
+                    break;
                 }
             }
-            catch (e) {
-                lastError = e;
-                // Check if this is a vision/multimodal format error
-                if ((0, spatial_reasoning_1.isVisionFormatError)(e)) {
-                    this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
-                    continue; // Try next format
-                }
-                // Not a format error - don't retry
-                this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
-                break;
+            allFormatsVisionError = visionFormatFailures > 0 && visionFormatFailures === formatsToTry.length;
+            if (allFormatsVisionError) {
+                const llmEntry = this.llmDevices.find(d => d.device === llm);
+                if (llmEntry) {
+                    llmEntry.visionCapable = false;
+                    excludeIds.add(llmEntry.id);
+                    this.console.warn(`[Discovery] ${llmEntry.name} does not support vision. Excluding from discovery.`);
+                }
             }
         }
         // All formats failed
         if (lastError) {
             // Track error for load balancing
-            if (llm) {
-                this.markLlmError(llm);
+            // Note: llm may be null here if no device was available
+            if (lastError && !this.isRetryableLlmError(lastError)) {
+                // Best-effort error accounting for the most recent device
+                const lastDevice = this.llmDevice;
+                if (lastDevice) {
+                    this.markLlmError(lastDevice);
+                }
             }
             const errorStr = String(lastError);
             if ((0, spatial_reasoning_1.isVisionFormatError)(lastError)) {
@@ -40166,6 +40241,19 @@ class SpatialAwarenessPlugin extends sdk_1.ScryptedDeviceBase {
         addGroup('AI & Spatial Reasoning');
         // ==================== 8. Auto-Topology Discovery ====================
         addGroup('Auto-Topology Discovery');
+        if (this.discoveryEngine) {
+            const excluded = this.discoveryEngine.getExcludedVisionLlmNames();
+            if (excluded.length > 0) {
+                settings.push({
+                    key: 'excludedVisionLlms',
+                    title: 'Excluded LLMs (No Vision)',
+                    type: 'string',
+                    readonly: true,
+                    value: excluded.join(', '),
+                    group: 'Auto-Topology Discovery',
+                });
+            }
+        }
         // ==================== 9. MQTT Integration ====================
         addGroup('MQTT Integration');
         return settings;
@@ -40179,6 +40267,7 @@ class SpatialAwarenessPlugin extends sdk_1.ScryptedDeviceBase {
             key === 'lostTimeout' ||
             key === 'useVisualMatching' ||
             key === 'loiteringThreshold' ||
+            key === 'minDetectionScore' ||
             key === 'objectAlertCooldown' ||
             key === 'useLlmDescriptions' ||
             key === 'llmDebounceInterval' ||
@@ -40291,7 +40380,7 @@ class SpatialAwarenessPlugin extends sdk_1.ScryptedDeviceBase {
             }
             // Training Mode endpoints
             if (path.endsWith('/api/training/start')) {
-                return this.handleTrainingStartRequest(request, response);
+                return await this.handleTrainingStartRequest(request, response);
             }
             if (path.endsWith('/api/training/pause')) {
                 return this.handleTrainingPauseRequest(response);
@@ -40861,13 +40950,23 @@ class SpatialAwarenessPlugin extends sdk_1.ScryptedDeviceBase {
         }
     }
     // ==================== Training Mode Handlers ====================
-    handleTrainingStartRequest(request, response) {
+    async handleTrainingStartRequest(request, response) {
         if (!this.trackingEngine) {
-            response.send(JSON.stringify({ error: 'Tracking engine not running. Configure topology first.' }), {
-                code: 500,
-                headers: { 'Content-Type': 'application/json' },
-            });
-            return;
+            const topologyJson = this.storage.getItem('topology');
+            const topology = topologyJson ? JSON.parse(topologyJson) : (0, topology_1.createEmptyTopology)();
+            if (!topology.cameras?.length) {
+                const cameras = this.buildTopologyCamerasFromSettings();
+                if (cameras.length === 0) {
+                    response.send(JSON.stringify({ error: 'No cameras configured. Select tracked cameras first.' }), {
+                        code: 400,
+                        headers: { 'Content-Type': 'application/json' },
+                    });
+                    return;
+                }
+                topology.cameras = cameras;
+                this.storage.setItem('topology', JSON.stringify(topology));
+            }
+            await this.startTrackingEngine(topology);
         }
         try {
             let config;
@@ -41576,6 +41675,25 @@ Access the visual topology editor at \`/ui/editor\` to configure camera relation
         const topologyJson = this.storage.getItem('topology');
         return topologyJson ? JSON.parse(topologyJson) : null;
     }
+    buildTopologyCamerasFromSettings() {
+        const value = this.storageSettings.values.trackedCameras;
+        const cameraIds = Array.isArray(value)
+            ? value.filter(Boolean)
+            : typeof value === 'string' && value.length
+                ? [value]
+                : [];
+        return cameraIds.map((deviceId) => {
+            const device = systemManager.getDeviceById(deviceId);
+            return {
+                deviceId,
+                nativeId: device?.nativeId || deviceId,
+                name: device?.name || deviceId,
+                isEntryPoint: false,
+                isExitPoint: false,
+                trackClasses: [],
+            };
+        });
+    }
 }
 exports.SpatialAwarenessPlugin = SpatialAwarenessPlugin;
 exports["default"] = SpatialAwarenessPlugin;