@blueharford/scrypted-spatial-awareness 0.6.24 → 0.6.26

package/out/main.nodejs.js

@@ -35407,25 +35407,23 @@ class SpatialReasoningEngine {
     llmSearched = false;
     llmProvider = null;
     llmProviderType = 'unknown';
-    /** Find or initialize LLM device - looks for ChatCompletion interface from @scrypted/llm plugin */
-    async findLlmDevice() {
-        if (this.llmDevice)
-            return this.llmDevice;
+    // Load balancing for multiple LLMs
+    llmDevices = [];
+    llmIndex = 0;
+    /** Find ALL LLM devices for load balancing */
+    async findAllLlmDevices() {
         if (this.llmSearched)
-            return null; // Already searched and found nothing
+            return;
         this.llmSearched = true;
         try {
-            // Look for devices with ChatCompletion interface (the correct interface for @scrypted/llm)
             for (const id of Object.keys(systemManager.getSystemState())) {
                 const device = systemManager.getDeviceById(id);
                 if (!device)
                     continue;
-                // Check if this device has ChatCompletion interface
-                // The @scrypted/llm plugin exposes ChatCompletion, not ObjectDetection
                 if (device.interfaces?.includes('ChatCompletion')) {
                     const deviceName = device.name?.toLowerCase() || '';
                     const pluginId = device.pluginId?.toLowerCase() || '';
-                    // Identify the provider type for logging and image format selection
+                    // Identify the provider type for image format selection
                     let providerType = 'Unknown';
                     let providerTypeEnum = 'unknown';
                     if (deviceName.includes('openai') || deviceName.includes('gpt')) {
@@ -35438,38 +35436,96 @@ class SpatialReasoningEngine {
                     }
                     else if (deviceName.includes('ollama')) {
                         providerType = 'Ollama';
-                        providerTypeEnum = 'openai'; // Ollama uses OpenAI-compatible format
+                        providerTypeEnum = 'openai';
                     }
                     else if (deviceName.includes('gemini') || deviceName.includes('google')) {
                         providerType = 'Google';
-                        providerTypeEnum = 'openai'; // Google uses OpenAI-compatible format
+                        providerTypeEnum = 'openai';
                     }
                     else if (deviceName.includes('llama')) {
                         providerType = 'llama.cpp';
-                        providerTypeEnum = 'openai'; // llama.cpp uses OpenAI-compatible format
+                        providerTypeEnum = 'openai';
                     }
                     else if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
                         providerType = 'Scrypted LLM';
                         providerTypeEnum = 'unknown';
                     }
-                    this.llmDevice = device;
-                    this.llmProvider = `${providerType} (${device.name})`;
-                    this.llmProviderType = providerTypeEnum;
-                    this.console.log(`[LLM] Connected to ${providerType}: ${device.name}`);
-                    this.console.log(`[LLM] Plugin: ${pluginId || 'N/A'}`);
-                    this.console.log(`[LLM] Image format: ${providerTypeEnum}`);
-                    this.console.log(`[LLM] Interfaces: ${device.interfaces?.join(', ')}`);
-                    return this.llmDevice;
+                    this.llmDevices.push({
+                        device: device,
+                        id,
+                        name: device.name || id,
+                        providerType: providerTypeEnum,
+                        lastUsed: 0,
+                        errorCount: 0,
+                    });
+                    this.console.log(`[LLM] Found ${providerType}: ${device.name}`);
                 }
             }
-            // If we get here, no LLM plugin found
-            this.console.warn('[LLM] No ChatCompletion device found. Install @scrypted/llm for enhanced descriptions.');
-            this.console.warn('[LLM] Falling back to rule-based descriptions using topology data.');
+            if (this.llmDevices.length === 0) {
+                this.console.warn('[LLM] No ChatCompletion devices found. Install @scrypted/llm for enhanced descriptions.');
+            }
+            else {
+                this.console.log(`[LLM] Load balancing across ${this.llmDevices.length} LLM device(s)`);
+            }
         }
         catch (e) {
-            this.console.error('[LLM] Error searching for LLM device:', e);
+            this.console.error('[LLM] Error searching for LLM devices:', e);
+        }
+    }
+    /** Get the next available LLM using round-robin with least-recently-used preference */
+    async findLlmDevice() {
+        await this.findAllLlmDevices();
+        if (this.llmDevices.length === 0)
+            return null;
+        // If only one LLM, just use it
+        if (this.llmDevices.length === 1) {
+            const llm = this.llmDevices[0];
+            this.llmDevice = llm.device;
+            this.llmProvider = llm.name;
+            this.llmProviderType = llm.providerType;
+            return llm.device;
+        }
+        // Find the LLM with the oldest lastUsed time (least recently used)
+        // Also prefer LLMs with fewer errors
+        let bestIndex = 0;
+        let bestScore = Infinity;
+        for (let i = 0; i < this.llmDevices.length; i++) {
+            const llm = this.llmDevices[i];
+            // Score = lastUsed time + (errorCount * 60 seconds penalty)
+            const score = llm.lastUsed + (llm.errorCount * 60000);
+            if (score < bestScore) {
+                bestScore = score;
+                bestIndex = i;
+            }
+        }
+        const selected = this.llmDevices[bestIndex];
+        this.llmDevice = selected.device;
+        this.llmProvider = selected.name;
+        this.llmProviderType = selected.providerType;
+        this.console.log(`[LLM] Selected: ${selected.name} (last used ${Math.round((Date.now() - selected.lastUsed) / 1000)}s ago, errors: ${selected.errorCount})`);
+        return selected.device;
+    }
+    /** Mark an LLM as used (for load balancing) */
+    markLlmUsed(device) {
+        const llm = this.llmDevices.find(l => l.device === device);
+        if (llm) {
+            llm.lastUsed = Date.now();
+        }
+    }
+    /** Mark an LLM as having an error (for load balancing - will be deprioritized) */
+    markLlmError(device) {
+        const llm = this.llmDevices.find(l => l.device === device);
+        if (llm) {
+            llm.errorCount++;
+            this.console.log(`[LLM] ${llm.name} error count: ${llm.errorCount}`);
+        }
+    }
+    /** Reset error count for an LLM after successful call */
+    markLlmSuccess(device) {
+        const llm = this.llmDevices.find(l => l.device === device);
+        if (llm && llm.errorCount > 0) {
+            llm.errorCount = Math.max(0, llm.errorCount - 1); // Gradually reduce error count
         }
-        return null;
     }
     /** Get the current LLM provider name */
     getLlmProvider() {
@@ -35804,6 +35860,8 @@ class SpatialReasoningEngine {
                 // Fallback to text-only if image conversion failed
                 messageContent = prompt;
             }
+            // Mark LLM as used for load balancing
+            this.markLlmUsed(llm);
             // Call LLM using ChatCompletion interface
             const result = await llm.getChatCompletion({
                 messages: [
@@ -35818,12 +35876,14 @@ class SpatialReasoningEngine {
             // Extract description from ChatCompletion result
             const content = result?.choices?.[0]?.message?.content;
             if (content && typeof content === 'string') {
+                this.markLlmSuccess(llm);
                 return content.trim();
             }
             return null;
         }
         catch (e) {
             this.console.warn('LLM description generation failed:', e);
+            this.markLlmError(llm);
             return null;
         }
     }
@@ -35892,6 +35952,8 @@ Examples of good descriptions:
 - "Landscaper with leaf blower heading to work truck"
 
 Generate ONLY the description, nothing else:`;
+            // Mark LLM as used for load balancing
+            this.markLlmUsed(llm);
             // Try multimodal format first, fall back to text-only if it fails
             let result;
             let usedVision = false;
@@ -35942,6 +36004,7 @@ Generate ONLY the description, nothing else:`;
             const content = result?.choices?.[0]?.message?.content;
             if (content && typeof content === 'string') {
                 this.console.log(`[LLM] Got ${eventType} description (vision=${usedVision}): ${content.trim().substring(0, 50)}...`);
+                this.markLlmSuccess(llm);
                 return content.trim();
             }
             this.console.warn(`[LLM] No content in response for ${eventType}`);
@@ -35949,6 +36012,7 @@ Generate ONLY the description, nothing else:`;
         }
         catch (e) {
             this.console.warn(`[LLM] ${eventType} description generation failed:`, e);
+            this.markLlmError(llm);
             return null;
         }
     }
@@ -36386,12 +36450,12 @@ class TopologyDiscoveryEngine {
     isEnabled() {
         return this.config.discoveryIntervalHours > 0;
     }
-    /** Find LLM device with ChatCompletion interface */
-    async findLlmDevice() {
-        if (this.llmDevice)
-            return this.llmDevice;
+    // Load balancing for multiple LLMs
+    llmDevices = [];
+    /** Find ALL LLM devices for load balancing */
+    async findAllLlmDevices() {
         if (this.llmSearched)
-            return null;
+            return;
         this.llmSearched = true;
         try {
             for (const id of Object.keys(systemManager.getSystemState())) {
@@ -36400,33 +36464,77 @@ class TopologyDiscoveryEngine {
                     continue;
                 if (device.interfaces?.includes('ChatCompletion')) {
                     const deviceName = device.name?.toLowerCase() || '';
-                    // Detect provider type for image format selection
+                    let providerType = 'unknown';
                     if (deviceName.includes('openai') || deviceName.includes('gpt')) {
-                        this.llmProviderType = 'openai';
+                        providerType = 'openai';
                     }
                     else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
-                        this.llmProviderType = 'anthropic';
+                        providerType = 'anthropic';
                     }
                     else if (deviceName.includes('ollama') || deviceName.includes('gemini') ||
                         deviceName.includes('google') || deviceName.includes('llama')) {
-                        // These providers use OpenAI-compatible format
-                        this.llmProviderType = 'openai';
+                        providerType = 'openai';
                     }
-                    else {
-                        this.llmProviderType = 'unknown';
-                    }
-                    this.llmDevice = device;
-                    this.console.log(`[Discovery] Connected to LLM: ${device.name}`);
-                    this.console.log(`[Discovery] Image format: ${this.llmProviderType}`);
-                    return this.llmDevice;
+                    this.llmDevices.push({
+                        device: device,
+                        id,
+                        name: device.name || id,
+                        providerType,
+                        lastUsed: 0,
+                        errorCount: 0,
+                    });
+                    this.console.log(`[Discovery] Found LLM: ${device.name}`);
                 }
             }
-            this.console.warn('[Discovery] No ChatCompletion device found. Vision-based discovery unavailable.');
+            if (this.llmDevices.length === 0) {
+                this.console.warn('[Discovery] No ChatCompletion devices found. Vision-based discovery unavailable.');
+            }
+            else {
+                this.console.log(`[Discovery] Load balancing across ${this.llmDevices.length} LLM device(s)`);
+            }
         }
         catch (e) {
-            this.console.error('[Discovery] Error finding LLM device:', e);
+            this.console.error('[Discovery] Error finding LLM devices:', e);
+        }
+    }
+    /** Find LLM device with ChatCompletion interface - uses load balancing */
+    async findLlmDevice() {
+        await this.findAllLlmDevices();
+        if (this.llmDevices.length === 0)
+            return null;
+        // If only one LLM, just use it
+        if (this.llmDevices.length === 1) {
+            const llm = this.llmDevices[0];
+            this.llmDevice = llm.device;
+            this.llmProviderType = llm.providerType;
+            return llm.device;
+        }
+        // Find the LLM with oldest lastUsed time (least recently used)
+        let bestIndex = 0;
+        let bestScore = Infinity;
+        for (let i = 0; i < this.llmDevices.length; i++) {
+            const llm = this.llmDevices[i];
+            const score = llm.lastUsed + (llm.errorCount * 60000);
+            if (score < bestScore) {
+                bestScore = score;
+                bestIndex = i;
+            }
+        }
+        const selected = this.llmDevices[bestIndex];
+        this.llmDevice = selected.device;
+        this.llmProviderType = selected.providerType;
+        // Mark as used
+        selected.lastUsed = Date.now();
+        this.console.log(`[Discovery] Selected LLM: ${selected.name}`);
+        return selected.device;
+    }
+    /** Mark an LLM as having an error */
+    markLlmError(device) {
+        const llm = this.llmDevices.find(l => l.device === device);
+        if (llm) {
+            llm.errorCount++;
+            this.console.log(`[Discovery] ${llm.name} error count: ${llm.errorCount}`);
         }
-        return null;
     }
     /** Get camera snapshot as ImageData */
     async getCameraSnapshot(cameraId) {
@@ -36602,6 +36710,10 @@ Use the mount height to help estimate distances - objects at ground level will a
         }
         // All formats failed
         if (lastError) {
+            // Track error for load balancing
+            if (llm) {
+                this.markLlmError(llm);
+            }
             const errorStr = String(lastError);
             if ((0, spatial_reasoning_1.isVisionFormatError)(lastError)) {
                 analysis.error = 'Vision/image analysis failed with all formats. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured and the @scrypted/llm plugin supports vision.';
@@ -37351,14 +37463,23 @@ class TrackingEngine {
     recordLlmCall() {
         this.lastLlmCallTime = Date.now();
     }
+    /** Check and record LLM call - returns false if rate limited */
+    tryLlmCall() {
+        if (!this.isLlmCallAllowed()) {
+            this.console.log('[LLM] Rate limited, skipping LLM call');
+            return false;
+        }
+        this.recordLlmCall();
+        return true;
+    }
     /** Get spatial reasoning result for movement (uses RAG + LLM) with debouncing and fallback */
     async getSpatialDescription(tracked, fromCameraId, toCameraId, transitTime, currentCameraId) {
         const fallbackEnabled = this.config.llmFallbackEnabled ?? true;
         const fallbackTimeout = this.config.llmFallbackTimeout ?? 3000;
         try {
             // Check rate limiting - if not allowed, return null to use basic description
-            if (!this.isLlmCallAllowed()) {
-                this.console.log('LLM rate-limited, using basic notification');
+            if (!this.tryLlmCall()) {
+                this.console.log('[Movement] LLM rate-limited, using basic notification');
                 return null;
             }
             // Get snapshot from camera for LLM analysis (if LLM is enabled)
@@ -37369,8 +37490,6 @@ class TrackingEngine {
                     mediaObject = await camera.takePicture();
                 }
             }
-            // Record that we're making an LLM call
-            this.recordLlmCall();
             // Use spatial reasoning engine for rich context-aware description
             // Apply timeout if fallback is enabled
             let result;
@@ -37524,17 +37643,25 @@ class TrackingEngine {
                     this.console.log(`[Entry Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
                 }
                 catch (e) {
-                    this.console.warn(`[Entry Alert] Prefetch failed, generating fallback: ${e}`);
+                    this.console.warn(`[Entry Alert] Prefetch failed, using basic description: ${e}`);
+                    // Don't make another LLM call - use basic description (no mediaObject = no LLM)
                     spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId);
                 }
                 this.pendingDescriptions.delete(globalId);
             }
             else {
-                // Fallback: generate description now (slower path)
-                this.console.log(`[Entry Alert] No prefetch available, generating now`);
-                const mediaObject = this.snapshotCache.get(globalId);
-                spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
-                this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+                // No prefetch available - only call LLM if rate limit allows
+                if (this.tryLlmCall()) {
+                    this.console.log(`[Entry Alert] No prefetch, generating with LLM`);
+                    const mediaObject = this.snapshotCache.get(globalId);
+                    spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
+                    this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+                }
+                else {
+                    // Rate limited - use basic description (no LLM)
+                    this.console.log(`[Entry Alert] Rate limited, using basic description`);
+                    spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId);
+                }
             }
             // Always use movement alert type for smart notifications with LLM descriptions
             // The property_entry/property_exit types are legacy and disabled by default
@@ -37561,9 +37688,9 @@ class TrackingEngine {
                 if (mediaObject) {
                     this.snapshotCache.set(globalId, mediaObject);
                     this.console.log(`[Snapshot] Cached snapshot for ${globalId.slice(0, 8)} from ${cameraId}`);
-                    // Start LLM analysis immediately in parallel (don't await)
+                    // Start LLM analysis immediately in parallel (don't await) - but respect rate limits
                     const tracked = this.state.getObject(globalId);
-                    if (tracked && this.config.useLlmDescriptions) {
+                    if (tracked && this.config.useLlmDescriptions && this.tryLlmCall()) {
                         this.console.log(`[LLM Prefetch] Starting ${eventType} analysis for ${globalId.slice(0, 8)}`);
                         const descriptionPromise = eventType === 'exit'
                             ? this.spatialReasoning.generateExitDescription(tracked, cameraId, mediaObject)
@@ -37576,6 +37703,9 @@ class TrackingEngine {
                             this.console.warn(`[LLM Prefetch] Failed for ${globalId.slice(0, 8)}: ${e}`);
                         });
                     }
+                    else if (tracked && this.config.useLlmDescriptions) {
+                        this.console.log(`[LLM Prefetch] Skipped for ${globalId.slice(0, 8)} - rate limited`);
+                    }
                 }
             }
         }
@@ -37644,17 +37774,25 @@ class TrackingEngine {
                         this.console.log(`[Exit Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
                     }
                     catch (e) {
-                        this.console.warn(`[Exit Alert] Prefetch failed, generating fallback: ${e}`);
+                        this.console.warn(`[Exit Alert] Prefetch failed, using basic description: ${e}`);
+                        // Don't make another LLM call - use basic description
                         spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId);
                     }
                     this.pendingDescriptions.delete(tracked.globalId);
                 }
                 else {
-                    // Fallback: generate description now (slower path)
-                    this.console.log(`[Exit Alert] No prefetch available, generating now`);
-                    const mediaObject = this.snapshotCache.get(tracked.globalId);
-                    spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
-                    this.console.log(`[Exit Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+                    // No prefetch available - only call LLM if rate limit allows
+                    if (this.tryLlmCall()) {
+                        this.console.log(`[Exit Alert] No prefetch, generating with LLM`);
+                        const mediaObject = this.snapshotCache.get(tracked.globalId);
+                        spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
+                        this.console.log(`[Exit Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+                    }
+                    else {
+                        // Rate limited - use basic description (no LLM)
+                        this.console.log(`[Exit Alert] Rate limited, using basic description`);
+                        spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId);
+                    }
                 }
                 // Use movement alert for exit too - smart notifications with LLM descriptions
                 await this.alertManager.checkAndAlert('movement', current, {
@@ -39197,8 +39335,8 @@ class SpatialAwarenessPlugin extends sdk_1.ScryptedDeviceBase {
         llmDebounceInterval: {
             title: 'LLM Rate Limit (seconds)',
             type: 'number',
-            defaultValue: 10,
-            description: 'Minimum time between LLM calls to prevent API overload (0 = no limit)',
+            defaultValue: 30,
+            description: 'Minimum time between LLM calls to prevent API rate limiting. Increase if you get rate limit errors. (0 = no limit)',
             group: 'AI & Spatial Reasoning',
         },
         llmFallbackEnabled: {