@blueharford/scrypted-spatial-awareness 0.6.24 → 0.6.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.24",
+  "version": "0.6.26",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -436,26 +436,32 @@ export class SpatialReasoningEngine {
   private llmProvider: string | null = null;
   private llmProviderType: LlmProvider = 'unknown';
 
-  /** Find or initialize LLM device - looks for ChatCompletion interface from @scrypted/llm plugin */
-  private async findLlmDevice(): Promise<ChatCompletionDevice | null> {
-    if (this.llmDevice) return this.llmDevice;
-    if (this.llmSearched) return null; // Already searched and found nothing
-
+  // Load balancing for multiple LLMs
+  private llmDevices: Array<{
+    device: ChatCompletionDevice;
+    id: string;
+    name: string;
+    providerType: LlmProvider;
+    lastUsed: number;
+    errorCount: number;
+  }> = [];
+  private llmIndex: number = 0;
+
+  /** Find ALL LLM devices for load balancing */
+  private async findAllLlmDevices(): Promise<void> {
+    if (this.llmSearched) return;
     this.llmSearched = true;
 
     try {
-      // Look for devices with ChatCompletion interface (the correct interface for @scrypted/llm)
       for (const id of Object.keys(systemManager.getSystemState())) {
         const device = systemManager.getDeviceById(id);
         if (!device) continue;
 
-        // Check if this device has ChatCompletion interface
-        // The @scrypted/llm plugin exposes ChatCompletion, not ObjectDetection
         if (device.interfaces?.includes('ChatCompletion')) {
           const deviceName = device.name?.toLowerCase() || '';
           const pluginId = (device as any).pluginId?.toLowerCase() || '';
 
-          // Identify the provider type for logging and image format selection
+          // Identify the provider type for image format selection
           let providerType = 'Unknown';
           let providerTypeEnum: LlmProvider = 'unknown';
 
@@ -467,38 +473,104 @@ export class SpatialReasoningEngine {
             providerTypeEnum = 'anthropic';
           } else if (deviceName.includes('ollama')) {
             providerType = 'Ollama';
-            providerTypeEnum = 'openai'; // Ollama uses OpenAI-compatible format
+            providerTypeEnum = 'openai';
           } else if (deviceName.includes('gemini') || deviceName.includes('google')) {
             providerType = 'Google';
-            providerTypeEnum = 'openai'; // Google uses OpenAI-compatible format
+            providerTypeEnum = 'openai';
           } else if (deviceName.includes('llama')) {
             providerType = 'llama.cpp';
-            providerTypeEnum = 'openai'; // llama.cpp uses OpenAI-compatible format
+            providerTypeEnum = 'openai';
           } else if (pluginId.includes('@scrypted/llm') || pluginId.includes('llm')) {
             providerType = 'Scrypted LLM';
             providerTypeEnum = 'unknown';
           }
 
-          this.llmDevice = device as unknown as ChatCompletionDevice;
-          this.llmProvider = `${providerType} (${device.name})`;
-          this.llmProviderType = providerTypeEnum;
-          this.console.log(`[LLM] Connected to ${providerType}: ${device.name}`);
-          this.console.log(`[LLM] Plugin: ${pluginId || 'N/A'}`);
-          this.console.log(`[LLM] Image format: ${providerTypeEnum}`);
-          this.console.log(`[LLM] Interfaces: ${device.interfaces?.join(', ')}`);
-          return this.llmDevice;
+          this.llmDevices.push({
+            device: device as unknown as ChatCompletionDevice,
+            id,
+            name: device.name || id,
+            providerType: providerTypeEnum,
+            lastUsed: 0,
+            errorCount: 0,
+          });
+
+          this.console.log(`[LLM] Found ${providerType}: ${device.name}`);
         }
       }
 
-      // If we get here, no LLM plugin found
-      this.console.warn('[LLM] No ChatCompletion device found. Install @scrypted/llm for enhanced descriptions.');
-      this.console.warn('[LLM] Falling back to rule-based descriptions using topology data.');
-
+      if (this.llmDevices.length === 0) {
+        this.console.warn('[LLM] No ChatCompletion devices found. Install @scrypted/llm for enhanced descriptions.');
+      } else {
+        this.console.log(`[LLM] Load balancing across ${this.llmDevices.length} LLM device(s)`);
+      }
     } catch (e) {
-      this.console.error('[LLM] Error searching for LLM device:', e);
+      this.console.error('[LLM] Error searching for LLM devices:', e);
     }
+  }
 
-    return null;
+  /** Get the next available LLM using round-robin with least-recently-used preference */
+  private async findLlmDevice(): Promise<ChatCompletionDevice | null> {
+    await this.findAllLlmDevices();
+
+    if (this.llmDevices.length === 0) return null;
+
+    // If only one LLM, just use it
+    if (this.llmDevices.length === 1) {
+      const llm = this.llmDevices[0];
+      this.llmDevice = llm.device;
+      this.llmProvider = llm.name;
+      this.llmProviderType = llm.providerType;
+      return llm.device;
+    }
+
+    // Find the LLM with the oldest lastUsed time (least recently used)
+    // Also prefer LLMs with fewer errors
+    let bestIndex = 0;
+    let bestScore = Infinity;
+
+    for (let i = 0; i < this.llmDevices.length; i++) {
+      const llm = this.llmDevices[i];
+      // Score = lastUsed time + (errorCount * 60 seconds penalty)
+      const score = llm.lastUsed + (llm.errorCount * 60000);
+      if (score < bestScore) {
+        bestScore = score;
+        bestIndex = i;
+      }
+    }
+
+    const selected = this.llmDevices[bestIndex];
+    this.llmDevice = selected.device;
+    this.llmProvider = selected.name;
+    this.llmProviderType = selected.providerType;
+
+    this.console.log(`[LLM] Selected: ${selected.name} (last used ${Math.round((Date.now() - selected.lastUsed) / 1000)}s ago, errors: ${selected.errorCount})`);
+
+    return selected.device;
+  }
+
+  /** Mark an LLM as used (for load balancing) */
+  private markLlmUsed(device: ChatCompletionDevice): void {
+    const llm = this.llmDevices.find(l => l.device === device);
+    if (llm) {
+      llm.lastUsed = Date.now();
+    }
+  }
+
+  /** Mark an LLM as having an error (for load balancing - will be deprioritized) */
+  private markLlmError(device: ChatCompletionDevice): void {
+    const llm = this.llmDevices.find(l => l.device === device);
+    if (llm) {
+      llm.errorCount++;
+      this.console.log(`[LLM] ${llm.name} error count: ${llm.errorCount}`);
+    }
+  }
+
+  /** Reset error count for an LLM after successful call */
+  private markLlmSuccess(device: ChatCompletionDevice): void {
+    const llm = this.llmDevices.find(l => l.device === device);
+    if (llm && llm.errorCount > 0) {
+      llm.errorCount = Math.max(0, llm.errorCount - 1); // Gradually reduce error count
+    }
   }
 
   /** Get the current LLM provider name */
@@ -946,6 +1018,9 @@ export class SpatialReasoningEngine {
         messageContent = prompt;
       }
 
+      // Mark LLM as used for load balancing
+      this.markLlmUsed(llm);
+
       // Call LLM using ChatCompletion interface
       const result = await llm.getChatCompletion({
         messages: [
@@ -961,12 +1036,14 @@ export class SpatialReasoningEngine {
       // Extract description from ChatCompletion result
       const content = result?.choices?.[0]?.message?.content;
       if (content && typeof content === 'string') {
+        this.markLlmSuccess(llm);
         return content.trim();
       }
 
       return null;
     } catch (e) {
       this.console.warn('LLM description generation failed:', e);
+      this.markLlmError(llm);
       return null;
     }
   }
@@ -1049,6 +1126,9 @@ Examples of good descriptions:
 
 Generate ONLY the description, nothing else:`;
 
+      // Mark LLM as used for load balancing
+      this.markLlmUsed(llm);
+
       // Try multimodal format first, fall back to text-only if it fails
       let result: any;
       let usedVision = false;
@@ -1101,6 +1181,7 @@ Generate ONLY the description, nothing else:`;
       const content = result?.choices?.[0]?.message?.content;
       if (content && typeof content === 'string') {
         this.console.log(`[LLM] Got ${eventType} description (vision=${usedVision}): ${content.trim().substring(0, 50)}...`);
+        this.markLlmSuccess(llm);
         return content.trim();
       }
 
@@ -1108,6 +1189,7 @@ Generate ONLY the description, nothing else:`;
       return null;
     } catch (e) {
       this.console.warn(`[LLM] ${eventType} description generation failed:`, e);
+      this.markLlmError(llm);
       return null;
     }
   }

package/src/core/topology-discovery.ts

@@ -254,11 +254,19 @@ export class TopologyDiscoveryEngine {
     return this.config.discoveryIntervalHours > 0;
   }
 
-  /** Find LLM device with ChatCompletion interface */
-  private async findLlmDevice(): Promise<ChatCompletionDevice | null> {
-    if (this.llmDevice) return this.llmDevice;
-    if (this.llmSearched) return null;
-
+  // Load balancing for multiple LLMs
+  private llmDevices: Array<{
+    device: ChatCompletionDevice;
+    id: string;
+    name: string;
+    providerType: LlmProvider;
+    lastUsed: number;
+    errorCount: number;
+  }> = [];
+
+  /** Find ALL LLM devices for load balancing */
+  private async findAllLlmDevices(): Promise<void> {
+    if (this.llmSearched) return;
     this.llmSearched = true;
 
     try {
@@ -269,32 +277,84 @@ export class TopologyDiscoveryEngine {
         if (device.interfaces?.includes('ChatCompletion')) {
           const deviceName = device.name?.toLowerCase() || '';
 
-          // Detect provider type for image format selection
+          let providerType: LlmProvider = 'unknown';
           if (deviceName.includes('openai') || deviceName.includes('gpt')) {
-            this.llmProviderType = 'openai';
+            providerType = 'openai';
           } else if (deviceName.includes('anthropic') || deviceName.includes('claude')) {
-            this.llmProviderType = 'anthropic';
+            providerType = 'anthropic';
           } else if (deviceName.includes('ollama') || deviceName.includes('gemini') ||
                      deviceName.includes('google') || deviceName.includes('llama')) {
-            // These providers use OpenAI-compatible format
-            this.llmProviderType = 'openai';
-          } else {
-            this.llmProviderType = 'unknown';
+            providerType = 'openai';
           }
 
-          this.llmDevice = device as unknown as ChatCompletionDevice;
-          this.console.log(`[Discovery] Connected to LLM: ${device.name}`);
-          this.console.log(`[Discovery] Image format: ${this.llmProviderType}`);
-          return this.llmDevice;
+          this.llmDevices.push({
+            device: device as unknown as ChatCompletionDevice,
+            id,
+            name: device.name || id,
+            providerType,
+            lastUsed: 0,
+            errorCount: 0,
+          });
+
+          this.console.log(`[Discovery] Found LLM: ${device.name}`);
         }
       }
 
-      this.console.warn('[Discovery] No ChatCompletion device found. Vision-based discovery unavailable.');
+      if (this.llmDevices.length === 0) {
+        this.console.warn('[Discovery] No ChatCompletion devices found. Vision-based discovery unavailable.');
+      } else {
+        this.console.log(`[Discovery] Load balancing across ${this.llmDevices.length} LLM device(s)`);
+      }
     } catch (e) {
-      this.console.error('[Discovery] Error finding LLM device:', e);
+      this.console.error('[Discovery] Error finding LLM devices:', e);
+    }
+  }
+
+  /** Find LLM device with ChatCompletion interface - uses load balancing */
+  private async findLlmDevice(): Promise<ChatCompletionDevice | null> {
+    await this.findAllLlmDevices();
+
+    if (this.llmDevices.length === 0) return null;
+
+    // If only one LLM, just use it
+    if (this.llmDevices.length === 1) {
+      const llm = this.llmDevices[0];
+      this.llmDevice = llm.device;
+      this.llmProviderType = llm.providerType;
+      return llm.device;
     }
 
-    return null;
+    // Find the LLM with oldest lastUsed time (least recently used)
+    let bestIndex = 0;
+    let bestScore = Infinity;
+
+    for (let i = 0; i < this.llmDevices.length; i++) {
+      const llm = this.llmDevices[i];
+      const score = llm.lastUsed + (llm.errorCount * 60000);
+      if (score < bestScore) {
+        bestScore = score;
+        bestIndex = i;
+      }
+    }
+
+    const selected = this.llmDevices[bestIndex];
+    this.llmDevice = selected.device;
+    this.llmProviderType = selected.providerType;
+
+    // Mark as used
+    selected.lastUsed = Date.now();
+
+    this.console.log(`[Discovery] Selected LLM: ${selected.name}`);
+    return selected.device;
+  }
+
+  /** Mark an LLM as having an error */
+  private markLlmError(device: ChatCompletionDevice): void {
+    const llm = this.llmDevices.find(l => l.device === device);
+    if (llm) {
+      llm.errorCount++;
+      this.console.log(`[Discovery] ${llm.name} error count: ${llm.errorCount}`);
+    }
   }
 
   /** Get camera snapshot as ImageData */
@@ -495,6 +555,11 @@ Use the mount height to help estimate distances - objects at ground level will a
 
     // All formats failed
     if (lastError) {
+      // Track error for load balancing
+      if (llm) {
+        this.markLlmError(llm);
+      }
+
       const errorStr = String(lastError);
       if (isVisionFormatError(lastError)) {
         analysis.error = 'Vision/image analysis failed with all formats. Ensure you have a vision-capable model (e.g., gpt-4o, gpt-4-turbo, claude-3-sonnet) configured and the @scrypted/llm plugin supports vision.';

package/src/core/tracking-engine.ts

@@ -324,6 +324,16 @@ export class TrackingEngine {
     this.lastLlmCallTime = Date.now();
   }
 
+  /** Check and record LLM call - returns false if rate limited */
+  private tryLlmCall(): boolean {
+    if (!this.isLlmCallAllowed()) {
+      this.console.log('[LLM] Rate limited, skipping LLM call');
+      return false;
+    }
+    this.recordLlmCall();
+    return true;
+  }
+
   /** Get spatial reasoning result for movement (uses RAG + LLM) with debouncing and fallback */
   private async getSpatialDescription(
     tracked: TrackedObject,
@@ -337,8 +347,8 @@ export class TrackingEngine {
 
     try {
       // Check rate limiting - if not allowed, return null to use basic description
-      if (!this.isLlmCallAllowed()) {
-        this.console.log('LLM rate-limited, using basic notification');
+      if (!this.tryLlmCall()) {
+        this.console.log('[Movement] LLM rate-limited, using basic notification');
         return null;
       }
 
@@ -351,9 +361,6 @@ export class TrackingEngine {
         }
       }
 
-      // Record that we're making an LLM call
-      this.recordLlmCall();
-
       // Use spatial reasoning engine for rich context-aware description
       // Apply timeout if fallback is enabled
       let result: SpatialReasoningResult;
@@ -567,16 +574,23 @@ export class TrackingEngine {
           spatialResult = await pendingDescription;
           this.console.log(`[Entry Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
         } catch (e) {
-          this.console.warn(`[Entry Alert] Prefetch failed, generating fallback: ${e}`);
+          this.console.warn(`[Entry Alert] Prefetch failed, using basic description: ${e}`);
+          // Don't make another LLM call - use basic description (no mediaObject = no LLM)
           spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId);
         }
         this.pendingDescriptions.delete(globalId);
       } else {
-        // Fallback: generate description now (slower path)
-        this.console.log(`[Entry Alert] No prefetch available, generating now`);
-        const mediaObject = this.snapshotCache.get(globalId);
-        spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
-        this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+        // No prefetch available - only call LLM if rate limit allows
+        if (this.tryLlmCall()) {
+          this.console.log(`[Entry Alert] No prefetch, generating with LLM`);
+          const mediaObject = this.snapshotCache.get(globalId);
+          spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
+          this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+        } else {
+          // Rate limited - use basic description (no LLM)
+          this.console.log(`[Entry Alert] Rate limited, using basic description`);
+          spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId);
+        }
       }
 
       // Always use movement alert type for smart notifications with LLM descriptions
@@ -611,9 +625,9 @@ export class TrackingEngine {
           this.snapshotCache.set(globalId, mediaObject);
           this.console.log(`[Snapshot] Cached snapshot for ${globalId.slice(0, 8)} from ${cameraId}`);
 
-          // Start LLM analysis immediately in parallel (don't await)
+          // Start LLM analysis immediately in parallel (don't await) - but respect rate limits
           const tracked = this.state.getObject(globalId);
-          if (tracked && this.config.useLlmDescriptions) {
+          if (tracked && this.config.useLlmDescriptions && this.tryLlmCall()) {
             this.console.log(`[LLM Prefetch] Starting ${eventType} analysis for ${globalId.slice(0, 8)}`);
             const descriptionPromise = eventType === 'exit'
               ? this.spatialReasoning.generateExitDescription(tracked, cameraId, mediaObject)
@@ -627,6 +641,8 @@ export class TrackingEngine {
             }).catch(e => {
               this.console.warn(`[LLM Prefetch] Failed for ${globalId.slice(0, 8)}: ${e}`);
             });
+          } else if (tracked && this.config.useLlmDescriptions) {
+            this.console.log(`[LLM Prefetch] Skipped for ${globalId.slice(0, 8)} - rate limited`);
           }
         }
       }
@@ -706,16 +722,23 @@ export class TrackingEngine {
             spatialResult = await pendingDescription;
             this.console.log(`[Exit Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
           } catch (e) {
-            this.console.warn(`[Exit Alert] Prefetch failed, generating fallback: ${e}`);
+            this.console.warn(`[Exit Alert] Prefetch failed, using basic description: ${e}`);
+            // Don't make another LLM call - use basic description
             spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId);
           }
           this.pendingDescriptions.delete(tracked.globalId);
         } else {
-          // Fallback: generate description now (slower path)
-          this.console.log(`[Exit Alert] No prefetch available, generating now`);
-          const mediaObject = this.snapshotCache.get(tracked.globalId);
-          spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
-          this.console.log(`[Exit Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+          // No prefetch available - only call LLM if rate limit allows
+          if (this.tryLlmCall()) {
+            this.console.log(`[Exit Alert] No prefetch, generating with LLM`);
+            const mediaObject = this.snapshotCache.get(tracked.globalId);
+            spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
+            this.console.log(`[Exit Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+          } else {
+            // Rate limited - use basic description (no LLM)
+            this.console.log(`[Exit Alert] Rate limited, using basic description`);
+            spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId);
+          }
         }
 
         // Use movement alert for exit too - smart notifications with LLM descriptions

package/src/main.ts

@@ -130,8 +130,8 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
     llmDebounceInterval: {
       title: 'LLM Rate Limit (seconds)',
       type: 'number',
-      defaultValue: 10,
-      description: 'Minimum time between LLM calls to prevent API overload (0 = no limit)',
+      defaultValue: 30,
+      description: 'Minimum time between LLM calls to prevent API rate limiting. Increase if you get rate limit errors. (0 = no limit)',
       group: 'AI & Spatial Reasoning',
     },
     llmFallbackEnabled: {