@blueharford/scrypted-spatial-awareness 0.6.33 → 0.6.34

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.33",
+  "version": "0.6.34",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/topology-discovery.ts

@@ -231,6 +231,13 @@ export class TopologyDiscoveryEngine {
     return { ...this.status };
   }
 
+  /** Get list of LLMs excluded for lack of vision support */
+  getExcludedVisionLlmNames(): string[] {
+    return this.llmDevices
+      .filter(l => !l.visionCapable)
+      .map(l => l.name || l.id);
+  }
+
   /** Get pending suggestions */
   getPendingSuggestions(): DiscoverySuggestion[] {
     return Array.from(this.suggestions.values())
@@ -262,6 +269,7 @@ export class TopologyDiscoveryEngine {
     providerType: LlmProvider;
     lastUsed: number;
     errorCount: number;
+    visionCapable: boolean;
   }> = [];
 
   /** Find ALL LLM devices for load balancing */
@@ -294,6 +302,7 @@ export class TopologyDiscoveryEngine {
             providerType,
             lastUsed: 0,
             errorCount: 0,
+            visionCapable: true,
           });
 
           this.console.log(`[Discovery] Found LLM: ${device.name}`);
@@ -348,6 +357,48 @@ export class TopologyDiscoveryEngine {
     return selected.device;
   }
 
+  /** Select an LLM device, excluding any IDs if provided */
+  private async selectLlmDevice(excludeIds: Set<string>): Promise<ChatCompletionDevice | null> {
+    await this.findAllLlmDevices();
+
+    if (this.llmDevices.length === 0) return null;
+
+    let bestIndex = -1;
+    let bestScore = Infinity;
+
+    for (let i = 0; i < this.llmDevices.length; i++) {
+      const llm = this.llmDevices[i];
+      if (excludeIds.has(llm.id)) continue;
+      if (!llm.visionCapable) continue;
+      const score = llm.lastUsed + (llm.errorCount * 60000);
+      if (score < bestScore) {
+        bestScore = score;
+        bestIndex = i;
+      }
+    }
+
+    if (bestIndex === -1) return null;
+
+    const selected = this.llmDevices[bestIndex];
+    this.llmDevice = selected.device;
+    this.llmProviderType = selected.providerType;
+    selected.lastUsed = Date.now();
+
+    this.console.log(`[Discovery] Selected LLM: ${selected.name}`);
+    return selected.device;
+  }
+
+  private isRetryableLlmError(error: any): boolean {
+    const errorStr = String(error).toLowerCase();
+    return (
+      errorStr.includes('404') ||
+      errorStr.includes('not found') ||
+      errorStr.includes('no such model') ||
+      errorStr.includes('model not found') ||
+      errorStr.includes('endpoint')
+    );
+  }
+
   /** Mark an LLM as having an error */
   private markLlmError(device: ChatCompletionDevice): void {
     const llm = this.llmDevices.find(l => l.device === device);
@@ -406,42 +457,49 @@ export class TopologyDiscoveryEngine {
       isValid: false,
     };
 
-    const llm = await this.findLlmDevice();
-    if (!llm?.getChatCompletion) {
-      analysis.error = 'No LLM device available';
-      return analysis;
-    }
-
     const imageData = await this.getCameraSnapshot(cameraId);
     if (!imageData) {
       analysis.error = 'Failed to capture camera snapshot';
       return analysis;
     }
 
-    // Try with detected provider format first, then fallback to alternates
-    // The order matters: try the most likely formats first
-    const formatsToTry: LlmProvider[] = [];
-
-    // Start with detected format
-    formatsToTry.push(this.llmProviderType);
-
-    // Add fallbacks based on detected provider
-    if (this.llmProviderType === 'openai') {
-      formatsToTry.push('scrypted', 'anthropic');
-    } else if (this.llmProviderType === 'anthropic') {
-      formatsToTry.push('scrypted', 'openai');
-    } else if (this.llmProviderType === 'scrypted') {
-      formatsToTry.push('anthropic', 'openai');
-    } else {
-      // Unknown - try all formats
-      formatsToTry.push('scrypted', 'anthropic', 'openai');
-    }
-
+    await this.findAllLlmDevices();
+    const excludeIds = new Set<string>();
     let lastError: any = null;
+    const maxAttempts = Math.max(1, this.llmDevices.length || 1);
 
-    for (const formatType of formatsToTry) {
-      try {
-        this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      const llm = await this.selectLlmDevice(excludeIds);
+      if (!llm?.getChatCompletion) {
+        analysis.error = 'No LLM device available';
+        return analysis;
+      }
+
+      let allFormatsVisionError = false;
+
+      // Try with detected provider format first, then fallback to alternates
+      // The order matters: try the most likely formats first
+      const formatsToTry: LlmProvider[] = [];
+
+      // Start with detected format
+      formatsToTry.push(this.llmProviderType);
+
+      // Add fallbacks based on detected provider
+      if (this.llmProviderType === 'openai') {
+        formatsToTry.push('scrypted', 'anthropic');
+      } else if (this.llmProviderType === 'anthropic') {
+        formatsToTry.push('scrypted', 'openai');
+      } else if (this.llmProviderType === 'scrypted') {
+        formatsToTry.push('anthropic', 'openai');
+      } else {
+        // Unknown - try all formats
+        formatsToTry.push('scrypted', 'anthropic', 'openai');
+      }
+
+      let visionFormatFailures = 0;
+      for (const formatType of formatsToTry) {
+        try {
+          this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
 
         // Build prompt with camera context (height)
         const cameraNode = this.topology ? findCamera(this.topology, cameraId) : null;
@@ -459,7 +517,7 @@ Use the mount height to help estimate distances - objects at ground level will a
 `;
 
         // Build multimodal message with provider-specific image format
-        const result = await llm.getChatCompletion({
+          const result = await llm.getChatCompletion({
           messages: [
             {
               role: 'user',
@@ -473,91 +531,119 @@ Use the mount height to help estimate distances - objects at ground level will a
           temperature: 0.3,
         });
 
-        const content = result?.choices?.[0]?.message?.content;
-        if (content && typeof content === 'string') {
-          try {
-            // Extract JSON from response (handle markdown code blocks)
-            let jsonStr = content.trim();
-            if (jsonStr.startsWith('```')) {
-              jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
-            }
+          const content = result?.choices?.[0]?.message?.content;
+          if (content && typeof content === 'string') {
+            try {
+              // Extract JSON from response (handle markdown code blocks)
+              let jsonStr = content.trim();
+              if (jsonStr.startsWith('```')) {
+                jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
+              }
 
-            // Try to recover truncated JSON
-            const parsed = this.parseJsonWithRecovery(jsonStr, cameraName);
+              // Try to recover truncated JSON
+              const parsed = this.parseJsonWithRecovery(jsonStr, cameraName);
+
+              // Map parsed data to our types
+              if (Array.isArray(parsed.landmarks)) {
+                analysis.landmarks = parsed.landmarks.map((l: any) => ({
+                  name: l.name || 'Unknown',
+                  type: this.mapLandmarkType(l.type),
+                  confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
+                  distance: this.mapDistance(l.distance),
+                  description: l.description || '',
+                  boundingBox: l.boundingBox,
+                }));
+              }
 
-            // Map parsed data to our types
-            if (Array.isArray(parsed.landmarks)) {
-              analysis.landmarks = parsed.landmarks.map((l: any) => ({
-                name: l.name || 'Unknown',
-                type: this.mapLandmarkType(l.type),
-                confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
-                distance: this.mapDistance(l.distance),
-                description: l.description || '',
-                boundingBox: l.boundingBox,
-              }));
-            }
+              if (Array.isArray(parsed.zones)) {
+                analysis.zones = parsed.zones.map((z: any) => ({
+                  name: z.name || 'Unknown',
+                  type: this.mapZoneType(z.type),
+                  coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
+                  description: z.description || '',
+                  boundingBox: z.boundingBox,
+                  distance: this.mapDistance(z.distance), // Parse distance for zones too
+                } as DiscoveredZone & { distance?: DistanceEstimate }));
+              }
 
-            if (Array.isArray(parsed.zones)) {
-              analysis.zones = parsed.zones.map((z: any) => ({
-                name: z.name || 'Unknown',
-                type: this.mapZoneType(z.type),
-                coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
-                description: z.description || '',
-                boundingBox: z.boundingBox,
-                distance: this.mapDistance(z.distance), // Parse distance for zones too
-              } as DiscoveredZone & { distance?: DistanceEstimate }));
-            }
+              if (parsed.edges && typeof parsed.edges === 'object') {
+                analysis.edges = {
+                  top: parsed.edges.top || '',
+                  left: parsed.edges.left || '',
+                  right: parsed.edges.right || '',
+                  bottom: parsed.edges.bottom || '',
+                };
+              }
 
-            if (parsed.edges && typeof parsed.edges === 'object') {
-              analysis.edges = {
-                top: parsed.edges.top || '',
-                left: parsed.edges.left || '',
-                right: parsed.edges.right || '',
-                bottom: parsed.edges.bottom || '',
-              };
-            }
+              if (parsed.orientation) {
+                analysis.orientation = this.mapOrientation(parsed.orientation);
+              }
 
-            if (parsed.orientation) {
-              analysis.orientation = this.mapOrientation(parsed.orientation);
-            }
+              analysis.isValid = true;
+              this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
 
-            analysis.isValid = true;
-            this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
+              // Update the preferred format for future requests
+              if (formatType !== this.llmProviderType) {
+                this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
+                this.llmProviderType = formatType;
+              }
 
-            // Update the preferred format for future requests
-            if (formatType !== this.llmProviderType) {
-              this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
-              this.llmProviderType = formatType;
+              // Success - exit the retry loop
+              return analysis;
+            } catch (parseError) {
+              this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
+              analysis.error = 'Failed to parse LLM response';
+              return analysis;
             }
+          }
+        } catch (e) {
+          lastError = e;
+
+          // Check if this is a vision/multimodal format error
+          if (isVisionFormatError(e)) {
+            this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
+            visionFormatFailures++;
+            continue; // Try next format
+          }
 
-            // Success - exit the retry loop
-            return analysis;
-          } catch (parseError) {
-            this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
-            analysis.error = 'Failed to parse LLM response';
-            return analysis;
+          // Retry with a different LLM if error indicates bad endpoint/model
+          if (this.isRetryableLlmError(e)) {
+            this.console.warn(`[Discovery] LLM error for ${cameraName}, trying another provider...`);
+            this.markLlmError(llm);
+            const llmEntry = this.llmDevices.find(d => d.device === llm);
+            if (llmEntry) {
+              excludeIds.add(llmEntry.id);
+            }
+            break;
           }
-        }
-      } catch (e) {
-        lastError = e;
 
-        // Check if this is a vision/multimodal format error
-        if (isVisionFormatError(e)) {
-          this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
-          continue; // Try next format
+          // Not a format error - don't retry
+          this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
+          break;
         }
+      }
 
-        // Not a format error - don't retry
-        this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
-        break;
+      allFormatsVisionError = visionFormatFailures > 0 && visionFormatFailures === formatsToTry.length;
+      if (allFormatsVisionError) {
+        const llmEntry = this.llmDevices.find(d => d.device === llm);
+        if (llmEntry) {
+          llmEntry.visionCapable = false;
+          excludeIds.add(llmEntry.id);
+          this.console.warn(`[Discovery] ${llmEntry.name} does not support vision. Excluding from discovery.`);
+        }
       }
     }
 
     // All formats failed
     if (lastError) {
       // Track error for load balancing
-      if (llm) {
-        this.markLlmError(llm);
+      // Note: llm may be null here if no device was available
+      if (lastError && !this.isRetryableLlmError(lastError)) {
+        // Best-effort error accounting for the most recent device
+        const lastDevice = this.llmDevice;
+        if (lastDevice) {
+          this.markLlmError(lastDevice);
+        }
       }
 
       const errorStr = String(lastError);

package/src/main.ts

@@ -6,6 +6,7 @@ import sdk, {
   Setting,
   SettingValue,
   ScryptedDeviceBase,
+  ScryptedDevice,
   ScryptedDeviceType,
   ScryptedInterface,
   ScryptedNativeId,
@@ -764,6 +765,20 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
     // ==================== 8. Auto-Topology Discovery ====================
     addGroup('Auto-Topology Discovery');
 
+    if (this.discoveryEngine) {
+      const excluded = this.discoveryEngine.getExcludedVisionLlmNames();
+      if (excluded.length > 0) {
+        settings.push({
+          key: 'excludedVisionLlms',
+          title: 'Excluded LLMs (No Vision)',
+          type: 'string',
+          readonly: true,
+          value: excluded.join(', '),
+          group: 'Auto-Topology Discovery',
+        });
+      }
+    }
+
     // ==================== 9. MQTT Integration ====================
     addGroup('MQTT Integration');
 
@@ -781,6 +796,7 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       key === 'lostTimeout' ||
       key === 'useVisualMatching' ||
       key === 'loiteringThreshold' ||
+      key === 'minDetectionScore' ||
       key === 'objectAlertCooldown' ||
       key === 'useLlmDescriptions' ||
       key === 'llmDebounceInterval' ||
@@ -916,7 +932,7 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
 
       // Training Mode endpoints
       if (path.endsWith('/api/training/start')) {
-        return this.handleTrainingStartRequest(request, response);
+        return await this.handleTrainingStartRequest(request, response);
       }
       if (path.endsWith('/api/training/pause')) {
         return this.handleTrainingPauseRequest(response);
@@ -1510,13 +1526,25 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
 
   // ==================== Training Mode Handlers ====================
 
-  private handleTrainingStartRequest(request: HttpRequest, response: HttpResponse): void {
+  private async handleTrainingStartRequest(request: HttpRequest, response: HttpResponse): Promise<void> {
     if (!this.trackingEngine) {
-      response.send(JSON.stringify({ error: 'Tracking engine not running. Configure topology first.' }), {
-        code: 500,
-        headers: { 'Content-Type': 'application/json' },
-      });
-      return;
+      const topologyJson = this.storage.getItem('topology');
+      const topology = topologyJson ? JSON.parse(topologyJson) as CameraTopology : createEmptyTopology();
+
+      if (!topology.cameras?.length) {
+        const cameras = this.buildTopologyCamerasFromSettings();
+        if (cameras.length === 0) {
+          response.send(JSON.stringify({ error: 'No cameras configured. Select tracked cameras first.' }), {
+            code: 400,
+            headers: { 'Content-Type': 'application/json' },
+          });
+          return;
+        }
+        topology.cameras = cameras;
+        this.storage.setItem('topology', JSON.stringify(topology));
+      }
+
+      await this.startTrackingEngine(topology);
     }
 
     try {
@@ -2324,6 +2352,27 @@ Access the visual topology editor at \`/ui/editor\` to configure camera relation
     const topologyJson = this.storage.getItem('topology');
     return topologyJson ? JSON.parse(topologyJson) : null;
   }
+
+  private buildTopologyCamerasFromSettings(): CameraTopology['cameras'] {
+    const value = this.storageSettings.values.trackedCameras;
+    const cameraIds = Array.isArray(value)
+      ? value.filter(Boolean)
+      : typeof value === 'string' && value.length
+        ? [value]
+        : [];
+
+    return cameraIds.map((deviceId: string) => {
+      const device = systemManager.getDeviceById<ScryptedDevice>(deviceId);
+      return {
+        deviceId,
+        nativeId: device?.nativeId || deviceId,
+        name: device?.name || deviceId,
+        isEntryPoint: false,
+        isExitPoint: false,
+        trackClasses: [],
+      };
+    });
+  }
 }
 
 export default SpatialAwarenessPlugin;