@blueharford/scrypted-spatial-awareness 0.6.32 → 0.6.34

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.32",
+  "version": "0.6.34",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",
@@ -24,7 +24,8 @@
     "scrypted-debug": "node node_modules/@scrypted/sdk/bin/scrypted-debug.js",
     "scrypted-deploy": "node node_modules/@scrypted/sdk/bin/scrypted-deploy.js",
     "scrypted-readme": "node node_modules/@scrypted/sdk/bin/scrypted-readme.js",
-    "scrypted-package-json": "node node_modules/@scrypted/sdk/bin/scrypted-package-json.js"
+    "scrypted-package-json": "node node_modules/@scrypted/sdk/bin/scrypted-package-json.js",
+    "test": "node -r ts-node/register/transpile-only ./tests/run-tests.ts"
   },
   "keywords": [
     "scrypted",
@@ -54,6 +55,7 @@
   "devDependencies": {
     "@scrypted/common": "^1.0.0",
     "@types/node": "^20.11.0",
+    "ts-node": "^10.9.2",
     "typescript": "^5.3.0"
   }
 }

package/src/alerts/alert-manager.ts

@@ -3,7 +3,7 @@
  * Generates and dispatches alerts based on tracking events
  */
 
-import sdk, { Notifier, Camera, ScryptedInterface, MediaObject } from '@scrypted/sdk';
+import type { Notifier, Camera, MediaObject } from '@scrypted/sdk';
 import {
   Alert,
   AlertRule,
@@ -12,15 +12,31 @@ import {
   AlertCondition,
   createAlert,
   createDefaultRules,
+  generateAlertMessage,
 } from '../models/alert';
 import { TrackedObject, GlobalTrackingId } from '../models/tracked-object';
-
-const { systemManager, mediaManager } = sdk;
+import {
+  getActiveAlertKey,
+  hasMeaningfulAlertChange,
+  shouldSendUpdateNotification,
+} from './alert-utils';
+
+let sdkModule: typeof import('@scrypted/sdk') | null = null;
+const getSdk = async () => {
+  if (!sdkModule) {
+    sdkModule = await import('@scrypted/sdk');
+  }
+  return sdkModule;
+};
 
 export class AlertManager {
   private rules: AlertRule[] = [];
   private recentAlerts: Alert[] = [];
   private cooldowns: Map<string, number> = new Map();
+  private activeAlerts: Map<string, { alert: Alert; lastUpdate: number; lastNotified: number }> = new Map();
+  private readonly activeAlertTtlMs: number = 10 * 60 * 1000;
+  private notifyOnUpdates: boolean = false;
+  private updateNotificationCooldownMs: number = 60000;
   private console: Console;
   private storage: Storage;
   private maxAlerts: number = 100;
@@ -37,7 +53,8 @@ export class AlertManager {
   async checkAndAlert(
     type: AlertType,
     tracked: TrackedObject,
-    details: Partial<AlertDetails>
+    details: Partial<AlertDetails>,
+    mediaObjectOverride?: MediaObject
   ): Promise<Alert | null> {
     // Find matching rule
     const rule = this.rules.find(r => r.type === type && r.enabled);
@@ -57,15 +74,21 @@ export class AlertManager {
       }
     }
 
-    // Check cooldown
-    const cooldownKey = `${rule.id}:${tracked.globalId}`;
-    const lastAlert = this.cooldowns.get(cooldownKey) || 0;
-    if (rule.cooldown > 0 && Date.now() - lastAlert < rule.cooldown) {
+    // Check conditions
+    if (!this.evaluateConditions(rule.conditions, tracked)) {
       return null;
     }
 
-    // Check conditions
-    if (!this.evaluateConditions(rule.conditions, tracked)) {
+    // Update existing movement alert if active (prevents alert spam)
+    if (type === 'movement') {
+      const updated = await this.updateActiveAlert(type, rule.id, tracked, details);
+      if (updated) return updated;
+    }
+
+    // Check cooldown (only for new alerts)
+    const cooldownKey = `${rule.id}:${tracked.globalId}`;
+    const lastAlert = this.cooldowns.get(cooldownKey) || 0;
+    if (rule.cooldown > 0 && Date.now() - lastAlert < rule.cooldown) {
       return null;
     }
 
@@ -91,21 +114,59 @@ export class AlertManager {
       this.recentAlerts.pop();
     }
 
+    if (type === 'movement') {
+      const key = getActiveAlertKey(type, rule.id, tracked.globalId);
+      this.activeAlerts.set(key, { alert, lastUpdate: Date.now(), lastNotified: alert.timestamp });
+    }
+
     // Update cooldown
     this.cooldowns.set(cooldownKey, Date.now());
 
     // Send notifications
-    await this.sendNotifications(alert, rule);
+    await this.sendNotifications(alert, rule, mediaObjectOverride);
 
     this.console.log(`Alert generated: [${alert.severity}] ${alert.message}`);
 
     return alert;
   }
 
+  async updateMovementAlert(
+    tracked: TrackedObject,
+    details: Partial<AlertDetails>
+  ): Promise<Alert | null> {
+    const rule = this.rules.find(r => r.type === 'movement' && r.enabled);
+    if (!rule) return null;
+
+    if (rule.objectClasses && rule.objectClasses.length > 0) {
+      if (!rule.objectClasses.includes(tracked.className)) {
+        return null;
+      }
+    }
+
+    if (rule.cameraIds && rule.cameraIds.length > 0 && details.cameraId) {
+      if (!rule.cameraIds.includes(details.cameraId)) {
+        return null;
+      }
+    }
+
+    if (!this.evaluateConditions(rule.conditions, tracked)) {
+      return null;
+    }
+
+    return this.updateActiveAlert('movement', rule.id, tracked, details);
+  }
+
   /**
    * Send notifications for an alert
    */
-  private async sendNotifications(alert: Alert, rule: AlertRule): Promise<void> {
+  private async sendNotifications(
+    alert: Alert,
+    rule: AlertRule,
+    mediaObjectOverride?: MediaObject
+  ): Promise<void> {
+    const sdkModule = await getSdk();
+    const { ScryptedInterface } = sdkModule;
+    const { systemManager } = sdkModule.default;
     const notifierIds = rule.notifiers.length > 0
       ? rule.notifiers
       : this.getDefaultNotifiers();
@@ -119,16 +180,18 @@ export class AlertManager {
     }
 
     // Try to get a thumbnail from the camera
-    let mediaObject: MediaObject | undefined;
-    const cameraId = alert.details.toCameraId || alert.details.cameraId;
-    if (cameraId) {
-      try {
-        const camera = systemManager.getDeviceById<Camera>(cameraId);
-        if (camera && camera.interfaces?.includes(ScryptedInterface.Camera)) {
-          mediaObject = await camera.takePicture();
+    let mediaObject: MediaObject | undefined = mediaObjectOverride;
+    if (!mediaObject) {
+      const cameraId = alert.details.toCameraId || alert.details.cameraId;
+      if (cameraId) {
+        try {
+          const camera = systemManager.getDeviceById<Camera>(cameraId);
+          if (camera && camera.interfaces?.includes(ScryptedInterface.Camera)) {
+            mediaObject = await camera.takePicture();
+          }
+        } catch (e) {
+          this.console.warn(`Failed to get thumbnail from camera ${cameraId}:`, e);
         }
-      } catch (e) {
-        this.console.warn(`Failed to get thumbnail from camera ${cameraId}:`, e);
       }
     }
 
@@ -161,6 +224,71 @@ export class AlertManager {
     }
   }
 
+  clearActiveAlertsForObject(globalId: GlobalTrackingId): void {
+    for (const [key, entry] of this.activeAlerts.entries()) {
+      if (entry.alert.trackedObjectId === globalId) {
+        this.activeAlerts.delete(key);
+      }
+    }
+  }
+
+  setUpdateNotificationOptions(enabled: boolean, cooldownMs: number): void {
+    this.notifyOnUpdates = enabled;
+    this.updateNotificationCooldownMs = Math.max(0, cooldownMs);
+  }
+
+  private async updateActiveAlert(
+    type: AlertType,
+    ruleId: string,
+    tracked: TrackedObject,
+    details: Partial<AlertDetails>
+  ): Promise<Alert | null> {
+    const key = getActiveAlertKey(type, ruleId, tracked.globalId);
+    const existing = this.activeAlerts.get(key);
+    if (!existing) return null;
+
+    const now = Date.now();
+    if (now - existing.lastUpdate > this.activeAlertTtlMs) {
+      this.activeAlerts.delete(key);
+      return null;
+    }
+
+    const updatedDetails: AlertDetails = {
+      ...existing.alert.details,
+      ...details,
+      objectClass: tracked.className,
+      objectLabel: details.objectLabel || tracked.label,
+    };
+
+    const shouldUpdate = hasMeaningfulAlertChange(existing.alert.details, updatedDetails);
+    if (!shouldUpdate) return existing.alert;
+
+    existing.alert.details = updatedDetails;
+    existing.alert.message = generateAlertMessage(type, updatedDetails);
+    existing.alert.timestamp = now;
+    existing.lastUpdate = now;
+
+    const idx = this.recentAlerts.findIndex(a => a.id === existing.alert.id);
+    if (idx >= 0) {
+      this.recentAlerts.splice(idx, 1);
+    }
+    this.recentAlerts.unshift(existing.alert);
+    if (this.recentAlerts.length > this.maxAlerts) {
+      this.recentAlerts.pop();
+    }
+
+    if (this.notifyOnUpdates) {
+      const rule = this.rules.find(r => r.id === ruleId);
+      if (rule && shouldSendUpdateNotification(this.notifyOnUpdates, existing.lastNotified, now, this.updateNotificationCooldownMs)) {
+        existing.lastNotified = now;
+        await this.sendNotifications(existing.alert, rule);
+      }
+    }
+
+    return existing.alert;
+  }
+
+
   /**
    * Get notification title based on alert type
    * For movement alerts with LLM descriptions, use the smart description as title

package/src/alerts/alert-utils.ts

@@ -0,0 +1,32 @@
+import { AlertDetails, AlertType } from '../models/alert';
+import { GlobalTrackingId } from '../models/tracked-object';
+
+export function hasMeaningfulAlertChange(prev: AlertDetails, next: AlertDetails): boolean {
+  return (
+    prev.fromCameraId !== next.fromCameraId ||
+    prev.toCameraId !== next.toCameraId ||
+    prev.cameraId !== next.cameraId ||
+    prev.objectLabel !== next.objectLabel ||
+    prev.pathDescription !== next.pathDescription ||
+    JSON.stringify(prev.involvedLandmarks || []) !== JSON.stringify(next.involvedLandmarks || [])
+  );
+}
+
+export function getActiveAlertKey(
+  type: AlertType,
+  ruleId: string,
+  trackedId: GlobalTrackingId
+): string {
+  return `${type}:${ruleId}:${trackedId}`;
+}
+
+export function shouldSendUpdateNotification(
+  enabled: boolean,
+  lastNotified: number,
+  now: number,
+  cooldownMs: number
+): boolean {
+  if (!enabled) return false;
+  if (cooldownMs <= 0) return true;
+  return now - lastNotified >= cooldownMs;
+}

package/src/core/topology-discovery.ts

@@ -231,6 +231,13 @@ export class TopologyDiscoveryEngine {
     return { ...this.status };
   }
 
+  /** Get list of LLMs excluded for lack of vision support */
+  getExcludedVisionLlmNames(): string[] {
+    return this.llmDevices
+      .filter(l => !l.visionCapable)
+      .map(l => l.name || l.id);
+  }
+
   /** Get pending suggestions */
   getPendingSuggestions(): DiscoverySuggestion[] {
     return Array.from(this.suggestions.values())
@@ -262,6 +269,7 @@ export class TopologyDiscoveryEngine {
     providerType: LlmProvider;
     lastUsed: number;
     errorCount: number;
+    visionCapable: boolean;
   }> = [];
 
   /** Find ALL LLM devices for load balancing */
@@ -294,6 +302,7 @@ export class TopologyDiscoveryEngine {
             providerType,
             lastUsed: 0,
             errorCount: 0,
+            visionCapable: true,
           });
 
           this.console.log(`[Discovery] Found LLM: ${device.name}`);
@@ -348,6 +357,48 @@ export class TopologyDiscoveryEngine {
     return selected.device;
   }
 
+  /** Select an LLM device, excluding any IDs if provided */
+  private async selectLlmDevice(excludeIds: Set<string>): Promise<ChatCompletionDevice | null> {
+    await this.findAllLlmDevices();
+
+    if (this.llmDevices.length === 0) return null;
+
+    let bestIndex = -1;
+    let bestScore = Infinity;
+
+    for (let i = 0; i < this.llmDevices.length; i++) {
+      const llm = this.llmDevices[i];
+      if (excludeIds.has(llm.id)) continue;
+      if (!llm.visionCapable) continue;
+      const score = llm.lastUsed + (llm.errorCount * 60000);
+      if (score < bestScore) {
+        bestScore = score;
+        bestIndex = i;
+      }
+    }
+
+    if (bestIndex === -1) return null;
+
+    const selected = this.llmDevices[bestIndex];
+    this.llmDevice = selected.device;
+    this.llmProviderType = selected.providerType;
+    selected.lastUsed = Date.now();
+
+    this.console.log(`[Discovery] Selected LLM: ${selected.name}`);
+    return selected.device;
+  }
+
+  private isRetryableLlmError(error: any): boolean {
+    const errorStr = String(error).toLowerCase();
+    return (
+      errorStr.includes('404') ||
+      errorStr.includes('not found') ||
+      errorStr.includes('no such model') ||
+      errorStr.includes('model not found') ||
+      errorStr.includes('endpoint')
+    );
+  }
+
   /** Mark an LLM as having an error */
   private markLlmError(device: ChatCompletionDevice): void {
     const llm = this.llmDevices.find(l => l.device === device);
@@ -406,42 +457,49 @@ export class TopologyDiscoveryEngine {
       isValid: false,
     };
 
-    const llm = await this.findLlmDevice();
-    if (!llm?.getChatCompletion) {
-      analysis.error = 'No LLM device available';
-      return analysis;
-    }
-
     const imageData = await this.getCameraSnapshot(cameraId);
     if (!imageData) {
       analysis.error = 'Failed to capture camera snapshot';
       return analysis;
     }
 
-    // Try with detected provider format first, then fallback to alternates
-    // The order matters: try the most likely formats first
-    const formatsToTry: LlmProvider[] = [];
-
-    // Start with detected format
-    formatsToTry.push(this.llmProviderType);
-
-    // Add fallbacks based on detected provider
-    if (this.llmProviderType === 'openai') {
-      formatsToTry.push('scrypted', 'anthropic');
-    } else if (this.llmProviderType === 'anthropic') {
-      formatsToTry.push('scrypted', 'openai');
-    } else if (this.llmProviderType === 'scrypted') {
-      formatsToTry.push('anthropic', 'openai');
-    } else {
-      // Unknown - try all formats
-      formatsToTry.push('scrypted', 'anthropic', 'openai');
-    }
-
+    await this.findAllLlmDevices();
+    const excludeIds = new Set<string>();
     let lastError: any = null;
+    const maxAttempts = Math.max(1, this.llmDevices.length || 1);
 
-    for (const formatType of formatsToTry) {
-      try {
-        this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      const llm = await this.selectLlmDevice(excludeIds);
+      if (!llm?.getChatCompletion) {
+        analysis.error = 'No LLM device available';
+        return analysis;
+      }
+
+      let allFormatsVisionError = false;
+
+      // Try with detected provider format first, then fallback to alternates
+      // The order matters: try the most likely formats first
+      const formatsToTry: LlmProvider[] = [];
+
+      // Start with detected format
+      formatsToTry.push(this.llmProviderType);
+
+      // Add fallbacks based on detected provider
+      if (this.llmProviderType === 'openai') {
+        formatsToTry.push('scrypted', 'anthropic');
+      } else if (this.llmProviderType === 'anthropic') {
+        formatsToTry.push('scrypted', 'openai');
+      } else if (this.llmProviderType === 'scrypted') {
+        formatsToTry.push('anthropic', 'openai');
+      } else {
+        // Unknown - try all formats
+        formatsToTry.push('scrypted', 'anthropic', 'openai');
+      }
+
+      let visionFormatFailures = 0;
+      for (const formatType of formatsToTry) {
+        try {
+          this.console.log(`[Discovery] Trying ${formatType} image format for ${cameraName}...`);
 
         // Build prompt with camera context (height)
         const cameraNode = this.topology ? findCamera(this.topology, cameraId) : null;
@@ -459,7 +517,7 @@ Use the mount height to help estimate distances - objects at ground level will a
 `;
 
         // Build multimodal message with provider-specific image format
-        const result = await llm.getChatCompletion({
+          const result = await llm.getChatCompletion({
           messages: [
             {
               role: 'user',
@@ -473,91 +531,119 @@ Use the mount height to help estimate distances - objects at ground level will a
           temperature: 0.3,
         });
 
-        const content = result?.choices?.[0]?.message?.content;
-        if (content && typeof content === 'string') {
-          try {
-            // Extract JSON from response (handle markdown code blocks)
-            let jsonStr = content.trim();
-            if (jsonStr.startsWith('```')) {
-              jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
-            }
+          const content = result?.choices?.[0]?.message?.content;
+          if (content && typeof content === 'string') {
+            try {
+              // Extract JSON from response (handle markdown code blocks)
+              let jsonStr = content.trim();
+              if (jsonStr.startsWith('```')) {
+                jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```$/g, '').trim();
+              }
 
-            // Try to recover truncated JSON
-            const parsed = this.parseJsonWithRecovery(jsonStr, cameraName);
+              // Try to recover truncated JSON
+              const parsed = this.parseJsonWithRecovery(jsonStr, cameraName);
+
+              // Map parsed data to our types
+              if (Array.isArray(parsed.landmarks)) {
+                analysis.landmarks = parsed.landmarks.map((l: any) => ({
+                  name: l.name || 'Unknown',
+                  type: this.mapLandmarkType(l.type),
+                  confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
+                  distance: this.mapDistance(l.distance),
+                  description: l.description || '',
+                  boundingBox: l.boundingBox,
+                }));
+              }
 
-            // Map parsed data to our types
-            if (Array.isArray(parsed.landmarks)) {
-              analysis.landmarks = parsed.landmarks.map((l: any) => ({
-                name: l.name || 'Unknown',
-                type: this.mapLandmarkType(l.type),
-                confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
-                distance: this.mapDistance(l.distance),
-                description: l.description || '',
-                boundingBox: l.boundingBox,
-              }));
-            }
+              if (Array.isArray(parsed.zones)) {
+                analysis.zones = parsed.zones.map((z: any) => ({
+                  name: z.name || 'Unknown',
+                  type: this.mapZoneType(z.type),
+                  coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
+                  description: z.description || '',
+                  boundingBox: z.boundingBox,
+                  distance: this.mapDistance(z.distance), // Parse distance for zones too
+                } as DiscoveredZone & { distance?: DistanceEstimate }));
+              }
 
-            if (Array.isArray(parsed.zones)) {
-              analysis.zones = parsed.zones.map((z: any) => ({
-                name: z.name || 'Unknown',
-                type: this.mapZoneType(z.type),
-                coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
-                description: z.description || '',
-                boundingBox: z.boundingBox,
-                distance: this.mapDistance(z.distance), // Parse distance for zones too
-              } as DiscoveredZone & { distance?: DistanceEstimate }));
-            }
+              if (parsed.edges && typeof parsed.edges === 'object') {
+                analysis.edges = {
+                  top: parsed.edges.top || '',
+                  left: parsed.edges.left || '',
+                  right: parsed.edges.right || '',
+                  bottom: parsed.edges.bottom || '',
+                };
+              }
 
-            if (parsed.edges && typeof parsed.edges === 'object') {
-              analysis.edges = {
-                top: parsed.edges.top || '',
-                left: parsed.edges.left || '',
-                right: parsed.edges.right || '',
-                bottom: parsed.edges.bottom || '',
-              };
-            }
+              if (parsed.orientation) {
+                analysis.orientation = this.mapOrientation(parsed.orientation);
+              }
 
-            if (parsed.orientation) {
-              analysis.orientation = this.mapOrientation(parsed.orientation);
-            }
+              analysis.isValid = true;
+              this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
 
-            analysis.isValid = true;
-            this.console.log(`[Discovery] Analyzed ${cameraName}: ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones (using ${formatType} format)`);
+              // Update the preferred format for future requests
+              if (formatType !== this.llmProviderType) {
+                this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
+                this.llmProviderType = formatType;
+              }
 
-            // Update the preferred format for future requests
-            if (formatType !== this.llmProviderType) {
-              this.console.log(`[Discovery] Switching to ${formatType} format for future requests`);
-              this.llmProviderType = formatType;
+              // Success - exit the retry loop
+              return analysis;
+            } catch (parseError) {
+              this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
+              analysis.error = 'Failed to parse LLM response';
+              return analysis;
             }
+          }
+        } catch (e) {
+          lastError = e;
+
+          // Check if this is a vision/multimodal format error
+          if (isVisionFormatError(e)) {
+            this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
+            visionFormatFailures++;
+            continue; // Try next format
+          }
 
-            // Success - exit the retry loop
-            return analysis;
-          } catch (parseError) {
-            this.console.warn(`[Discovery] Failed to parse LLM response for ${cameraName}:`, parseError);
-            analysis.error = 'Failed to parse LLM response';
-            return analysis;
+          // Retry with a different LLM if error indicates bad endpoint/model
+          if (this.isRetryableLlmError(e)) {
+            this.console.warn(`[Discovery] LLM error for ${cameraName}, trying another provider...`);
+            this.markLlmError(llm);
+            const llmEntry = this.llmDevices.find(d => d.device === llm);
+            if (llmEntry) {
+              excludeIds.add(llmEntry.id);
+            }
+            break;
           }
-        }
-      } catch (e) {
-        lastError = e;
 
-        // Check if this is a vision/multimodal format error
-        if (isVisionFormatError(e)) {
-          this.console.warn(`[Discovery] ${formatType} format failed, trying fallback...`);
-          continue; // Try next format
+          // Not a format error - don't retry
+          this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
+          break;
         }
+      }
 
-        // Not a format error - don't retry
-        this.console.warn(`[Discovery] Scene analysis failed for ${cameraName}:`, e);
-        break;
+      allFormatsVisionError = visionFormatFailures > 0 && visionFormatFailures === formatsToTry.length;
+      if (allFormatsVisionError) {
+        const llmEntry = this.llmDevices.find(d => d.device === llm);
+        if (llmEntry) {
+          llmEntry.visionCapable = false;
+          excludeIds.add(llmEntry.id);
+          this.console.warn(`[Discovery] ${llmEntry.name} does not support vision. Excluding from discovery.`);
+        }
       }
     }
 
     // All formats failed
     if (lastError) {
       // Track error for load balancing
-      if (llm) {
-        this.markLlmError(llm);
+      // Note: llm may be null here if no device was available
+      if (lastError && !this.isRetryableLlmError(lastError)) {
+        // Best-effort error accounting for the most recent device
+        const lastDevice = this.llmDevice;
+        if (lastDevice) {
+          this.markLlmError(lastDevice);
+        }
       }
 
       const errorStr = String(lastError);