@blueharford/scrypted-spatial-awareness 0.6.12 → 0.6.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.12",
+  "version": "0.6.13",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/tracking-engine.ts

@@ -138,6 +138,10 @@ export class TrackingEngine {
   /** Callback for training status updates */
   private onTrainingStatusUpdate?: (status: TrainingStatusUpdate) => void;
 
+  // ==================== Snapshot Cache ====================
+  /** Cached snapshots for tracked objects (for faster notifications) */
+  private snapshotCache: Map<GlobalTrackingId, MediaObject> = new Map();
+
   constructor(
     topology: CameraTopology,
     state: TrackingState,
@@ -437,6 +441,13 @@ export class TrackingEngine {
       if (lastSighting && lastSighting.cameraId !== sighting.cameraId) {
         const transitDuration = sighting.timestamp - lastSighting.timestamp;
 
+        // Update cached snapshot from new camera (object is now visible here)
+        if (this.config.useLlmDescriptions) {
+          this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId).catch(e => {
+            this.console.warn(`[Transition Snapshot] Failed to update snapshot: ${e}`);
+          });
+        }
+
         // Add journey segment
         this.state.addJourney(tracked.globalId, {
           fromCameraId: lastSighting.cameraId,
@@ -528,6 +539,14 @@ export class TrackingEngine {
     sighting: ObjectSighting,
     isEntryPoint: boolean
   ): void {
+    // Capture snapshot IMMEDIATELY when object is first detected (don't wait for loitering threshold)
+    // This ensures we have a good image while the person/object is still in frame
+    if (this.config.useLlmDescriptions) {
+      this.captureAndCacheSnapshot(globalId, sighting.cameraId).catch(e => {
+        this.console.warn(`[Snapshot] Failed to cache initial snapshot: ${e}`);
+      });
+    }
+
     // Check after loitering threshold if object is still being tracked
     setTimeout(async () => {
       const tracked = this.state.getObject(globalId);
@@ -536,21 +555,9 @@ export class TrackingEngine {
       // Check if we've already alerted for this object
       if (this.isInAlertCooldown(globalId)) return;
 
-      // Get snapshot for LLM description (if LLM is enabled)
-      let mediaObject: MediaObject | undefined;
-      this.console.log(`[Entry Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
-      if (this.config.useLlmDescriptions) {
-        try {
-          const camera = systemManager.getDeviceById<Camera>(sighting.cameraId);
-          this.console.log(`[Entry Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(ScryptedInterface.Camera)}`);
-          if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
-            mediaObject = await camera.takePicture();
-            this.console.log(`[Entry Alert] Got snapshot: ${!!mediaObject}`);
-          }
-        } catch (e) {
-          this.console.warn('[Entry Alert] Failed to get snapshot:', e);
-        }
-      }
+      // Use cached snapshot (captured when object was first detected)
+      let mediaObject = this.snapshotCache.get(globalId);
+      this.console.log(`[Entry Alert] Using cached snapshot: ${!!mediaObject}`);
 
       // Generate spatial description (now async with LLM support)
       this.console.log(`[Entry Alert] Calling generateEntryDescription with mediaObject=${!!mediaObject}`);
@@ -592,6 +599,22 @@ export class TrackingEngine {
     }, this.config.loiteringThreshold);
   }
 
+  /** Capture and cache a snapshot for a tracked object */
+  private async captureAndCacheSnapshot(globalId: GlobalTrackingId, cameraId: string): Promise<void> {
+    try {
+      const camera = systemManager.getDeviceById<Camera>(cameraId);
+      if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
+        const mediaObject = await camera.takePicture();
+        if (mediaObject) {
+          this.snapshotCache.set(globalId, mediaObject);
+          this.console.log(`[Snapshot] Cached snapshot for ${globalId.slice(0, 8)} from ${cameraId}`);
+        }
+      }
+    } catch (e) {
+      this.console.warn(`[Snapshot] Failed to capture snapshot: ${e}`);
+    }
+  }
+
   /** Attempt to correlate a sighting with existing tracked objects */
   private async correlateDetection(
     sighting: ObjectSighting
@@ -639,27 +662,22 @@ export class TrackingEngine {
     // Mark as pending and set timer
     this.state.markPending(tracked.globalId);
 
+    // Capture a fresh snapshot now while object is still visible (before they leave)
+    if (this.config.useLlmDescriptions) {
+      this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId).catch(e => {
+        this.console.warn(`[Exit Snapshot] Failed to update snapshot: ${e}`);
+      });
+    }
+
     // Wait for correlation window before marking as exited
     const timer = setTimeout(async () => {
       const current = this.state.getObject(tracked.globalId);
       if (current && current.state === 'pending') {
         this.state.markExited(tracked.globalId, sighting.cameraId, sighting.cameraName);
 
-        // Get snapshot for LLM description (if LLM is enabled)
-        let mediaObject: MediaObject | undefined;
-        this.console.log(`[Exit Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
-        if (this.config.useLlmDescriptions) {
-          try {
-            const camera = systemManager.getDeviceById<Camera>(sighting.cameraId);
-            this.console.log(`[Exit Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(ScryptedInterface.Camera)}`);
-            if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
-              mediaObject = await camera.takePicture();
-              this.console.log(`[Exit Alert] Got snapshot: ${!!mediaObject}`);
-            }
-          } catch (e) {
-            this.console.warn('[Exit Alert] Failed to get snapshot:', e);
-          }
-        }
+        // Use cached snapshot (captured when exit was first detected, while object was still visible)
+        let mediaObject = this.snapshotCache.get(tracked.globalId);
+        this.console.log(`[Exit Alert] Using cached snapshot: ${!!mediaObject}`);
 
         // Generate rich exit description using topology context (now async with LLM support)
         this.console.log(`[Exit Alert] Calling generateExitDescription with mediaObject=${!!mediaObject}`);
@@ -681,6 +699,9 @@ export class TrackingEngine {
           involvedLandmarks: spatialResult.involvedLandmarks?.map(l => l.name),
           usedLlm: spatialResult.usedLlm,
         });
+
+        // Clean up cached snapshot after exit alert
+        this.snapshotCache.delete(tracked.globalId);
       }
       this.pendingTimers.delete(tracked.globalId);
     }, this.config.correlationWindow);
@@ -703,6 +724,9 @@ export class TrackingEngine {
           `(not seen for ${Math.round(timeSinceSeen / 1000)}s)`
         );
 
+        // Clean up cached snapshot
+        this.snapshotCache.delete(tracked.globalId);
+
         this.alertManager.checkAndAlert('lost_tracking', tracked, {
           objectClass: tracked.className,
           objectLabel: tracked.label,