@blueharford/scrypted-spatial-awareness 0.6.12 → 0.6.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.12",
+  "version": "0.6.14",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/tracking-engine.ts

@@ -138,6 +138,12 @@ export class TrackingEngine {
   /** Callback for training status updates */
   private onTrainingStatusUpdate?: (status: TrainingStatusUpdate) => void;
 
+  // ==================== Snapshot Cache ====================
+  /** Cached snapshots for tracked objects (for faster notifications) */
+  private snapshotCache: Map<GlobalTrackingId, MediaObject> = new Map();
+  /** Pending LLM description promises (started when snapshot is captured) */
+  private pendingDescriptions: Map<GlobalTrackingId, Promise<SpatialReasoningResult>> = new Map();
+
   constructor(
     topology: CameraTopology,
     state: TrackingState,
@@ -437,6 +443,13 @@ export class TrackingEngine {
       if (lastSighting && lastSighting.cameraId !== sighting.cameraId) {
         const transitDuration = sighting.timestamp - lastSighting.timestamp;
 
+        // Update cached snapshot from new camera (object is now visible here)
+        if (this.config.useLlmDescriptions) {
+          this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId).catch(e => {
+            this.console.warn(`[Transition Snapshot] Failed to update snapshot: ${e}`);
+          });
+        }
+
         // Add journey segment
         this.state.addJourney(tracked.globalId, {
           fromCameraId: lastSighting.cameraId,
@@ -528,6 +541,14 @@ export class TrackingEngine {
     sighting: ObjectSighting,
     isEntryPoint: boolean
   ): void {
+    // Capture snapshot IMMEDIATELY when object is first detected (don't wait for loitering threshold)
+    // This ensures we have a good image while the person/object is still in frame
+    if (this.config.useLlmDescriptions) {
+      this.captureAndCacheSnapshot(globalId, sighting.cameraId).catch(e => {
+        this.console.warn(`[Snapshot] Failed to cache initial snapshot: ${e}`);
+      });
+    }
+
     // Check after loitering threshold if object is still being tracked
     setTimeout(async () => {
       const tracked = this.state.getObject(globalId);
@@ -536,31 +557,28 @@ export class TrackingEngine {
       // Check if we've already alerted for this object
       if (this.isInAlertCooldown(globalId)) return;
 
-      // Get snapshot for LLM description (if LLM is enabled)
-      let mediaObject: MediaObject | undefined;
-      this.console.log(`[Entry Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
-      if (this.config.useLlmDescriptions) {
+      // Use prefetched LLM result if available (started when snapshot was captured)
+      let spatialResult: SpatialReasoningResult;
+      const pendingDescription = this.pendingDescriptions.get(globalId);
+
+      if (pendingDescription) {
+        this.console.log(`[Entry Alert] Using prefetched LLM result for ${globalId.slice(0, 8)}`);
         try {
-          const camera = systemManager.getDeviceById<Camera>(sighting.cameraId);
-          this.console.log(`[Entry Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(ScryptedInterface.Camera)}`);
-          if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
-            mediaObject = await camera.takePicture();
-            this.console.log(`[Entry Alert] Got snapshot: ${!!mediaObject}`);
-          }
+          spatialResult = await pendingDescription;
+          this.console.log(`[Entry Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
         } catch (e) {
-          this.console.warn('[Entry Alert] Failed to get snapshot:', e);
+          this.console.warn(`[Entry Alert] Prefetch failed, generating fallback: ${e}`);
+          spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId);
         }
+        this.pendingDescriptions.delete(globalId);
+      } else {
+        // Fallback: generate description now (slower path)
+        this.console.log(`[Entry Alert] No prefetch available, generating now`);
+        const mediaObject = this.snapshotCache.get(globalId);
+        spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
+        this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
       }
 
-      // Generate spatial description (now async with LLM support)
-      this.console.log(`[Entry Alert] Calling generateEntryDescription with mediaObject=${!!mediaObject}`);
-      const spatialResult = await this.spatialReasoning.generateEntryDescription(
-        tracked,
-        sighting.cameraId,
-        mediaObject
-      );
-      this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
-
       if (isEntryPoint) {
         // Entry point - generate property entry alert
         await this.alertManager.checkAndAlert('property_entry', tracked, {
@@ -592,6 +610,44 @@ export class TrackingEngine {
     }, this.config.loiteringThreshold);
   }
 
+  /** Capture and cache a snapshot for a tracked object, and start LLM analysis immediately */
+  private async captureAndCacheSnapshot(
+    globalId: GlobalTrackingId,
+    cameraId: string,
+    eventType: 'entry' | 'exit' | 'movement' = 'entry'
+  ): Promise<void> {
+    try {
+      const camera = systemManager.getDeviceById<Camera>(cameraId);
+      if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
+        const mediaObject = await camera.takePicture();
+        if (mediaObject) {
+          this.snapshotCache.set(globalId, mediaObject);
+          this.console.log(`[Snapshot] Cached snapshot for ${globalId.slice(0, 8)} from ${cameraId}`);
+
+          // Start LLM analysis immediately in parallel (don't await)
+          const tracked = this.state.getObject(globalId);
+          if (tracked && this.config.useLlmDescriptions) {
+            this.console.log(`[LLM Prefetch] Starting ${eventType} analysis for ${globalId.slice(0, 8)}`);
+            const descriptionPromise = eventType === 'exit'
+              ? this.spatialReasoning.generateExitDescription(tracked, cameraId, mediaObject)
+              : this.spatialReasoning.generateEntryDescription(tracked, cameraId, mediaObject);
+
+            this.pendingDescriptions.set(globalId, descriptionPromise);
+
+            // Log when complete (for debugging)
+            descriptionPromise.then(result => {
+              this.console.log(`[LLM Prefetch] ${eventType} analysis ready for ${globalId.slice(0, 8)}: "${result.description.substring(0, 40)}..."`);
+            }).catch(e => {
+              this.console.warn(`[LLM Prefetch] Failed for ${globalId.slice(0, 8)}: ${e}`);
+            });
+          }
+        }
+      }
+    } catch (e) {
+      this.console.warn(`[Snapshot] Failed to capture snapshot: ${e}`);
+    }
+  }
+
   /** Attempt to correlate a sighting with existing tracked objects */
   private async correlateDetection(
     sighting: ObjectSighting
@@ -639,40 +695,42 @@ export class TrackingEngine {
     // Mark as pending and set timer
     this.state.markPending(tracked.globalId);
 
+    // Capture a fresh snapshot now while object is still visible (before they leave)
+    // Also starts LLM analysis immediately in parallel
+    if (this.config.useLlmDescriptions) {
+      this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId, 'exit').catch(e => {
+        this.console.warn(`[Exit Snapshot] Failed to update snapshot: ${e}`);
+      });
+    }
+
     // Wait for correlation window before marking as exited
     const timer = setTimeout(async () => {
       const current = this.state.getObject(tracked.globalId);
       if (current && current.state === 'pending') {
         this.state.markExited(tracked.globalId, sighting.cameraId, sighting.cameraName);
 
-        // Get snapshot for LLM description (if LLM is enabled)
-        let mediaObject: MediaObject | undefined;
-        this.console.log(`[Exit Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
-        if (this.config.useLlmDescriptions) {
+        // Use prefetched LLM result if available (started when exit was first detected)
+        let spatialResult: SpatialReasoningResult;
+        const pendingDescription = this.pendingDescriptions.get(tracked.globalId);
+
+        if (pendingDescription) {
+          this.console.log(`[Exit Alert] Using prefetched LLM result for ${tracked.globalId.slice(0, 8)}`);
           try {
-            const camera = systemManager.getDeviceById<Camera>(sighting.cameraId);
-            this.console.log(`[Exit Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(ScryptedInterface.Camera)}`);
-            if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
-              mediaObject = await camera.takePicture();
-              this.console.log(`[Exit Alert] Got snapshot: ${!!mediaObject}`);
-            }
+            spatialResult = await pendingDescription;
+            this.console.log(`[Exit Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
           } catch (e) {
-            this.console.warn('[Exit Alert] Failed to get snapshot:', e);
+            this.console.warn(`[Exit Alert] Prefetch failed, generating fallback: ${e}`);
+            spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId);
           }
+          this.pendingDescriptions.delete(tracked.globalId);
+        } else {
+          // Fallback: generate description now (slower path)
+          this.console.log(`[Exit Alert] No prefetch available, generating now`);
+          const mediaObject = this.snapshotCache.get(tracked.globalId);
+          spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
+          this.console.log(`[Exit Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
         }
 
-        // Generate rich exit description using topology context (now async with LLM support)
-        this.console.log(`[Exit Alert] Calling generateExitDescription with mediaObject=${!!mediaObject}`);
-        const spatialResult = await this.spatialReasoning.generateExitDescription(
-          current,
-          sighting.cameraId,
-          mediaObject
-        );
-
-        this.console.log(
-          `[Exit Alert] Object ${tracked.globalId.slice(0, 8)} exited: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`
-        );
-
         await this.alertManager.checkAndAlert('property_exit', current, {
           cameraId: sighting.cameraId,
           cameraName: sighting.cameraName,
@@ -681,6 +739,10 @@ export class TrackingEngine {
           involvedLandmarks: spatialResult.involvedLandmarks?.map(l => l.name),
           usedLlm: spatialResult.usedLlm,
         });
+
+        // Clean up cached snapshot and pending descriptions after exit alert
+        this.snapshotCache.delete(tracked.globalId);
+        this.pendingDescriptions.delete(tracked.globalId);
       }
       this.pendingTimers.delete(tracked.globalId);
     }, this.config.correlationWindow);
@@ -703,6 +765,10 @@ export class TrackingEngine {
           `(not seen for ${Math.round(timeSinceSeen / 1000)}s)`
         );
 
+        // Clean up cached snapshot and pending descriptions
+        this.snapshotCache.delete(tracked.globalId);
+        this.pendingDescriptions.delete(tracked.globalId);
+
         this.alertManager.checkAndAlert('lost_tracking', tracked, {
           objectClass: tracked.className,
           objectLabel: tracked.label,