@blueharford/scrypted-spatial-awareness 0.6.13 → 0.6.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.13",
+  "version": "0.6.14",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/tracking-engine.ts

@@ -141,6 +141,8 @@ export class TrackingEngine {
   // ==================== Snapshot Cache ====================
   /** Cached snapshots for tracked objects (for faster notifications) */
   private snapshotCache: Map<GlobalTrackingId, MediaObject> = new Map();
+  /** Pending LLM description promises (started when snapshot is captured) */
+  private pendingDescriptions: Map<GlobalTrackingId, Promise<SpatialReasoningResult>> = new Map();
 
   constructor(
     topology: CameraTopology,
@@ -555,18 +557,27 @@ export class TrackingEngine {
       // Check if we've already alerted for this object
       if (this.isInAlertCooldown(globalId)) return;
 
-      // Use cached snapshot (captured when object was first detected)
-      let mediaObject = this.snapshotCache.get(globalId);
-      this.console.log(`[Entry Alert] Using cached snapshot: ${!!mediaObject}`);
+      // Use prefetched LLM result if available (started when snapshot was captured)
+      let spatialResult: SpatialReasoningResult;
+      const pendingDescription = this.pendingDescriptions.get(globalId);
 
-      // Generate spatial description (now async with LLM support)
-      this.console.log(`[Entry Alert] Calling generateEntryDescription with mediaObject=${!!mediaObject}`);
-      const spatialResult = await this.spatialReasoning.generateEntryDescription(
-        tracked,
-        sighting.cameraId,
-        mediaObject
-      );
-      this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+      if (pendingDescription) {
+        this.console.log(`[Entry Alert] Using prefetched LLM result for ${globalId.slice(0, 8)}`);
+        try {
+          spatialResult = await pendingDescription;
+          this.console.log(`[Entry Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+        } catch (e) {
+          this.console.warn(`[Entry Alert] Prefetch failed, generating fallback: ${e}`);
+          spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId);
+        }
+        this.pendingDescriptions.delete(globalId);
+      } else {
+        // Fallback: generate description now (slower path)
+        this.console.log(`[Entry Alert] No prefetch available, generating now`);
+        const mediaObject = this.snapshotCache.get(globalId);
+        spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
+        this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+      }
 
       if (isEntryPoint) {
         // Entry point - generate property entry alert
@@ -599,8 +610,12 @@ export class TrackingEngine {
     }, this.config.loiteringThreshold);
   }
 
-  /** Capture and cache a snapshot for a tracked object */
-  private async captureAndCacheSnapshot(globalId: GlobalTrackingId, cameraId: string): Promise<void> {
+  /** Capture and cache a snapshot for a tracked object, and start LLM analysis immediately */
+  private async captureAndCacheSnapshot(
+    globalId: GlobalTrackingId,
+    cameraId: string,
+    eventType: 'entry' | 'exit' | 'movement' = 'entry'
+  ): Promise<void> {
     try {
       const camera = systemManager.getDeviceById<Camera>(cameraId);
       if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
@@ -608,6 +623,24 @@ export class TrackingEngine {
         if (mediaObject) {
           this.snapshotCache.set(globalId, mediaObject);
           this.console.log(`[Snapshot] Cached snapshot for ${globalId.slice(0, 8)} from ${cameraId}`);
+
+          // Start LLM analysis immediately in parallel (don't await)
+          const tracked = this.state.getObject(globalId);
+          if (tracked && this.config.useLlmDescriptions) {
+            this.console.log(`[LLM Prefetch] Starting ${eventType} analysis for ${globalId.slice(0, 8)}`);
+            const descriptionPromise = eventType === 'exit'
+              ? this.spatialReasoning.generateExitDescription(tracked, cameraId, mediaObject)
+              : this.spatialReasoning.generateEntryDescription(tracked, cameraId, mediaObject);
+
+            this.pendingDescriptions.set(globalId, descriptionPromise);
+
+            // Log when complete (for debugging)
+            descriptionPromise.then(result => {
+              this.console.log(`[LLM Prefetch] ${eventType} analysis ready for ${globalId.slice(0, 8)}: "${result.description.substring(0, 40)}..."`);
+            }).catch(e => {
+              this.console.warn(`[LLM Prefetch] Failed for ${globalId.slice(0, 8)}: ${e}`);
+            });
+          }
         }
       }
     } catch (e) {
@@ -663,8 +696,9 @@ export class TrackingEngine {
     this.state.markPending(tracked.globalId);
 
     // Capture a fresh snapshot now while object is still visible (before they leave)
+    // Also starts LLM analysis immediately in parallel
     if (this.config.useLlmDescriptions) {
-      this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId).catch(e => {
+      this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId, 'exit').catch(e => {
         this.console.warn(`[Exit Snapshot] Failed to update snapshot: ${e}`);
       });
     }
@@ -675,21 +709,27 @@ export class TrackingEngine {
       if (current && current.state === 'pending') {
         this.state.markExited(tracked.globalId, sighting.cameraId, sighting.cameraName);
 
-        // Use cached snapshot (captured when exit was first detected, while object was still visible)
-        let mediaObject = this.snapshotCache.get(tracked.globalId);
-        this.console.log(`[Exit Alert] Using cached snapshot: ${!!mediaObject}`);
-
-        // Generate rich exit description using topology context (now async with LLM support)
-        this.console.log(`[Exit Alert] Calling generateExitDescription with mediaObject=${!!mediaObject}`);
-        const spatialResult = await this.spatialReasoning.generateExitDescription(
-          current,
-          sighting.cameraId,
-          mediaObject
-        );
-
-        this.console.log(
-          `[Exit Alert] Object ${tracked.globalId.slice(0, 8)} exited: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`
-        );
+        // Use prefetched LLM result if available (started when exit was first detected)
+        let spatialResult: SpatialReasoningResult;
+        const pendingDescription = this.pendingDescriptions.get(tracked.globalId);
+
+        if (pendingDescription) {
+          this.console.log(`[Exit Alert] Using prefetched LLM result for ${tracked.globalId.slice(0, 8)}`);
+          try {
+            spatialResult = await pendingDescription;
+            this.console.log(`[Exit Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+          } catch (e) {
+            this.console.warn(`[Exit Alert] Prefetch failed, generating fallback: ${e}`);
+            spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId);
+          }
+          this.pendingDescriptions.delete(tracked.globalId);
+        } else {
+          // Fallback: generate description now (slower path)
+          this.console.log(`[Exit Alert] No prefetch available, generating now`);
+          const mediaObject = this.snapshotCache.get(tracked.globalId);
+          spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
+          this.console.log(`[Exit Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+        }
 
         await this.alertManager.checkAndAlert('property_exit', current, {
           cameraId: sighting.cameraId,
@@ -700,8 +740,9 @@ export class TrackingEngine {
           usedLlm: spatialResult.usedLlm,
         });
 
-        // Clean up cached snapshot after exit alert
+        // Clean up cached snapshot and pending descriptions after exit alert
         this.snapshotCache.delete(tracked.globalId);
+        this.pendingDescriptions.delete(tracked.globalId);
       }
       this.pendingTimers.delete(tracked.globalId);
     }, this.config.correlationWindow);
@@ -724,8 +765,9 @@ export class TrackingEngine {
           `(not seen for ${Math.round(timeSinceSeen / 1000)}s)`
         );
 
-        // Clean up cached snapshot
+        // Clean up cached snapshot and pending descriptions
         this.snapshotCache.delete(tracked.globalId);
+        this.pendingDescriptions.delete(tracked.globalId);
 
         this.alertManager.checkAndAlert('lost_tracking', tracked, {
           objectClass: tracked.className,