@blueharford/scrypted-spatial-awareness 0.6.12 → 0.6.14

package/out/main.nodejs.js

@@ -36943,6 +36943,11 @@ class TrackingEngine {
     trainingConfig = training_1.DEFAULT_TRAINING_CONFIG;
     /** Callback for training status updates */
     onTrainingStatusUpdate;
+    // ==================== Snapshot Cache ====================
+    /** Cached snapshots for tracked objects (for faster notifications) */
+    snapshotCache = new Map();
+    /** Pending LLM description promises (started when snapshot is captured) */
+    pendingDescriptions = new Map();
     constructor(topology, state, alertManager, config, console) {
         this.topology = topology;
         this.state = state;
@@ -37166,6 +37171,12 @@ class TrackingEngine {
             const lastSighting = (0, tracked_object_1.getLastSighting)(tracked);
             if (lastSighting && lastSighting.cameraId !== sighting.cameraId) {
                 const transitDuration = sighting.timestamp - lastSighting.timestamp;
+                // Update cached snapshot from new camera (object is now visible here)
+                if (this.config.useLlmDescriptions) {
+                    this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId).catch(e => {
+                        this.console.warn(`[Transition Snapshot] Failed to update snapshot: ${e}`);
+                    });
+                }
                 // Add journey segment
                 this.state.addJourney(tracked.globalId, {
                     fromCameraId: lastSighting.cameraId,
@@ -37232,6 +37243,13 @@ class TrackingEngine {
     }
     /** Schedule an alert after loitering threshold passes */
     scheduleLoiteringAlert(globalId, sighting, isEntryPoint) {
+        // Capture snapshot IMMEDIATELY when object is first detected (don't wait for loitering threshold)
+        // This ensures we have a good image while the person/object is still in frame
+        if (this.config.useLlmDescriptions) {
+            this.captureAndCacheSnapshot(globalId, sighting.cameraId).catch(e => {
+                this.console.warn(`[Snapshot] Failed to cache initial snapshot: ${e}`);
+            });
+        }
         // Check after loitering threshold if object is still being tracked
         setTimeout(async () => {
             const tracked = this.state.getObject(globalId);
@@ -37240,26 +37258,28 @@ class TrackingEngine {
             // Check if we've already alerted for this object
             if (this.isInAlertCooldown(globalId))
                 return;
-            // Get snapshot for LLM description (if LLM is enabled)
-            let mediaObject;
-            this.console.log(`[Entry Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
-            if (this.config.useLlmDescriptions) {
+            // Use prefetched LLM result if available (started when snapshot was captured)
+            let spatialResult;
+            const pendingDescription = this.pendingDescriptions.get(globalId);
+            if (pendingDescription) {
+                this.console.log(`[Entry Alert] Using prefetched LLM result for ${globalId.slice(0, 8)}`);
                 try {
-                    const camera = systemManager.getDeviceById(sighting.cameraId);
-                    this.console.log(`[Entry Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)}`);
-                    if (camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)) {
-                        mediaObject = await camera.takePicture();
-                        this.console.log(`[Entry Alert] Got snapshot: ${!!mediaObject}`);
-                    }
+                    spatialResult = await pendingDescription;
+                    this.console.log(`[Entry Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
                 }
                 catch (e) {
-                    this.console.warn('[Entry Alert] Failed to get snapshot:', e);
+                    this.console.warn(`[Entry Alert] Prefetch failed, generating fallback: ${e}`);
+                    spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId);
                 }
+                this.pendingDescriptions.delete(globalId);
+            }
+            else {
+                // Fallback: generate description now (slower path)
+                this.console.log(`[Entry Alert] No prefetch available, generating now`);
+                const mediaObject = this.snapshotCache.get(globalId);
+                spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
+                this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
             }
-            // Generate spatial description (now async with LLM support)
-            this.console.log(`[Entry Alert] Calling generateEntryDescription with mediaObject=${!!mediaObject}`);
-            const spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
-            this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
             if (isEntryPoint) {
                 // Entry point - generate property entry alert
                 await this.alertManager.checkAndAlert('property_entry', tracked, {
@@ -37290,6 +37310,37 @@ class TrackingEngine {
             this.recordAlertTime(globalId);
         }, this.config.loiteringThreshold);
     }
+    /** Capture and cache a snapshot for a tracked object, and start LLM analysis immediately */
+    async captureAndCacheSnapshot(globalId, cameraId, eventType = 'entry') {
+        try {
+            const camera = systemManager.getDeviceById(cameraId);
+            if (camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)) {
+                const mediaObject = await camera.takePicture();
+                if (mediaObject) {
+                    this.snapshotCache.set(globalId, mediaObject);
+                    this.console.log(`[Snapshot] Cached snapshot for ${globalId.slice(0, 8)} from ${cameraId}`);
+                    // Start LLM analysis immediately in parallel (don't await)
+                    const tracked = this.state.getObject(globalId);
+                    if (tracked && this.config.useLlmDescriptions) {
+                        this.console.log(`[LLM Prefetch] Starting ${eventType} analysis for ${globalId.slice(0, 8)}`);
+                        const descriptionPromise = eventType === 'exit'
+                            ? this.spatialReasoning.generateExitDescription(tracked, cameraId, mediaObject)
+                            : this.spatialReasoning.generateEntryDescription(tracked, cameraId, mediaObject);
+                        this.pendingDescriptions.set(globalId, descriptionPromise);
+                        // Log when complete (for debugging)
+                        descriptionPromise.then(result => {
+                            this.console.log(`[LLM Prefetch] ${eventType} analysis ready for ${globalId.slice(0, 8)}: "${result.description.substring(0, 40)}..."`);
+                        }).catch(e => {
+                            this.console.warn(`[LLM Prefetch] Failed for ${globalId.slice(0, 8)}: ${e}`);
+                        });
+                    }
+                }
+            }
+        }
+        catch (e) {
+            this.console.warn(`[Snapshot] Failed to capture snapshot: ${e}`);
+        }
+    }
     /** Attempt to correlate a sighting with existing tracked objects */
     async correlateDetection(sighting) {
         const activeObjects = this.state.getActiveObjects();
@@ -37329,31 +37380,40 @@ class TrackingEngine {
     handlePotentialExit(tracked, sighting) {
         // Mark as pending and set timer
         this.state.markPending(tracked.globalId);
+        // Capture a fresh snapshot now while object is still visible (before they leave)
+        // Also starts LLM analysis immediately in parallel
+        if (this.config.useLlmDescriptions) {
+            this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId, 'exit').catch(e => {
+                this.console.warn(`[Exit Snapshot] Failed to update snapshot: ${e}`);
+            });
+        }
         // Wait for correlation window before marking as exited
         const timer = setTimeout(async () => {
             const current = this.state.getObject(tracked.globalId);
             if (current && current.state === 'pending') {
                 this.state.markExited(tracked.globalId, sighting.cameraId, sighting.cameraName);
-                // Get snapshot for LLM description (if LLM is enabled)
-                let mediaObject;
-                this.console.log(`[Exit Alert] useLlmDescriptions=${this.config.useLlmDescriptions}`);
-                if (this.config.useLlmDescriptions) {
+                // Use prefetched LLM result if available (started when exit was first detected)
+                let spatialResult;
+                const pendingDescription = this.pendingDescriptions.get(tracked.globalId);
+                if (pendingDescription) {
+                    this.console.log(`[Exit Alert] Using prefetched LLM result for ${tracked.globalId.slice(0, 8)}`);
                     try {
-                        const camera = systemManager.getDeviceById(sighting.cameraId);
-                        this.console.log(`[Exit Alert] Camera ${sighting.cameraId} has Camera interface: ${camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)}`);
-                        if (camera?.interfaces?.includes(sdk_1.ScryptedInterface.Camera)) {
-                            mediaObject = await camera.takePicture();
-                            this.console.log(`[Exit Alert] Got snapshot: ${!!mediaObject}`);
-                        }
+                        spatialResult = await pendingDescription;
+                        this.console.log(`[Exit Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
                     }
                     catch (e) {
-                        this.console.warn('[Exit Alert] Failed to get snapshot:', e);
+                        this.console.warn(`[Exit Alert] Prefetch failed, generating fallback: ${e}`);
+                        spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId);
                     }
+                    this.pendingDescriptions.delete(tracked.globalId);
+                }
+                else {
+                    // Fallback: generate description now (slower path)
+                    this.console.log(`[Exit Alert] No prefetch available, generating now`);
+                    const mediaObject = this.snapshotCache.get(tracked.globalId);
+                    spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
+                    this.console.log(`[Exit Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
                 }
-                // Generate rich exit description using topology context (now async with LLM support)
-                this.console.log(`[Exit Alert] Calling generateExitDescription with mediaObject=${!!mediaObject}`);
-                const spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
-                this.console.log(`[Exit Alert] Object ${tracked.globalId.slice(0, 8)} exited: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
                 await this.alertManager.checkAndAlert('property_exit', current, {
                     cameraId: sighting.cameraId,
                     cameraName: sighting.cameraName,
@@ -37362,6 +37422,9 @@ class TrackingEngine {
                     involvedLandmarks: spatialResult.involvedLandmarks?.map(l => l.name),
                     usedLlm: spatialResult.usedLlm,
                 });
+                // Clean up cached snapshot and pending descriptions after exit alert
+                this.snapshotCache.delete(tracked.globalId);
+                this.pendingDescriptions.delete(tracked.globalId);
             }
             this.pendingTimers.delete(tracked.globalId);
         }, this.config.correlationWindow);
@@ -37377,6 +37440,9 @@ class TrackingEngine {
                 this.state.markLost(tracked.globalId);
                 this.console.log(`Object ${tracked.globalId.slice(0, 8)} marked as lost ` +
                     `(not seen for ${Math.round(timeSinceSeen / 1000)}s)`);
+                // Clean up cached snapshot and pending descriptions
+                this.snapshotCache.delete(tracked.globalId);
+                this.pendingDescriptions.delete(tracked.globalId);
                 this.alertManager.checkAndAlert('lost_tracking', tracked, {
                     objectClass: tracked.className,
                     objectLabel: tracked.label,