@blueharford/scrypted-spatial-awareness 0.6.13 → 0.6.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.13",
+  "version": "0.6.15",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/topology-discovery.ts

@@ -23,6 +23,8 @@ import {
   DiscoveryStatus,
   DEFAULT_DISCOVERY_STATUS,
   RATE_LIMIT_WARNING_THRESHOLD,
+  DistanceEstimate,
+  distanceToFeet,
 } from '../models/discovery';
 import {
   CameraTopology,
@@ -40,60 +42,141 @@ interface ChatCompletionDevice extends ScryptedDevice {
 }
 
 /** Scene analysis prompt for single camera */
-const SCENE_ANALYSIS_PROMPT = `Analyze this security camera image and identify what you see.
+const SCENE_ANALYSIS_PROMPT = `You are analyzing a security camera image. Describe EVERYTHING you can see in detail.
+
+## INSTRUCTIONS
+Look at this image carefully and identify ALL visible objects, structures, and areas. Be thorough - even small or partially visible items are important for security awareness.
+
+## 1. LANDMARKS - List EVERY distinct object or feature you can see:
+
+**Structures** (buildings, parts of buildings):
+- Houses, garages, sheds, porches, decks, patios, carports, gazebos
+- Walls, pillars, columns, railings, stairs, steps
+
+**Vegetation** (plants, trees, landscaping):
+- Trees (describe type if identifiable: oak, palm, pine, etc.)
+- Bushes, shrubs, hedges
+- Flower beds, gardens, planters, potted plants
+- Grass/lawn areas, mulch beds
+
+**Boundaries & Barriers**:
+- Fences (wood, chain-link, aluminum, vinyl, iron, privacy)
+- Walls (brick, stone, concrete, retaining)
+- Gates, gate posts
+- Hedges used as boundaries
+
+**Access Points & Pathways**:
+- Doors (front, side, garage, screen)
+- Driveways (concrete, asphalt, gravel, pavers)
+- Walkways, sidewalks, paths, stepping stones
+- Stairs, ramps, porches
+
+**Utility & Fixtures**:
+- Mailboxes, package boxes
+- Light fixtures, lamp posts, solar lights
+- A/C units, utility boxes, meters
+- Trash cans, recycling bins
+- Hoses, spigots, sprinklers
+
+**Outdoor Items**:
+- Vehicles (cars, trucks, motorcycles, boats, trailers)
+- Furniture (chairs, tables, benches, swings)
+- Grills, fire pits, outdoor kitchens
+- Play equipment, trampolines, pools
+- Decorations, flags, signs
+
+**Off-Property Elements** (important for security context):
+- Street, road, sidewalk
+- Neighbor's property/fence/house
+- Public areas visible
+
+For EACH landmark, estimate its DISTANCE from the camera:
+- "close" = 0-10 feet (within arm's reach of camera)
+- "near" = 10-30 feet
+- "medium" = 30-60 feet
+- "far" = 60-100 feet
+- "distant" = 100+ feet (edge of property or beyond)
+
+## 2. ZONES - Identify distinct AREAS visible:
+- Front yard, backyard, side yard
+- Driveway, parking area
+- Patio, deck, porch
+- Garden area, lawn
+- Street/road
+- Neighbor's yard
+
+For each zone, estimate what percentage of the image it covers (0.0 to 1.0).
+
+## 3. EDGES - What's at each edge of the frame:
+This helps understand what's just out of view.
+
+## 4. CAMERA CONTEXT:
+- Estimated mounting height (ground level, 8ft, 12ft, roofline, etc.)
+- Approximate field of view (narrow, medium, wide)
+- Facing direction if determinable (north, south, street-facing, etc.)
 
-1. LANDMARKS - Identify fixed features visible:
-   - Structures (house, garage, shed, porch, deck)
-   - Features (mailbox, tree, pool, garden, fountain)
-   - Access points (door, gate, driveway entrance, walkway)
-   - Boundaries (fence, wall, hedge)
-
-2. ZONES - Identify area types visible:
-   - What type of area is this? (front yard, backyard, driveway, street, patio, walkway)
-   - Estimate what percentage of the frame each zone covers (0.0 to 1.0)
-
-3. EDGES - What's visible at the frame edges:
-   - Top edge: (sky, roof, trees, etc.)
-   - Left edge: (fence, neighbor, street, etc.)
-   - Right edge: (fence, garage, etc.)
-   - Bottom edge: (ground, driveway, grass, etc.)
-
-4. ORIENTATION - Estimate camera facing direction based on shadows, sun position, or landmarks
-
-Respond with ONLY valid JSON in this exact format:
+Respond with ONLY valid JSON:
 {
   "landmarks": [
-    {"name": "Front Door", "type": "access", "confidence": 0.9, "description": "White front door with black frame"}
+    {"name": "Mailbox", "type": "feature", "distance": "medium", "confidence": 0.95, "description": "Black metal mailbox on wooden post, approximately 40 feet from camera"},
+    {"name": "Aluminum Fence", "type": "boundary", "distance": "near", "confidence": 0.9, "description": "Silver aluminum fence running along left side of property, about 15-20 feet away"},
+    {"name": "Large Oak Tree", "type": "feature", "distance": "far", "confidence": 0.85, "description": "Mature oak tree near property line, roughly 80 feet from camera"}
   ],
   "zones": [
-    {"name": "Front Yard", "type": "yard", "coverage": 0.4, "description": "Grass lawn area"}
+    {"name": "Front Yard", "type": "yard", "coverage": 0.5, "description": "Grass lawn with some bare patches"},
+    {"name": "Driveway", "type": "driveway", "coverage": 0.25, "description": "Concrete driveway leading to garage"}
   ],
-  "edges": {"top": "sky with clouds", "left": "fence and trees", "right": "garage wall", "bottom": "concrete walkway"},
-  "orientation": "north"
-}`;
+  "edges": {
+    "top": "sky, tree canopy",
+    "left": "aluminum fence, neighbor's yard beyond",
+    "right": "side of house, garage door",
+    "bottom": "concrete walkway, grass edge"
+  },
+  "cameraContext": {
+    "mountHeight": "8 feet",
+    "fieldOfView": "wide",
+    "facingDirection": "street-facing"
+  }
+}
+
+BE THOROUGH. List every distinct item you can identify. A typical outdoor scene should have 5-15+ landmarks.`;
 
 /** Multi-camera correlation prompt */
-const CORRELATION_PROMPT = `I have scene analyses from multiple security cameras at the same property. Help me correlate them to understand the property layout.
+const CORRELATION_PROMPT = `I have detailed scene analyses from multiple security cameras at the same property. Help me understand which landmarks appear in multiple camera views.
 
 CAMERA SCENES:
 {scenes}
 
-Identify:
-1. Shared landmarks - Features that appear in multiple camera views
-2. Camera connections - How someone could move between camera views and estimated walking time
-3. Overall layout - Describe the property layout based on what you see
+## PRIORITY ORDER (most important first):
+
+### 1. SHARED LANDMARKS (HIGHEST PRIORITY)
+Identify features that are visible from MULTIPLE cameras. This is crucial for understanding the property layout.
+- Look for the SAME fence, tree, mailbox, driveway, structure, etc. appearing in different camera views
+- Even partial visibility counts (e.g., a tree visible in full from one camera and just the edge from another)
+- Include landmarks that are at the boundary between camera views
+
+### 2. PROPERTY LAYOUT
+Based on what each camera sees and their overlapping features, describe:
+- Which areas each camera covers
+- How the cameras relate spatially (e.g., "Camera A looks toward Camera B's direction")
+- Overall property shape and features
+
+### 3. CONNECTIONS (Lower Priority)
+Only if clearly determinable, suggest walking paths between camera views.
 
 IMPORTANT: For camera references, use the EXACT device ID shown in parentheses (e.g., "device_123"), NOT the camera name.
 
 Respond with ONLY valid JSON:
 {
   "sharedLandmarks": [
-    {"name": "Driveway", "type": "access", "seenByCameras": ["device_123", "device_456"], "confidence": 0.8, "description": "Concrete driveway"}
+    {"name": "Aluminum Fence", "type": "boundary", "seenByCameras": ["device_123", "device_456"], "confidence": 0.85, "description": "Silver aluminum fence visible on right edge of Camera A and left edge of Camera B"},
+    {"name": "Large Oak Tree", "type": "feature", "seenByCameras": ["device_123", "device_789"], "confidence": 0.9, "description": "Mature oak tree in front yard, visible from both front and side cameras"},
+    {"name": "Concrete Driveway", "type": "access", "seenByCameras": ["device_123", "device_456", "device_789"], "confidence": 0.95, "description": "Driveway visible from multiple angles"}
   ],
   "connections": [
-    {"from": "device_123", "to": "device_456", "transitSeconds": 10, "via": "driveway", "confidence": 0.7, "bidirectional": true}
+    {"from": "device_123", "to": "device_456", "transitSeconds": 8, "via": "along driveway", "confidence": 0.6, "bidirectional": true}
   ],
-  "layoutDescription": "Single-story house with front yard facing street, driveway on the left side, backyard accessible through side gate"
+  "layoutDescription": "Ranch-style house. Front camera covers front yard and street. Garage camera covers driveway entrance. Side camera covers side yard with aluminum fence separating from neighbor. Backyard camera shows deck and pool area."
 }`;
 
 export class TopologyDiscoveryEngine {
@@ -328,6 +411,7 @@ export class TopologyDiscoveryEngine {
                 name: l.name || 'Unknown',
                 type: this.mapLandmarkType(l.type),
                 confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
+                distance: this.mapDistance(l.distance),
                 description: l.description || '',
                 boundingBox: l.boundingBox,
               }));
@@ -449,6 +533,17 @@ export class TopologyDiscoveryEngine {
     return 'unknown';
   }
 
+  /** Map LLM distance to our type */
+  private mapDistance(distance: string): DistanceEstimate {
+    const dist = distance?.toLowerCase();
+    if (dist?.includes('close')) return 'close';
+    if (dist?.includes('near')) return 'near';
+    if (dist?.includes('medium')) return 'medium';
+    if (dist?.includes('far') && !dist?.includes('distant')) return 'far';
+    if (dist?.includes('distant')) return 'distant';
+    return 'medium'; // Default to medium if not specified
+  }
+
   /** Resolve a camera reference (name or deviceId) to its deviceId */
   private resolveCameraRef(ref: string): string | null {
     if (!this.topology?.cameras || !ref) return null;
@@ -644,9 +739,14 @@ export class TopologyDiscoveryEngine {
   private generateSuggestionsFromAnalysis(analysis: SceneAnalysis): void {
     if (!analysis.isValid) return;
 
+    this.console.log(`[Discovery] Generating suggestions from ${analysis.landmarks.length} landmarks, ${analysis.zones.length} zones`);
+
     // Generate landmark suggestions
     for (const landmark of analysis.landmarks) {
       if (landmark.confidence >= this.config.minLandmarkConfidence) {
+        // Calculate distance in feet from distance estimate
+        const distanceFeet = landmark.distance ? distanceToFeet(landmark.distance) : 50;
+
         const suggestion: DiscoverySuggestion = {
           id: `landmark_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
           type: 'landmark',
@@ -659,27 +759,31 @@ export class TopologyDiscoveryEngine {
             type: landmark.type,
             description: landmark.description,
             visibleFromCameras: [analysis.cameraId],
-            // Include bounding box for positioning (will be used by applyDiscoverySuggestion)
+            // Include extra metadata for positioning
             boundingBox: landmark.boundingBox,
-          } as any, // boundingBox is extra metadata not in Landmark interface
+            distance: landmark.distance,
+            distanceFeet: distanceFeet,
+          } as any, // Extra metadata not in base Landmark interface
         };
         this.suggestions.set(suggestion.id, suggestion);
+        this.console.log(`[Discovery] Landmark suggestion: ${landmark.name} (${landmark.type}, ${landmark.distance || 'medium'}, ~${distanceFeet}ft)`);
       }
     }
 
-    // Generate zone suggestions
+    // Generate zone suggestions (even for smaller coverage - 10% is enough)
     for (const zone of analysis.zones) {
-      if (zone.coverage >= 0.2) {
+      if (zone.coverage >= 0.1) {
         const suggestion: DiscoverySuggestion = {
           id: `zone_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
           type: 'zone',
           timestamp: Date.now(),
           sourceCameras: [analysis.cameraId],
-          confidence: 0.7,
+          confidence: Math.min(0.9, 0.5 + zone.coverage), // Higher coverage = higher confidence
           status: 'pending',
           zone: zone,
         };
         this.suggestions.set(suggestion.id, suggestion);
+        this.console.log(`[Discovery] Zone suggestion: ${zone.name} (${zone.type}, ${Math.round(zone.coverage * 100)}% coverage)`);
       }
     }
   }

package/src/core/tracking-engine.ts

@@ -141,6 +141,8 @@ export class TrackingEngine {
   // ==================== Snapshot Cache ====================
   /** Cached snapshots for tracked objects (for faster notifications) */
   private snapshotCache: Map<GlobalTrackingId, MediaObject> = new Map();
+  /** Pending LLM description promises (started when snapshot is captured) */
+  private pendingDescriptions: Map<GlobalTrackingId, Promise<SpatialReasoningResult>> = new Map();
 
   constructor(
     topology: CameraTopology,
@@ -555,18 +557,27 @@ export class TrackingEngine {
       // Check if we've already alerted for this object
       if (this.isInAlertCooldown(globalId)) return;
 
-      // Use cached snapshot (captured when object was first detected)
-      let mediaObject = this.snapshotCache.get(globalId);
-      this.console.log(`[Entry Alert] Using cached snapshot: ${!!mediaObject}`);
+      // Use prefetched LLM result if available (started when snapshot was captured)
+      let spatialResult: SpatialReasoningResult;
+      const pendingDescription = this.pendingDescriptions.get(globalId);
 
-      // Generate spatial description (now async with LLM support)
-      this.console.log(`[Entry Alert] Calling generateEntryDescription with mediaObject=${!!mediaObject}`);
-      const spatialResult = await this.spatialReasoning.generateEntryDescription(
-        tracked,
-        sighting.cameraId,
-        mediaObject
-      );
-      this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+      if (pendingDescription) {
+        this.console.log(`[Entry Alert] Using prefetched LLM result for ${globalId.slice(0, 8)}`);
+        try {
+          spatialResult = await pendingDescription;
+          this.console.log(`[Entry Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+        } catch (e) {
+          this.console.warn(`[Entry Alert] Prefetch failed, generating fallback: ${e}`);
+          spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId);
+        }
+        this.pendingDescriptions.delete(globalId);
+      } else {
+        // Fallback: generate description now (slower path)
+        this.console.log(`[Entry Alert] No prefetch available, generating now`);
+        const mediaObject = this.snapshotCache.get(globalId);
+        spatialResult = await this.spatialReasoning.generateEntryDescription(tracked, sighting.cameraId, mediaObject);
+        this.console.log(`[Entry Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+      }
 
       if (isEntryPoint) {
         // Entry point - generate property entry alert
@@ -599,8 +610,12 @@ export class TrackingEngine {
     }, this.config.loiteringThreshold);
   }
 
-  /** Capture and cache a snapshot for a tracked object */
-  private async captureAndCacheSnapshot(globalId: GlobalTrackingId, cameraId: string): Promise<void> {
+  /** Capture and cache a snapshot for a tracked object, and start LLM analysis immediately */
+  private async captureAndCacheSnapshot(
+    globalId: GlobalTrackingId,
+    cameraId: string,
+    eventType: 'entry' | 'exit' | 'movement' = 'entry'
+  ): Promise<void> {
     try {
       const camera = systemManager.getDeviceById<Camera>(cameraId);
       if (camera?.interfaces?.includes(ScryptedInterface.Camera)) {
@@ -608,6 +623,24 @@ export class TrackingEngine {
         if (mediaObject) {
           this.snapshotCache.set(globalId, mediaObject);
           this.console.log(`[Snapshot] Cached snapshot for ${globalId.slice(0, 8)} from ${cameraId}`);
+
+          // Start LLM analysis immediately in parallel (don't await)
+          const tracked = this.state.getObject(globalId);
+          if (tracked && this.config.useLlmDescriptions) {
+            this.console.log(`[LLM Prefetch] Starting ${eventType} analysis for ${globalId.slice(0, 8)}`);
+            const descriptionPromise = eventType === 'exit'
+              ? this.spatialReasoning.generateExitDescription(tracked, cameraId, mediaObject)
+              : this.spatialReasoning.generateEntryDescription(tracked, cameraId, mediaObject);
+
+            this.pendingDescriptions.set(globalId, descriptionPromise);
+
+            // Log when complete (for debugging)
+            descriptionPromise.then(result => {
+              this.console.log(`[LLM Prefetch] ${eventType} analysis ready for ${globalId.slice(0, 8)}: "${result.description.substring(0, 40)}..."`);
+            }).catch(e => {
+              this.console.warn(`[LLM Prefetch] Failed for ${globalId.slice(0, 8)}: ${e}`);
+            });
+          }
         }
       }
     } catch (e) {
@@ -663,8 +696,9 @@ export class TrackingEngine {
     this.state.markPending(tracked.globalId);
 
     // Capture a fresh snapshot now while object is still visible (before they leave)
+    // Also starts LLM analysis immediately in parallel
     if (this.config.useLlmDescriptions) {
-      this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId).catch(e => {
+      this.captureAndCacheSnapshot(tracked.globalId, sighting.cameraId, 'exit').catch(e => {
         this.console.warn(`[Exit Snapshot] Failed to update snapshot: ${e}`);
       });
     }
@@ -675,21 +709,27 @@ export class TrackingEngine {
       if (current && current.state === 'pending') {
         this.state.markExited(tracked.globalId, sighting.cameraId, sighting.cameraName);
 
-        // Use cached snapshot (captured when exit was first detected, while object was still visible)
-        let mediaObject = this.snapshotCache.get(tracked.globalId);
-        this.console.log(`[Exit Alert] Using cached snapshot: ${!!mediaObject}`);
-
-        // Generate rich exit description using topology context (now async with LLM support)
-        this.console.log(`[Exit Alert] Calling generateExitDescription with mediaObject=${!!mediaObject}`);
-        const spatialResult = await this.spatialReasoning.generateExitDescription(
-          current,
-          sighting.cameraId,
-          mediaObject
-        );
-
-        this.console.log(
-          `[Exit Alert] Object ${tracked.globalId.slice(0, 8)} exited: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`
-        );
+        // Use prefetched LLM result if available (started when exit was first detected)
+        let spatialResult: SpatialReasoningResult;
+        const pendingDescription = this.pendingDescriptions.get(tracked.globalId);
+
+        if (pendingDescription) {
+          this.console.log(`[Exit Alert] Using prefetched LLM result for ${tracked.globalId.slice(0, 8)}`);
+          try {
+            spatialResult = await pendingDescription;
+            this.console.log(`[Exit Alert] Prefetch result: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+          } catch (e) {
+            this.console.warn(`[Exit Alert] Prefetch failed, generating fallback: ${e}`);
+            spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId);
+          }
+          this.pendingDescriptions.delete(tracked.globalId);
+        } else {
+          // Fallback: generate description now (slower path)
+          this.console.log(`[Exit Alert] No prefetch available, generating now`);
+          const mediaObject = this.snapshotCache.get(tracked.globalId);
+          spatialResult = await this.spatialReasoning.generateExitDescription(current, sighting.cameraId, mediaObject);
+          this.console.log(`[Exit Alert] Got description: "${spatialResult.description.substring(0, 60)}...", usedLlm=${spatialResult.usedLlm}`);
+        }
 
         await this.alertManager.checkAndAlert('property_exit', current, {
           cameraId: sighting.cameraId,
@@ -700,8 +740,9 @@ export class TrackingEngine {
           usedLlm: spatialResult.usedLlm,
         });
 
-        // Clean up cached snapshot after exit alert
+        // Clean up cached snapshot and pending descriptions after exit alert
         this.snapshotCache.delete(tracked.globalId);
+        this.pendingDescriptions.delete(tracked.globalId);
       }
       this.pendingTimers.delete(tracked.globalId);
     }, this.config.correlationWindow);
@@ -724,8 +765,9 @@ export class TrackingEngine {
           `(not seen for ${Math.round(timeSinceSeen / 1000)}s)`
         );
 
-        // Clean up cached snapshot
+        // Clean up cached snapshot and pending descriptions
         this.snapshotCache.delete(tracked.globalId);
+        this.pendingDescriptions.delete(tracked.globalId);
 
         this.alertManager.checkAndAlert('lost_tracking', tracked, {
           objectClass: tracked.className,

package/src/models/discovery.ts

@@ -58,6 +58,21 @@ export interface DiscoveredZone {
   boundingBox?: [number, number, number, number];
 }
 
+/** Distance estimate from camera */
+export type DistanceEstimate = 'close' | 'near' | 'medium' | 'far' | 'distant';
+
+/** Convert distance estimate to approximate feet */
+export function distanceToFeet(distance: DistanceEstimate): number {
+  switch (distance) {
+    case 'close': return 5;      // 0-10 feet
+    case 'near': return 20;      // 10-30 feet
+    case 'medium': return 45;    // 30-60 feet
+    case 'far': return 80;       // 60-100 feet
+    case 'distant': return 150;  // 100+ feet
+    default: return 50;
+  }
+}
+
 /** A landmark discovered in a camera view */
 export interface DiscoveredLandmark {
   /** Name of the landmark */
@@ -66,6 +81,8 @@ export interface DiscoveredLandmark {
   type: LandmarkType;
   /** Confidence score from LLM (0-1) */
   confidence: number;
+  /** Estimated distance from camera */
+  distance?: DistanceEstimate;
   /** Bounding box in normalized coordinates [x, y, width, height] (0-1) */
   boundingBox?: [number, number, number, number];
   /** Description from LLM analysis */