@blueharford/scrypted-spatial-awareness 0.6.3 → 0.6.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.3",
+  "version": "0.6.5",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/topology-discovery.ts

@@ -83,13 +83,15 @@ Identify:
 2. Camera connections - How someone could move between camera views and estimated walking time
 3. Overall layout - Describe the property layout based on what you see
 
+IMPORTANT: For camera references, use the EXACT device ID shown in parentheses (e.g., "device_123"), NOT the camera name.
+
 Respond with ONLY valid JSON:
 {
   "sharedLandmarks": [
-    {"name": "Driveway", "type": "access", "seenByCameras": ["camera1", "camera2"], "confidence": 0.8, "description": "Concrete driveway"}
+    {"name": "Driveway", "type": "access", "seenByCameras": ["device_123", "device_456"], "confidence": 0.8, "description": "Concrete driveway"}
   ],
   "connections": [
-    {"from": "camera1", "to": "camera2", "transitSeconds": 10, "via": "driveway", "confidence": 0.7, "bidirectional": true}
+    {"from": "device_123", "to": "device_456", "transitSeconds": 10, "via": "driveway", "confidence": 0.7, "bidirectional": true}
   ],
   "layoutDescription": "Single-story house with front yard facing street, driveway on the left side, backyard accessible through side gate"
 }`;
@@ -447,6 +449,40 @@ export class TopologyDiscoveryEngine {
     return 'unknown';
   }
 
+  /** Resolve a camera reference (name or deviceId) to its deviceId */
+  private resolveCameraRef(ref: string): string | null {
+    if (!this.topology?.cameras || !ref) return null;
+
+    // Try exact deviceId match first
+    const byId = this.topology.cameras.find(c => c.deviceId === ref);
+    if (byId) return byId.deviceId;
+
+    // Try exact name match
+    const byName = this.topology.cameras.find(c => c.name === ref);
+    if (byName) return byName.deviceId;
+
+    // Try case-insensitive name match
+    const refLower = ref.toLowerCase();
+    const byNameLower = this.topology.cameras.find(c => c.name.toLowerCase() === refLower);
+    if (byNameLower) return byNameLower.deviceId;
+
+    // Try partial name match (LLM might truncate or abbreviate)
+    const byPartial = this.topology.cameras.find(c =>
+      c.name.toLowerCase().includes(refLower) || refLower.includes(c.name.toLowerCase())
+    );
+    if (byPartial) return byPartial.deviceId;
+
+    this.console.warn(`[Discovery] Could not resolve camera reference: "${ref}"`);
+    return null;
+  }
+
+  /** Normalize camera references in an array to deviceIds */
+  private normalizeCameraRefs(refs: string[]): string[] {
+    return refs
+      .map(ref => this.resolveCameraRef(ref))
+      .filter((id): id is string => id !== null);
+  }
+
   /** Analyze all cameras and correlate findings */
   async runFullDiscovery(): Promise<TopologyCorrelation | null> {
     if (!this.topology?.cameras?.length) {
@@ -552,24 +588,42 @@ export class TopologyDiscoveryEngine {
           };
 
           if (Array.isArray(parsed.sharedLandmarks)) {
-            correlation.sharedLandmarks = parsed.sharedLandmarks.map((l: any) => ({
-              name: l.name || 'Unknown',
-              type: this.mapLandmarkType(l.type),
-              seenByCameras: Array.isArray(l.seenByCameras) ? l.seenByCameras : [],
-              confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
-              description: l.description,
-            }));
+            correlation.sharedLandmarks = parsed.sharedLandmarks.map((l: any) => {
+              // Normalize camera references to deviceIds
+              const rawRefs = Array.isArray(l.seenByCameras) ? l.seenByCameras : [];
+              const normalizedRefs = this.normalizeCameraRefs(rawRefs);
+              if (rawRefs.length > 0 && normalizedRefs.length === 0) {
+                this.console.warn(`[Discovery] Landmark "${l.name}" has no resolvable camera refs: ${JSON.stringify(rawRefs)}`);
+              }
+              return {
+                name: l.name || 'Unknown',
+                type: this.mapLandmarkType(l.type),
+                seenByCameras: normalizedRefs,
+                confidence: typeof l.confidence === 'number' ? l.confidence : 0.7,
+                description: l.description,
+              };
+            });
           }
 
           if (Array.isArray(parsed.connections)) {
-            correlation.suggestedConnections = parsed.connections.map((c: any) => ({
-              fromCameraId: c.from || c.fromCameraId || '',
-              toCameraId: c.to || c.toCameraId || '',
-              transitSeconds: typeof c.transitSeconds === 'number' ? c.transitSeconds : 15,
-              via: c.via || '',
-              confidence: typeof c.confidence === 'number' ? c.confidence : 0.6,
-              bidirectional: c.bidirectional !== false,
-            }));
+            correlation.suggestedConnections = parsed.connections.map((c: any) => {
+              // Normalize camera references to deviceIds
+              const fromRef = c.from || c.fromCameraId || '';
+              const toRef = c.to || c.toCameraId || '';
+              const fromId = this.resolveCameraRef(fromRef);
+              const toId = this.resolveCameraRef(toRef);
+              if (!fromId || !toId) {
+                this.console.warn(`[Discovery] Connection has unresolvable camera refs: from="${fromRef}" to="${toRef}"`);
+              }
+              return {
+                fromCameraId: fromId || fromRef,
+                toCameraId: toId || toRef,
+                transitSeconds: typeof c.transitSeconds === 'number' ? c.transitSeconds : 15,
+                via: c.via || '',
+                confidence: typeof c.confidence === 'number' ? c.confidence : 0.6,
+                bidirectional: c.bidirectional !== false,
+              };
+            });
           }
 
           this.console.log(`[Discovery] Correlation found ${correlation.sharedLandmarks.length} shared landmarks, ${correlation.suggestedConnections.length} connections`);
@@ -605,7 +659,9 @@ export class TopologyDiscoveryEngine {
             type: landmark.type,
             description: landmark.description,
             visibleFromCameras: [analysis.cameraId],
-          },
+            // Include bounding box for positioning (will be used by applyDiscoverySuggestion)
+            boundingBox: landmark.boundingBox,
+          } as any, // boundingBox is extra metadata not in Landmark interface
         };
         this.suggestions.set(suggestion.id, suggestion);
       }

package/src/main.ts

@@ -1829,21 +1829,29 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       if (!position || (position.x === 0 && position.y === 0)) {
         // Debug logging
         this.console.log(`[Discovery] Processing landmark "${suggestion.landmark.name}"`);
+        this.console.log(`[Discovery] sourceCameras: ${JSON.stringify(suggestion.sourceCameras)}`);
         this.console.log(`[Discovery] visibleFromCameras: ${JSON.stringify(suggestion.landmark.visibleFromCameras)}`);
         this.console.log(`[Discovery] Available cameras: ${topology.cameras.map(c => `${c.name}(${c.deviceId})`).join(', ')}`);
 
-        // Find a camera that can see this landmark - use flexible matching (deviceId, name, or case-insensitive)
+        // Find a camera that can see this landmark
+        // PREFER sourceCameras (set from analysis.cameraId) over visibleFromCameras (from LLM parsing)
+        const sourceCameraRef = suggestion.sourceCameras?.[0];
         const visibleCameraRef = suggestion.landmark.visibleFromCameras?.[0];
-        const camera = visibleCameraRef ? topology.cameras.find(c =>
-          c.deviceId === visibleCameraRef ||
-          c.name === visibleCameraRef ||
-          c.name.toLowerCase() === visibleCameraRef.toLowerCase()
+        const cameraRef = sourceCameraRef || visibleCameraRef;
+
+        // Use flexible matching (deviceId, name, or case-insensitive)
+        const camera = cameraRef ? topology.cameras.find(c =>
+          c.deviceId === cameraRef ||
+          c.name === cameraRef ||
+          c.name.toLowerCase() === cameraRef.toLowerCase()
         ) : null;
 
         if (camera) {
-          this.console.log(`[Discovery] Matched camera: ${camera.name}, position: ${JSON.stringify(camera.floorPlanPosition)}, fov: ${JSON.stringify(camera.fov)}`);
+          this.console.log(`[Discovery] Matched camera: ${camera.name} (${camera.deviceId})`);
+          this.console.log(`[Discovery] Camera position: ${JSON.stringify(camera.floorPlanPosition)}`);
+          this.console.log(`[Discovery] Camera FOV: ${JSON.stringify(camera.fov)}`);
         } else {
-          this.console.warn(`[Discovery] No camera matched for "${visibleCameraRef}"`);
+          this.console.warn(`[Discovery] No camera matched for ref="${cameraRef}" (source="${sourceCameraRef}", visible="${visibleCameraRef}")`);
         }
 
         if (camera?.floorPlanPosition) {
@@ -1853,21 +1861,47 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
           const range = fov.range || 80;
           const fovAngle = fov.angle || 90;
 
-          // Count existing landmarks from this camera to spread them out
-          const cameraDeviceId = camera.deviceId;
-          const existingFromCamera = (topology.landmarks || []).filter(l =>
-            l.visibleFromCameras?.includes(cameraDeviceId) ||
-            l.visibleFromCameras?.includes(camera.name)
-          ).length;
-
           // Calculate position in front of camera within its FOV
           // Convert direction to radians (0 = up/north, 90 = right/east)
           const dirRad = (direction - 90) * Math.PI / 180;
           const halfFov = (fovAngle / 2) * Math.PI / 180;
 
-          // Spread landmarks across the FOV cone at varying distances
-          const angleOffset = (existingFromCamera % 3 - 1) * halfFov * 0.6; // -0.6, 0, +0.6 of half FOV
-          const distanceMultiplier = 0.5 + (existingFromCamera % 2) * 0.3; // 50% or 80% of range
+          // Use bounding box if available to position landmark accurately within FOV
+          // boundingBox format: [x, y, width, height] normalized 0-1
+          const bbox = (suggestion.landmark as any).boundingBox as [number, number, number, number] | undefined;
+
+          let angleOffset: number;
+          let distanceMultiplier: number;
+
+          if (bbox && bbox.length >= 2) {
+            // Use bounding box center to determine position in FOV
+            const bboxCenterX = bbox[0] + (bbox[2] || 0) / 2; // 0 = left edge, 1 = right edge
+            const bboxCenterY = bbox[1] + (bbox[3] || 0) / 2; // 0 = top (far), 1 = bottom (close)
+
+            // Map X position to angle within FOV
+            // bboxCenterX 0 → left side of FOV (-halfFov)
+            // bboxCenterX 0.5 → center of FOV (0)
+            // bboxCenterX 1 → right side of FOV (+halfFov)
+            angleOffset = (bboxCenterX - 0.5) * 2 * halfFov;
+
+            // Map Y position to distance from camera
+            // bboxCenterY 0 (top of frame) → far from camera (90% of range)
+            // bboxCenterY 1 (bottom of frame) → close to camera (30% of range)
+            distanceMultiplier = 0.9 - (bboxCenterY * 0.6);
+
+            this.console.log(`[Discovery] Using bounding box [${bbox.join(',')}] → center (${bboxCenterX.toFixed(2)}, ${bboxCenterY.toFixed(2)})`);
+          } else {
+            // Fallback: spread landmarks across FOV if no bounding box
+            const cameraDeviceId = camera.deviceId;
+            const existingFromCamera = (topology.landmarks || []).filter(l =>
+              l.visibleFromCameras?.includes(cameraDeviceId) ||
+              l.visibleFromCameras?.includes(camera.name)
+            ).length;
+
+            angleOffset = (existingFromCamera % 3 - 1) * halfFov * 0.6;
+            distanceMultiplier = 0.5 + (existingFromCamera % 2) * 0.3;
+            this.console.log(`[Discovery] No bounding box, using fallback spread (existing: ${existingFromCamera})`);
+          }
 
           const finalAngle = dirRad + angleOffset;
           const distance = range * distanceMultiplier;
@@ -1877,7 +1911,7 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
             y: camera.floorPlanPosition.y + Math.sin(finalAngle) * distance,
           };
 
-          this.console.log(`[Discovery] Placing landmark "${suggestion.landmark.name}" in ${camera.name}'s FOV: dir=${direction}°, dist=${distance.toFixed(0)}px`);
+          this.console.log(`[Discovery] Placing landmark "${suggestion.landmark.name}" in ${camera.name}'s FOV: dir=${direction}°, angle=${(angleOffset * 180 / Math.PI).toFixed(1)}°, dist=${distance.toFixed(0)}px`);
         } else {
           // Position in a grid pattern starting from center
           const landmarkCount = topology.landmarks?.length || 0;
@@ -1917,10 +1951,22 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
 
       // Find cameras that see this zone
       const sourceCameras = suggestion.sourceCameras || [];
-      const camera = sourceCameras[0]
-        ? topology.cameras.find(c => c.deviceId === sourceCameras[0] || c.name === sourceCameras[0])
+      const cameraRef = sourceCameras[0];
+      const camera = cameraRef
+        ? topology.cameras.find(c =>
+            c.deviceId === cameraRef ||
+            c.name === cameraRef ||
+            c.name.toLowerCase() === cameraRef.toLowerCase()
+          )
         : null;
 
+      this.console.log(`[Discovery] Processing zone "${zone.name}" from camera ref="${cameraRef}"`);
+      if (camera) {
+        this.console.log(`[Discovery] Matched camera: ${camera.name} (${camera.deviceId})`);
+      } else if (cameraRef) {
+        this.console.warn(`[Discovery] No camera matched for zone source ref="${cameraRef}"`);
+      }
+
       // Create zone polygon WITHIN the camera's field of view
       let polygon: { x: number; y: number }[] = [];
       const timestamp = Date.now();
@@ -1936,35 +1982,65 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
         const dirRad = (direction - 90) * Math.PI / 180;
         const halfFov = (fovAngle / 2) * Math.PI / 180;
 
-        // Count existing zones from this camera to offset new ones
-        const existingFromCamera = (topology.drawnZones || []).filter((z: any) =>
-          z.linkedCameras?.includes(sourceCameras[0])
-        ).length;
+        const camX = camera.floorPlanPosition.x;
+        const camY = camera.floorPlanPosition.y;
 
-        // Create a wedge-shaped zone within the camera's FOV
-        // Offset based on existing zones to avoid overlap
-        const innerRadius = range * 0.3 + existingFromCamera * 20;
-        const outerRadius = range * 0.8 + existingFromCamera * 20;
+        // Use bounding box if available to position zone accurately within FOV
+        const bbox = zone.boundingBox; // [x, y, width, height] normalized 0-1
 
-        // Use a portion of the FOV for each zone
-        const zoneSpread = halfFov * 0.7; // 70% of half FOV
+        let innerRadius: number;
+        let outerRadius: number;
+        let angleStart: number;
+        let angleEnd: number;
 
-        const camX = camera.floorPlanPosition.x;
-        const camY = camera.floorPlanPosition.y;
+        if (bbox && bbox.length >= 4) {
+          // Map bounding box to position within FOV
+          const bboxLeft = bbox[0];
+          const bboxRight = bbox[0] + bbox[2];
+          const bboxTop = bbox[1];
+          const bboxBottom = bbox[1] + bbox[3];
+
+          // Map X to angle within FOV (0 = left edge, 1 = right edge)
+          angleStart = dirRad + (bboxLeft - 0.5) * 2 * halfFov;
+          angleEnd = dirRad + (bboxRight - 0.5) * 2 * halfFov;
 
-        // Create arc polygon (wedge shape)
+          // Map Y to distance (0 = far, 1 = close)
+          innerRadius = range * (0.9 - bboxBottom * 0.6);
+          outerRadius = range * (0.9 - bboxTop * 0.6);
+
+          // Ensure min size
+          if (outerRadius - innerRadius < 20) {
+            outerRadius = innerRadius + 20;
+          }
+
+          this.console.log(`[Discovery] Zone "${zone.name}" using bbox [${bbox.join(',')}] → angles ${(angleStart * 180 / Math.PI).toFixed(1)}° to ${(angleEnd * 180 / Math.PI).toFixed(1)}°`);
+        } else {
+          // Fallback: wedge-shaped zone offset by existing count
+          const existingFromCamera = (topology.drawnZones || []).filter((z: any) =>
+            z.linkedCameras?.includes(sourceCameras[0])
+          ).length;
+
+          innerRadius = range * 0.3 + existingFromCamera * 20;
+          outerRadius = range * 0.8 + existingFromCamera * 20;
+          angleStart = dirRad - halfFov * 0.7;
+          angleEnd = dirRad + halfFov * 0.7;
+
+          this.console.log(`[Discovery] Zone "${zone.name}" using fallback spread (existing: ${existingFromCamera})`);
+        }
+
+        // Create arc polygon
         const steps = 8;
-        // Inner arc (from left to right)
+        // Inner arc (from start angle to end angle)
         for (let i = 0; i <= steps; i++) {
-          const angle = dirRad - zoneSpread + (zoneSpread * 2 * i / steps);
+          const angle = angleStart + (angleEnd - angleStart) * i / steps;
           polygon.push({
             x: camX + Math.cos(angle) * innerRadius,
             y: camY + Math.sin(angle) * innerRadius,
           });
         }
-        // Outer arc (from right to left)
+        // Outer arc (from end angle to start angle)
         for (let i = steps; i >= 0; i--) {
-          const angle = dirRad - zoneSpread + (zoneSpread * 2 * i / steps);
+          const angle = angleStart + (angleEnd - angleStart) * i / steps;
           polygon.push({
             x: camX + Math.cos(angle) * outerRadius,
             y: camY + Math.sin(angle) * outerRadius,
@@ -2004,8 +2080,15 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       const globalZoneType = this.mapZoneTypeToGlobalType(zone.type);
       const cameraZones: CameraZoneMapping[] = sourceCameras
         .map(camRef => {
-          const cam = topology.cameras.find(c => c.deviceId === camRef || c.name === camRef);
-          if (!cam) return null;
+          const cam = topology.cameras.find(c =>
+            c.deviceId === camRef ||
+            c.name === camRef ||
+            c.name.toLowerCase() === camRef.toLowerCase()
+          );
+          if (!cam) {
+            this.console.warn(`[Discovery] GlobalZone: No camera matched for ref="${camRef}"`);
+            return null;
+          }
           return {
             cameraId: cam.deviceId,
             zone: [[0, 0], [100, 0], [100, 100], [0, 100]] as ClipPath, // Full frame default