@blueharford/scrypted-spatial-awareness 0.6.17 → 0.6.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.17",
+  "version": "0.6.20",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/topology-discovery.ts

@@ -105,7 +105,10 @@ For EACH landmark, estimate its DISTANCE from the camera:
 - Street/road
 - Neighbor's yard
 
-For each zone, estimate what percentage of the image it covers (0.0 to 1.0).
+For each zone, provide:
+- coverage: percentage of the image it covers (0.0 to 1.0)
+- distance: how far the CENTER of the zone is from camera ("close", "near", "medium", "far", "distant")
+- boundingBox: [x, y, width, height] in normalized coordinates (0-1) where the zone appears in the image
 
 ## 3. EDGES - What's at each edge of the frame:
 This helps understand what's just out of view.
@@ -123,8 +126,9 @@ Respond with ONLY valid JSON:
     {"name": "Large Oak Tree", "type": "feature", "distance": "far", "confidence": 0.85, "description": "Mature oak tree near property line, roughly 80 feet from camera"}
   ],
   "zones": [
-    {"name": "Front Yard", "type": "yard", "coverage": 0.5, "description": "Grass lawn with some bare patches"},
-    {"name": "Driveway", "type": "driveway", "coverage": 0.25, "description": "Concrete driveway leading to garage"}
+    {"name": "Front Yard", "type": "yard", "coverage": 0.5, "distance": "medium", "boundingBox": [0.2, 0.4, 0.6, 0.4], "description": "Grass lawn with some bare patches"},
+    {"name": "Driveway", "type": "driveway", "coverage": 0.25, "distance": "near", "boundingBox": [0.6, 0.5, 0.3, 0.4], "description": "Concrete driveway leading to garage"},
+    {"name": "Street", "type": "street", "coverage": 0.1, "distance": "distant", "boundingBox": [0.0, 0.1, 1.0, 0.15], "description": "Public road beyond property line"}
   ],
   "edges": {
     "top": "sky, tree canopy",
@@ -440,7 +444,8 @@ Use the mount height to help estimate distances - objects at ground level will a
                 coverage: typeof z.coverage === 'number' ? z.coverage : 0.5,
                 description: z.description || '',
                 boundingBox: z.boundingBox,
-              }));
+                distance: this.mapDistance(z.distance), // Parse distance for zones too
+              } as DiscoveredZone & { distance?: DistanceEstimate }));
             }
 
             if (parsed.edges && typeof parsed.edges === 'object') {
@@ -560,6 +565,27 @@ Use the mount height to help estimate distances - objects at ground level will a
     return 'medium'; // Default to medium if not specified
   }
 
+  /** Get default distance in feet based on zone type */
+  private getDefaultZoneDistance(zoneType: string): number {
+    switch (zoneType) {
+      case 'patio':
+      case 'walkway':
+        return 10; // Close zones
+      case 'driveway':
+      case 'parking':
+        return 25; // Near zones
+      case 'yard':
+      case 'garden':
+      case 'pool':
+        return 40; // Medium zones
+      case 'street':
+        return 100; // Far zones
+      case 'unknown':
+      default:
+        return 50; // Default to medium distance
+    }
+  }
+
   /** Try to parse JSON with recovery for truncated responses */
   private parseJsonWithRecovery(jsonStr: string, context: string): any {
     // First, try direct parse
@@ -882,6 +908,10 @@ Use the mount height to help estimate distances - objects at ground level will a
     // Generate zone suggestions (even for smaller coverage - 10% is enough)
     for (const zone of analysis.zones) {
       if (zone.coverage >= 0.1) {
+        // Calculate distance in feet from distance estimate (for zones with distance info)
+        const zoneWithDist = zone as DiscoveredZone & { distance?: DistanceEstimate };
+        const distanceFeet = zoneWithDist.distance ? distanceToFeet(zoneWithDist.distance) : this.getDefaultZoneDistance(zone.type);
+
         const suggestion: DiscoverySuggestion = {
           id: `zone_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
           type: 'zone',
@@ -889,10 +919,14 @@ Use the mount height to help estimate distances - objects at ground level will a
           sourceCameras: [analysis.cameraId],
           confidence: Math.min(0.9, 0.5 + zone.coverage), // Higher coverage = higher confidence
           status: 'pending',
-          zone: zone,
+          zone: {
+            ...zone,
+            // Include distance metadata for positioning
+            distanceFeet: distanceFeet,
+          } as any,
         };
         this.suggestions.set(suggestion.id, suggestion);
-        this.console.log(`[Discovery] Zone suggestion: ${zone.name} (${zone.type}, ${Math.round(zone.coverage * 100)}% coverage)`);
+        this.console.log(`[Discovery] Zone suggestion: ${zone.name} (${zone.type}, ${Math.round(zone.coverage * 100)}% coverage, ~${distanceFeet}ft)`);
       }
     }
   }

package/src/main.ts

@@ -1062,18 +1062,24 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
     if (request.method === 'GET') {
       const topologyJson = this.storage.getItem('topology');
       const topology = topologyJson ? JSON.parse(topologyJson) : createEmptyTopology();
+      this.console.log(`[Topology API] GET - drawnZones: ${topology.drawnZones?.length || 0}`);
       response.send(JSON.stringify(topology), {
         headers: { 'Content-Type': 'application/json' },
       });
     } else if (request.method === 'PUT' || request.method === 'POST') {
       try {
         const topology = JSON.parse(request.body!) as CameraTopology;
+        this.console.log(`[Topology API] PUT received - drawnZones: ${topology.drawnZones?.length || 0}`);
+        if (topology.drawnZones?.length) {
+          this.console.log(`[Topology API] Zone names: ${topology.drawnZones.map(z => z.name).join(', ')}`);
+        }
         this.storage.setItem('topology', JSON.stringify(topology));
         await this.startTrackingEngine(topology);
         response.send(JSON.stringify({ success: true }), {
           headers: { 'Content-Type': 'application/json' },
         });
       } catch (e) {
+        this.console.error(`[Topology API] PUT error:`, e);
         response.send(JSON.stringify({ error: 'Invalid topology JSON' }), {
           code: 400,
           headers: { 'Content-Type': 'application/json' },
@@ -1904,71 +1910,46 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
           const dirRad = (direction - 90) * Math.PI / 180;
           const halfFov = (fovAngle / 2) * Math.PI / 180;
 
-          // Use landmark TYPE to determine distance - some things are inherently far away
-          // regardless of where they appear in the camera frame
-          const landmarkType = suggestion.landmark.type;
-          const farTypes = ['neighbor', 'boundary', 'street']; // Always place at edge of FOV
-          const isFarType = farTypes.includes(landmarkType || '');
+          // Get floor plan scale (pixels per foot) - default 5
+          const floorPlanScale = topology.floorPlanScale || 5;
+
+          // Use the ACTUAL distance from LLM analysis (distanceFeet) - this is the key fix!
+          // The LLM estimates distance based on object size, perspective, and camera context
+          const landmarkData = suggestion.landmark as any;
+          const distanceFeet = landmarkData.distanceFeet || 50; // Default 50ft if not set
+          const distanceInPixels = distanceFeet * floorPlanScale;
 
-          // Use bounding box if available for horizontal positioning
-          // boundingBox format: [x, y, width, height] normalized 0-1
-          const bbox = (suggestion.landmark as any).boundingBox as [number, number, number, number] | undefined;
+          // Use bounding box for horizontal positioning within the FOV
+          const bbox = landmarkData.boundingBox as [number, number, number, number] | undefined;
 
           let angleOffset: number;
-          let distanceMultiplier: number;
 
           if (bbox && bbox.length >= 2) {
             // Use bounding box X for horizontal position in FOV
             const bboxCenterX = bbox[0] + (bbox[2] || 0) / 2; // 0 = left edge, 1 = right edge
-
             // Map X position to angle within FOV
             angleOffset = (bboxCenterX - 0.5) * 2 * halfFov;
-
-            // For distance: use TYPE first, then bbox Y as hint
-            if (isFarType) {
-              // Neighbors, boundaries, streets are BEYOND the camera's normal range
-              // Place at 150-200% of range to indicate they're distant background features
-              distanceMultiplier = 1.5 + Math.random() * 0.5; // 150-200% of range
-              this.console.log(`[Discovery] Far-type landmark "${landmarkType}" placed BEYOND FOV (${(distanceMultiplier * 100).toFixed(0)}% of range)`);
-            } else {
-              // For other types, use bbox Y as a hint (but cap minimum distance)
-              const bboxCenterY = bbox[1] + (bbox[3] || 0) / 2;
-              // Map Y to distance: 0 (top) = far, 1 (bottom) = closer (but not too close)
-              distanceMultiplier = Math.max(0.4, 0.9 - (bboxCenterY * 0.5));
-            }
-
-            this.console.log(`[Discovery] Using bounding box [${bbox.join(',')}] for horizontal position`);
+            this.console.log(`[Discovery] Using bounding box [${bbox.join(',')}] for horizontal angle offset`);
           } else {
-            // No bounding box - use type and spread pattern
+            // No bounding box - spread horizontally based on existing landmarks
             const cameraDeviceId = camera.deviceId;
             const existingFromCamera = (topology.landmarks || []).filter(l =>
               l.visibleFromCameras?.includes(cameraDeviceId) ||
               l.visibleFromCameras?.includes(camera.name)
             ).length;
-
             // Spread horizontally across FOV
             angleOffset = (existingFromCamera % 3 - 1) * halfFov * 0.6;
-
-            // Distance based on type
-            if (isFarType) {
-              // Neighbors, boundaries, streets are BEYOND the camera's normal range
-              distanceMultiplier = 1.5 + Math.random() * 0.5; // 150-200% of range
-              this.console.log(`[Discovery] Far-type landmark "${landmarkType}" (no bbox) placed BEYOND FOV`);
-            } else {
-              distanceMultiplier = 0.5 + (existingFromCamera % 2) * 0.3;
-              this.console.log(`[Discovery] No bounding box, using fallback spread (existing: ${existingFromCamera})`);
-            }
+            this.console.log(`[Discovery] No bounding box, spreading horizontally (existing: ${existingFromCamera})`);
           }
 
           const finalAngle = dirRad + angleOffset;
-          const distance = range * distanceMultiplier;
 
           position = {
-            x: camera.floorPlanPosition.x + Math.cos(finalAngle) * distance,
-            y: camera.floorPlanPosition.y + Math.sin(finalAngle) * distance,
+            x: camera.floorPlanPosition.x + Math.cos(finalAngle) * distanceInPixels,
+            y: camera.floorPlanPosition.y + Math.sin(finalAngle) * distanceInPixels,
           };
 
-          this.console.log(`[Discovery] Placing landmark "${suggestion.landmark.name}" in ${camera.name}'s FOV: dir=${direction}°, angle=${(angleOffset * 180 / Math.PI).toFixed(1)}°, dist=${distance.toFixed(0)}px`);
+          this.console.log(`[Discovery] Placing landmark "${suggestion.landmark.name}" at ${distanceFeet}ft (${distanceInPixels}px) from ${camera.name}: dir=${direction}°, angle=${(angleOffset * 180 / Math.PI).toFixed(1)}°`);
         } else {
           // Position in a grid pattern starting from center
           const landmarkCount = topology.landmarks?.length || 0;
@@ -2029,12 +2010,14 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
       const timestamp = Date.now();
 
       if (camera?.floorPlanPosition) {
-        // Get camera's FOV direction and range (cast to any for flexible access)
+        // Get camera's FOV direction (cast to any for flexible access)
         const fov = (camera.fov || { mode: 'simple', angle: 90, direction: 0, range: 80 }) as any;
         const direction = fov.direction || 0;
-        const range = fov.range || 80;
         const fovAngle = fov.angle || 90;
 
+        // Get floor plan scale (pixels per foot)
+        const floorPlanScale = topology.floorPlanScale || 5;
+
         // Convert direction to radians (0 = up/north, 90 = right/east)
         const dirRad = (direction - 90) * Math.PI / 180;
         const halfFov = (fovAngle / 2) * Math.PI / 180;
@@ -2042,47 +2025,52 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
         const camX = camera.floorPlanPosition.x;
         const camY = camera.floorPlanPosition.y;
 
-        // Use bounding box if available to position zone accurately within FOV
+        // Use distanceFeet from the zone metadata for accurate positioning
+        const zoneData = zone as any;
+        const distanceFeet = zoneData.distanceFeet || 40; // Default 40ft if not set
+        const distanceInPixels = distanceFeet * floorPlanScale;
+
+        // Zone size based on coverage (larger coverage = wider zone)
+        const zoneWidthFeet = Math.sqrt(zone.coverage) * 30; // e.g., 50% coverage = ~21ft wide
+        const zoneWidthPixels = zoneWidthFeet * floorPlanScale;
+        const zoneDepthPixels = (zone.coverage * 20) * floorPlanScale; // Depth based on coverage
+
+        // Use bounding box for horizontal positioning if available
         const bbox = zone.boundingBox; // [x, y, width, height] normalized 0-1
 
-        let innerRadius: number;
-        let outerRadius: number;
         let angleStart: number;
         let angleEnd: number;
+        let innerRadius: number;
+        let outerRadius: number;
 
         if (bbox && bbox.length >= 4) {
-          // Map bounding box to position within FOV
+          // Map bounding box X to angle within FOV
           const bboxLeft = bbox[0];
           const bboxRight = bbox[0] + bbox[2];
-          const bboxTop = bbox[1];
-          const bboxBottom = bbox[1] + bbox[3];
-
-          // Map X to angle within FOV (0 = left edge, 1 = right edge)
           angleStart = dirRad + (bboxLeft - 0.5) * 2 * halfFov;
           angleEnd = dirRad + (bboxRight - 0.5) * 2 * halfFov;
 
-          // Map Y to distance (0 = far, 1 = close)
-          innerRadius = range * (0.9 - bboxBottom * 0.6);
-          outerRadius = range * (0.9 - bboxTop * 0.6);
+          // Use distanceFeet for depth, with a spread based on coverage
+          innerRadius = Math.max(distanceInPixels - zoneDepthPixels / 2, 10);
+          outerRadius = distanceInPixels + zoneDepthPixels / 2;
 
-          // Ensure min size
-          if (outerRadius - innerRadius < 20) {
-            outerRadius = innerRadius + 20;
-          }
-
-          this.console.log(`[Discovery] Zone "${zone.name}" using bbox [${bbox.join(',')}] → angles ${(angleStart * 180 / Math.PI).toFixed(1)}° to ${(angleEnd * 180 / Math.PI).toFixed(1)}°`);
+          this.console.log(`[Discovery] Zone "${zone.name}" at ${distanceFeet}ft using bbox [${bbox.join(',')}]`);
         } else {
-          // Fallback: wedge-shaped zone offset by existing count
+          // Fallback: wedge-shaped zone covering portion of FOV
           const existingFromCamera = (topology.drawnZones || []).filter((z: any) =>
             z.linkedCameras?.includes(sourceCameras[0])
           ).length;
 
-          innerRadius = range * 0.3 + existingFromCamera * 20;
-          outerRadius = range * 0.8 + existingFromCamera * 20;
-          angleStart = dirRad - halfFov * 0.7;
-          angleEnd = dirRad + halfFov * 0.7;
+          // Spread horizontally based on existing zones
+          const offset = (existingFromCamera % 3 - 1) * halfFov * 0.5;
+          angleStart = dirRad + offset - halfFov * 0.3;
+          angleEnd = dirRad + offset + halfFov * 0.3;
+
+          // Use distanceFeet for depth
+          innerRadius = Math.max(distanceInPixels - zoneDepthPixels / 2, 10);
+          outerRadius = distanceInPixels + zoneDepthPixels / 2;
 
-          this.console.log(`[Discovery] Zone "${zone.name}" using fallback spread (existing: ${existingFromCamera})`);
+          this.console.log(`[Discovery] Zone "${zone.name}" at ${distanceFeet}ft using fallback spread`);
         }
 
         // Create arc polygon
@@ -2104,7 +2092,7 @@ export class SpatialAwarenessPlugin extends ScryptedDeviceBase
           });
         }
 
-        this.console.log(`[Discovery] Creating zone "${zone.name}" in ${camera.name}'s FOV: dir=${direction}°`);
+        this.console.log(`[Discovery] Creating zone "${zone.name}" at ${distanceFeet}ft (${distanceInPixels}px) from ${camera.name}`);
       } else {
         // Fallback: rectangular zone at default location
         const centerX = 300 + (topology.drawnZones?.length || 0) * 120;

package/src/models/topology.ts

@@ -238,7 +238,7 @@ export interface DrawnZone {
   /** Zone type */
   type: DrawnZoneType;
   /** Polygon points on floor plan (x, y coordinates) */
-  polygon: Point[];
+  polygon: FloorPlanPosition[];
   /** Custom color override (optional) */
   color?: string;
   /** Linked camera IDs that can see this zone */
@@ -354,6 +354,8 @@ export interface CameraTopology {
   floorPlan?: FloorPlanConfig;
   /** Pending AI landmark suggestions */
   pendingSuggestions?: LandmarkSuggestion[];
+  /** Floor plan scale in pixels per foot (for distance calculations) */
+  floorPlanScale?: number;
 }
 
 // ==================== Helper Functions ====================

package/src/ui/editor-html.ts

@@ -454,6 +454,7 @@ export const EDITOR_HTML = `<!DOCTYPE html>
         const response = await fetch('../api/topology');
         if (response.ok) {
           topology = await response.json();
+          console.log('[Load] Loaded topology with', (topology.drawnZones || []).length, 'zones');
           if (!topology.drawings) topology.drawings = [];
           // Load floor plan scale if saved
           if (topology.floorPlanScale) {
@@ -971,14 +972,24 @@ export const EDITOR_HTML = `<!DOCTYPE html>
     async function saveTopology() {
       try {
         setStatus('Saving...', 'warning');
+        console.log('[Save] Saving topology with', (topology.drawnZones || []).length, 'zones');
         const response = await fetch('../api/topology', {
           method: 'PUT',
           headers: { 'Content-Type': 'application/json' },
           body: JSON.stringify(topology)
         });
-        if (response.ok) { setStatus('Saved successfully', 'success'); }
-        else { setStatus('Failed to save', 'error'); }
-      } catch (e) { console.error('Failed to save topology:', e); setStatus('Failed to save', 'error'); }
+        if (response.ok) {
+          console.log('[Save] Save successful');
+          setStatus('Saved successfully', 'success');
+        } else {
+          const errorText = await response.text();
+          console.error('[Save] Save failed:', response.status, errorText);
+          setStatus('Failed to save: ' + errorText, 'error');
+        }
+      } catch (e) {
+        console.error('Failed to save topology:', e);
+        setStatus('Failed to save', 'error');
+      }
     }
 
     function resizeCanvas() {
@@ -1598,9 +1609,11 @@ export const EDITOR_HTML = `<!DOCTYPE html>
       // Zone list
       const zoneList = document.getElementById('zone-list');
       const zones = topology.drawnZones || [];
+      console.log('[updateUI] Zone list update - topology.drawnZones:', zones.length, 'zones');
       if (zones.length === 0) {
         zoneList.innerHTML = '<div class="zone-item" style="color: #666; text-align: center; cursor: default; padding: 8px;">No zones drawn</div>';
       } else {
+        console.log('[updateUI] Rendering', zones.length, 'zones:', zones.map(z => z.name).join(', '));
         zoneList.innerHTML = zones.map(z => {
           const color = ZONE_STROKE_COLORS[z.type] || ZONE_STROKE_COLORS.custom;
           return '<div class="camera-item ' + (selectedItem?.type === 'zone' && selectedItem?.id === z.id ? 'selected' : '') + '" onclick="selectZone(\\'' + z.id + '\\')" style="border-left: 3px solid ' + color + ';"><div class="camera-name">' + z.name + '</div><div class="camera-info">' + z.type + ' | ' + z.polygon.length + ' points</div></div>';
@@ -1778,9 +1791,13 @@ export const EDITOR_HTML = `<!DOCTYPE html>
         polygon: currentZonePoints.map(pt => ({ x: pt.x, y: pt.y })),
       };
 
+      console.log('[Zone] Creating zone:', zone.name, 'with', zone.polygon.length, 'points');
+
       if (!topology.drawnZones) topology.drawnZones = [];
       topology.drawnZones.push(zone);
 
+      console.log('[Zone] Total zones now:', topology.drawnZones.length);
+
       // Reset state
       zoneDrawingMode = false;
       currentZonePoints = [];
@@ -1789,7 +1806,19 @@ export const EDITOR_HTML = `<!DOCTYPE html>
       setTool('select');
       updateUI();
       render();
-      setStatus('Zone "' + zone.name + '" created with ' + zone.polygon.length + ' points', 'success');
+
+      console.log('[Zone] After updateUI/render, zones in topology:', (topology.drawnZones || []).length);
+
+      setStatus('Zone "' + zone.name + '" created - saving...', 'success');
+
+      // Auto-save after creating a zone
+      saveTopology().then(() => {
+        console.log('[Zone] Save completed, zones in topology:', (topology.drawnZones || []).length);
+        setStatus('Zone "' + zone.name + '" saved', 'success');
+      }).catch(err => {
+        console.error('Failed to auto-save zone:', err);
+        setStatus('Zone created but auto-save failed - click Save', 'warning');
+      });
     }
 
     function selectZone(id) {
@@ -1814,9 +1843,9 @@ export const EDITOR_HTML = `<!DOCTYPE html>
         '<button class="btn btn-primary" onclick="deleteZone(\\'' + id + '\\')" style="background: #dc2626; width: 100%;">Delete Zone</button>';
     }
 
-    function updateZoneName(id, value) { const z = (topology.drawnZones || []).find(z => z.id === id); if (z) { z.name = value; updateUI(); render(); } }
-    function updateZoneType(id, value) { const z = (topology.drawnZones || []).find(z => z.id === id); if (z) { z.type = value; updateUI(); render(); } }
-    function updateZoneDesc(id, value) { const z = (topology.drawnZones || []).find(z => z.id === id); if (z) { z.description = value || undefined; } }
+    function updateZoneName(id, value) { const z = (topology.drawnZones || []).find(z => z.id === id); if (z) { z.name = value; updateUI(); render(); saveTopology(); } }
+    function updateZoneType(id, value) { const z = (topology.drawnZones || []).find(z => z.id === id); if (z) { z.type = value; updateUI(); render(); saveTopology(); } }
+    function updateZoneDesc(id, value) { const z = (topology.drawnZones || []).find(z => z.id === id); if (z) { z.description = value || undefined; saveTopology(); } }
     function deleteZone(id) {
       if (!confirm('Delete this zone?')) return;
       topology.drawnZones = (topology.drawnZones || []).filter(z => z.id !== id);
@@ -1824,7 +1853,8 @@ export const EDITOR_HTML = `<!DOCTYPE html>
       document.getElementById('properties-panel').innerHTML = '<h3>Properties</h3><p style="color: #666;">Select an item to edit.</p>';
       updateUI();
       render();
-      setStatus('Zone deleted', 'success');
+      setStatus('Zone deleted - saving...', 'success');
+      saveTopology().then(() => setStatus('Zone deleted', 'success'));
     }
 
     function useBlankCanvas() {