@blueharford/scrypted-spatial-awareness 0.1.0

package/src/core/tracking-engine.ts

@@ -0,0 +1,367 @@
+/**
+ * Tracking Engine
+ * Central orchestrator for cross-camera object tracking
+ */
+
+import sdk, {
+  ScryptedDevice,
+  ObjectDetector,
+  ObjectsDetected,
+  ObjectDetectionResult,
+  ScryptedInterface,
+  EventListenerRegister,
+} from '@scrypted/sdk';
+import { CameraTopology, findCamera, findConnection, findConnectionsFrom } from '../models/topology';
+import {
+  TrackedObject,
+  ObjectSighting,
+  GlobalTrackingId,
+  CorrelationCandidate,
+  getLastSighting,
+} from '../models/tracked-object';
+import { TrackingState } from '../state/tracking-state';
+import { AlertManager } from '../alerts/alert-manager';
+import { ObjectCorrelator } from './object-correlator';
+
+const { systemManager } = sdk;
+
+export interface TrackingEngineConfig {
+  /** Maximum time to wait for correlation (ms) */
+  correlationWindow: number;
+  /** Minimum confidence for automatic correlation */
+  correlationThreshold: number;
+  /** Time before marking object as 'lost' */
+  lostTimeout: number;
+  /** Enable visual embedding matching */
+  useVisualMatching: boolean;
+}
+
+export class TrackingEngine {
+  private topology: CameraTopology;
+  private state: TrackingState;
+  private alertManager: AlertManager;
+  private config: TrackingEngineConfig;
+  private console: Console;
+  private correlator: ObjectCorrelator;
+  private listeners: Map<string, EventListenerRegister> = new Map();
+  private pendingTimers: Map<GlobalTrackingId, NodeJS.Timeout> = new Map();
+  private lostCheckInterval: NodeJS.Timeout | null = null;
+
+  constructor(
+    topology: CameraTopology,
+    state: TrackingState,
+    alertManager: AlertManager,
+    config: TrackingEngineConfig,
+    console: Console
+  ) {
+    this.topology = topology;
+    this.state = state;
+    this.alertManager = alertManager;
+    this.config = config;
+    this.console = console;
+    this.correlator = new ObjectCorrelator(topology, config);
+  }
+
+  /** Start listening to all cameras in topology */
+  async startTracking(): Promise<void> {
+    this.console.log('Starting tracking engine...');
+
+    // Stop any existing listeners
+    await this.stopTracking();
+
+    // Subscribe to each camera's object detection events
+    for (const camera of this.topology.cameras) {
+      try {
+        const device = systemManager.getDeviceById<ObjectDetector>(camera.deviceId);
+        if (!device) {
+          this.console.warn(`Camera not found: ${camera.deviceId} (${camera.name})`);
+          continue;
+        }
+
+        // Check if device has ObjectDetector interface
+        if (!device.interfaces?.includes(ScryptedInterface.ObjectDetector)) {
+          this.console.warn(`Camera ${camera.name} does not support object detection`);
+          continue;
+        }
+
+        const listener = device.listen(ScryptedInterface.ObjectDetector, (source, details, data) => {
+          this.handleDetection(camera.deviceId, camera.name, data as ObjectsDetected);
+        });
+
+        this.listeners.set(camera.deviceId, listener);
+        this.console.log(`Listening to camera: ${camera.name}`);
+      } catch (e) {
+        this.console.error(`Failed to subscribe to camera ${camera.name}:`, e);
+      }
+    }
+
+    // Start periodic check for lost objects
+    this.lostCheckInterval = setInterval(() => {
+      this.checkForLostObjects();
+    }, 30000); // Check every 30 seconds
+
+    this.console.log(`Tracking engine started with ${this.listeners.size} cameras`);
+  }
+
+  /** Stop all camera listeners */
+  async stopTracking(): Promise<void> {
+    // Remove all event listeners
+    for (const [cameraId, listener] of this.listeners.entries()) {
+      try {
+        listener.removeListener();
+      } catch (e) {
+        this.console.error(`Failed to remove listener for ${cameraId}:`, e);
+      }
+    }
+    this.listeners.clear();
+
+    // Clear pending timers
+    for (const timer of this.pendingTimers.values()) {
+      clearTimeout(timer);
+    }
+    this.pendingTimers.clear();
+
+    // Stop lost check interval
+    if (this.lostCheckInterval) {
+      clearInterval(this.lostCheckInterval);
+      this.lostCheckInterval = null;
+    }
+
+    this.console.log('Tracking engine stopped');
+  }
+
+  /** Handle detection event from a camera */
+  private async handleDetection(
+    cameraId: string,
+    cameraName: string,
+    detected: ObjectsDetected
+  ): Promise<void> {
+    if (!detected.detections || detected.detections.length === 0) {
+      return;
+    }
+
+    const camera = findCamera(this.topology, cameraId);
+    if (!camera) {
+      return;
+    }
+
+    const timestamp = detected.timestamp || Date.now();
+
+    for (const detection of detected.detections) {
+      // Skip low-confidence detections
+      if (detection.score < 0.5) continue;
+
+      // Skip classes we're not tracking on this camera
+      if (camera.trackClasses.length > 0 &&
+          !camera.trackClasses.includes(detection.className)) {
+        continue;
+      }
+
+      // Create sighting object
+      const sighting: ObjectSighting = {
+        detection,
+        cameraId,
+        cameraName,
+        timestamp,
+        confidence: detection.score,
+        embedding: detection.embedding,
+        detectionId: detected.detectionId,
+        position: detection.boundingBox ? {
+          x: (detection.boundingBox[0] + detection.boundingBox[2] / 2) / 100,
+          y: (detection.boundingBox[1] + detection.boundingBox[3] / 2) / 100,
+        } : undefined,
+      };
+
+      await this.processSighting(sighting, camera.isEntryPoint, camera.isExitPoint);
+    }
+  }
+
+  /** Process a single sighting */
+  private async processSighting(
+    sighting: ObjectSighting,
+    isEntryPoint: boolean,
+    isExitPoint: boolean
+  ): Promise<void> {
+    // Try to correlate with existing tracked objects
+    const correlation = await this.correlateDetection(sighting);
+
+    if (correlation) {
+      // Matched to existing object
+      const tracked = correlation.trackedObject;
+
+      // Check if this is a cross-camera transition
+      const lastSighting = getLastSighting(tracked);
+      if (lastSighting && lastSighting.cameraId !== sighting.cameraId) {
+        // Add journey segment
+        this.state.addJourney(tracked.globalId, {
+          fromCameraId: lastSighting.cameraId,
+          fromCameraName: lastSighting.cameraName,
+          toCameraId: sighting.cameraId,
+          toCameraName: sighting.cameraName,
+          exitTime: lastSighting.timestamp,
+          entryTime: sighting.timestamp,
+          transitDuration: sighting.timestamp - lastSighting.timestamp,
+          correlationConfidence: correlation.confidence,
+        });
+
+        this.console.log(
+          `Object ${tracked.globalId.slice(0, 8)} transited: ` +
+          `${lastSighting.cameraName} → ${sighting.cameraName} ` +
+          `(confidence: ${(correlation.confidence * 100).toFixed(0)}%)`
+        );
+      }
+
+      // Add sighting to tracked object
+      this.state.addSighting(tracked.globalId, sighting);
+
+      // Cancel any pending lost timer
+      const pendingTimer = this.pendingTimers.get(tracked.globalId);
+      if (pendingTimer) {
+        clearTimeout(pendingTimer);
+        this.pendingTimers.delete(tracked.globalId);
+      }
+
+      // Reactivate if was pending
+      this.state.reactivate(tracked.globalId);
+
+      // Check for exit
+      if (isExitPoint && this.isLeavingFrame(sighting)) {
+        this.handlePotentialExit(tracked, sighting);
+      }
+    } else {
+      // New object - create tracking entry
+      const globalId = this.state.generateId();
+      const tracked = this.state.createObject(globalId, sighting, isEntryPoint);
+
+      this.console.log(
+        `New ${sighting.detection.className} detected on ${sighting.cameraName} ` +
+        `(ID: ${globalId.slice(0, 8)})`
+      );
+
+      // Generate entry alert if this is an entry point
+      if (isEntryPoint) {
+        await this.alertManager.checkAndAlert('property_entry', tracked, {
+          cameraId: sighting.cameraId,
+          cameraName: sighting.cameraName,
+          objectClass: sighting.detection.className,
+          objectLabel: sighting.detection.label,
+          detectionId: sighting.detectionId,
+        });
+      }
+    }
+  }
+
+  /** Attempt to correlate a sighting with existing tracked objects */
+  private async correlateDetection(
+    sighting: ObjectSighting
+  ): Promise<CorrelationCandidate | null> {
+    const activeObjects = this.state.getActiveObjects();
+    if (activeObjects.length === 0) return null;
+
+    // First, check for same-camera tracking (using detection ID)
+    for (const tracked of activeObjects) {
+      const lastSighting = getLastSighting(tracked);
+      if (lastSighting &&
+          lastSighting.cameraId === sighting.cameraId &&
+          lastSighting.detection.id === sighting.detection.id) {
+        // Same object on same camera (continuing track)
+        return {
+          trackedObject: tracked,
+          newSighting: sighting,
+          confidence: 1.0,
+          factors: { timing: 1, visual: 1, spatial: 1, class: 1 },
+        };
+      }
+    }
+
+    // Check for cross-camera correlation
+    const candidate = await this.correlator.findBestMatch(sighting, activeObjects);
+    return candidate;
+  }
+
+  /** Check if a detection is leaving the camera frame */
+  private isLeavingFrame(sighting: ObjectSighting): boolean {
+    if (!sighting.position) return false;
+
+    // Consider leaving if near edge of frame
+    const edgeThreshold = 0.1; // 10% from edge
+    return (
+      sighting.position.x < edgeThreshold ||
+      sighting.position.x > (1 - edgeThreshold) ||
+      sighting.position.y < edgeThreshold ||
+      sighting.position.y > (1 - edgeThreshold)
+    );
+  }
+
+  /** Handle potential exit from property */
+  private handlePotentialExit(tracked: TrackedObject, sighting: ObjectSighting): void {
+    // Mark as pending and set timer
+    this.state.markPending(tracked.globalId);
+
+    // Wait for correlation window before marking as exited
+    const timer = setTimeout(async () => {
+      const current = this.state.getObject(tracked.globalId);
+      if (current && current.state === 'pending') {
+        this.state.markExited(tracked.globalId, sighting.cameraId, sighting.cameraName);
+
+        this.console.log(
+          `Object ${tracked.globalId.slice(0, 8)} exited via ${sighting.cameraName}`
+        );
+
+        await this.alertManager.checkAndAlert('property_exit', current, {
+          cameraId: sighting.cameraId,
+          cameraName: sighting.cameraName,
+          objectClass: current.className,
+          objectLabel: current.label,
+        });
+      }
+      this.pendingTimers.delete(tracked.globalId);
+    }, this.config.correlationWindow);
+
+    this.pendingTimers.set(tracked.globalId, timer);
+  }
+
+  /** Check for objects that haven't been seen recently */
+  private checkForLostObjects(): void {
+    const now = Date.now();
+    const activeObjects = this.state.getActiveObjects();
+
+    for (const tracked of activeObjects) {
+      const timeSinceSeen = now - tracked.lastSeen;
+
+      if (timeSinceSeen > this.config.lostTimeout) {
+        this.state.markLost(tracked.globalId);
+        this.console.log(
+          `Object ${tracked.globalId.slice(0, 8)} marked as lost ` +
+          `(not seen for ${Math.round(timeSinceSeen / 1000)}s)`
+        );
+
+        this.alertManager.checkAndAlert('lost_tracking', tracked, {
+          objectClass: tracked.className,
+          objectLabel: tracked.label,
+        });
+      }
+    }
+  }
+
+  /** Update topology configuration */
+  updateTopology(topology: CameraTopology): void {
+    this.topology = topology;
+    this.correlator = new ObjectCorrelator(topology, this.config);
+  }
+
+  /** Get current topology */
+  getTopology(): CameraTopology {
+    return this.topology;
+  }
+
+  /** Get all currently tracked objects */
+  getTrackedObjects(): TrackedObject[] {
+    return this.state.getAllObjects();
+  }
+
+  /** Get tracked object by ID */
+  getTrackedObject(globalId: GlobalTrackingId): TrackedObject | undefined {
+    return this.state.getObject(globalId);
+  }
+}

package/src/devices/global-tracker-sensor.ts

@@ -0,0 +1,191 @@
+/**
+ * Global Tracker Sensor
+ * Main hub device that provides property-wide occupancy tracking
+ */
+
+import {
+  OccupancySensor,
+  ScryptedDeviceBase,
+  Settings,
+  Setting,
+  Readme,
+  ScryptedNativeId,
+} from '@scrypted/sdk';
+import { TrackingState } from '../state/tracking-state';
+import { TrackedObject, getJourneySummary, calculateDwellTime } from '../models/tracked-object';
+
+export class GlobalTrackerSensor extends ScryptedDeviceBase
+  implements OccupancySensor, Settings, Readme {
+
+  private trackingState: TrackingState;
+  private plugin: any;
+
+  constructor(plugin: any, nativeId: ScryptedNativeId, trackingState: TrackingState) {
+    super(nativeId);
+    this.plugin = plugin;
+    this.trackingState = trackingState;
+
+    // Update occupancy when tracking state changes
+    trackingState.onStateChange(() => this.updateOccupancy());
+
+    // Initial update
+    this.updateOccupancy();
+  }
+
+  /**
+   * Update the occupied state based on active tracked objects
+   */
+  private updateOccupancy(): void {
+    const activeCount = this.trackingState.getActiveCount();
+    this.occupied = activeCount > 0;
+  }
+
+  // ==================== Settings Implementation ====================
+
+  async getSettings(): Promise<Setting[]> {
+    const active = this.trackingState.getActiveObjects();
+    const all = this.trackingState.getAllObjects();
+    const settings: Setting[] = [];
+
+    // Summary stats
+    settings.push({
+      key: 'activeCount',
+      title: 'Currently Active',
+      type: 'string',
+      readonly: true,
+      value: `${active.length} object${active.length !== 1 ? 's' : ''}`,
+      group: 'Status',
+    });
+
+    settings.push({
+      key: 'totalTracked',
+      title: 'Total Tracked (24h)',
+      type: 'string',
+      readonly: true,
+      value: `${all.length} object${all.length !== 1 ? 's' : ''}`,
+      group: 'Status',
+    });
+
+    // Active objects list
+    if (active.length > 0) {
+      settings.push({
+        key: 'activeObjectsHeader',
+        title: 'Active Objects',
+        type: 'html',
+        value: '<h4 style="margin: 0;">Currently Tracked</h4>',
+        group: 'Active Objects',
+      });
+
+      for (const obj of active.slice(0, 10)) {
+        const dwellTime = calculateDwellTime(obj);
+        const dwellMinutes = Math.round(dwellTime / 60000);
+
+        settings.push({
+          key: `active-${obj.globalId}`,
+          title: `${obj.className}${obj.label ? ` (${obj.label})` : ''}`,
+          type: 'string',
+          readonly: true,
+          value: `Cameras: ${obj.activeOnCameras.join(', ')} | Time: ${dwellMinutes}m`,
+          group: 'Active Objects',
+        });
+      }
+
+      if (active.length > 10) {
+        settings.push({
+          key: 'moreActive',
+          title: 'More',
+          type: 'string',
+          readonly: true,
+          value: `...and ${active.length - 10} more`,
+          group: 'Active Objects',
+        });
+      }
+    }
+
+    // Recent exits
+    const recentExits = all
+      .filter(o => o.state === 'exited')
+      .sort((a, b) => b.lastSeen - a.lastSeen)
+      .slice(0, 5);
+
+    if (recentExits.length > 0) {
+      settings.push({
+        key: 'recentExitsHeader',
+        title: 'Recent Exits',
+        type: 'html',
+        value: '<h4 style="margin: 0;">Recently Exited</h4>',
+        group: 'Recent Activity',
+      });
+
+      for (const obj of recentExits) {
+        const exitTime = new Date(obj.lastSeen).toLocaleTimeString();
+        const journey = getJourneySummary(obj);
+
+        settings.push({
+          key: `exit-${obj.globalId}`,
+          title: `${obj.className} at ${exitTime}`,
+          type: 'string',
+          readonly: true,
+          value: journey || 'No journey recorded',
+          group: 'Recent Activity',
+        });
+      }
+    }
+
+    return settings;
+  }
+
+  async putSetting(key: string, value: any): Promise<void> {
+    // No editable settings
+  }
+
+  // ==================== Readme Implementation ====================
+
+  async getReadmeMarkdown(): Promise<string> {
+    const active = this.trackingState.getActiveObjects();
+    const all = this.trackingState.getAllObjects();
+
+    // Generate stats
+    const personCount = active.filter(o => o.className === 'person').length;
+    const vehicleCount = active.filter(o => ['car', 'vehicle', 'truck'].includes(o.className)).length;
+    const animalCount = active.filter(o => o.className === 'animal').length;
+
+    let activeBreakdown = '';
+    if (personCount > 0) activeBreakdown += `- People: ${personCount}\n`;
+    if (vehicleCount > 0) activeBreakdown += `- Vehicles: ${vehicleCount}\n`;
+    if (animalCount > 0) activeBreakdown += `- Animals: ${animalCount}\n`;
+
+    return `
+# Global Object Tracker
+
+This sensor tracks the presence of objects across all connected cameras in your property.
+
+## Current Status
+
+**Occupied**: ${this.occupied ? 'Yes' : 'No'}
+
+**Active Objects**: ${active.length}
+${activeBreakdown || '- None currently'}
+
+**Total Tracked (24h)**: ${all.length}
+
+## How It Works
+
+The Global Tracker combines detection events from all configured cameras to maintain a unified view of objects on your property. When an object moves from one camera's view to another, the system correlates these sightings to track the object's journey.
+
+## States
+
+- **Active**: Object is currently visible on camera
+- **Pending**: Object left camera view, waiting for correlation
+- **Exited**: Object left the property
+- **Lost**: Object disappeared without exiting (timeout)
+
+## Integration
+
+This sensor can be used in automations:
+- Trigger actions when property becomes occupied/unoccupied
+- Create presence-based automations
+- Monitor overall property activity
+`;
+  }
+}