@camera.ui/server 2.2.7 → 2.3.0

package/dist/server/src/camera/decoder/detection-coordinator.js

@@ -35,14 +35,15 @@ import { PtzAutotracker } from './ptz/autotracker.js';
 import { SecondaryStage } from './secondary-stage.js';
 import { BufferedSource } from './sources/buffered-source.js';
 import { FrameSource } from './sources/frame-source.js';
-import { DETECT_TIMEOUT_MS, DETECTOR_METRIC_TYPES, ensureDetectionBoxes, isFullFrameBox, isTrainingSubject, MOTION_WIDTH_MAP, touchesFrameEdge } from './types.js';
+import { trainingValue } from './training-sink.js';
+import { DETECT_TIMEOUT_MS, DETECTOR_METRIC_TYPES, ensureDetectionBoxes, isFullFrameBox, isMovingTrack, isTrainingSubject, MOTION_WIDTH_MAP } from './types.js';
 const TRAINING_FRAME_MAX_WIDTH = 1280;
 const TRAINING_ATTRIBUTE_BONUS = 0.2;
+const TRAINING_SUGGESTION_WEIGHT = 0.5;
 const TRAINING_FRAME_QUALITY = 80;
 const MOMENT_EVENTS = new Set(['objectEntered', 'objectWoke', 'objectRecovered', 'bestShotUpdated']);
 const WITNESS_WINDOW_MS = 3000;
 const HELD_MOMENT_MS = 4000;
-const MOMENT_MOVING_SPEED = 0.05;
 const MOMENT_ATTRIBUTE_MIN_AREA = 600;
 const DEFAULT_CASCADE_TIMEOUT = 10;
 const OBJECT_DWELL_SECONDS = 2;
@@ -101,6 +102,7 @@ let DetectionCoordinator = class DetectionCoordinator {
     trackAnchorBoxes = [];
     farewellBoxes = [];
     lastTrackedById = new Map();
+    faceReads = [];
     detectionWindow = new DetectionWindow();
     mainStreamActive = false;
     idleSince = 0;
@@ -190,7 +192,6 @@ let DetectionCoordinator = class DetectionCoordinator {
             proxy: this.proxy,
             cameraId: this.config.cameraId,
             settings: this.config.ptzAutotrack,
-            getFps: () => this.targetFps,
             onSuppressionActivated: () => this.handleSuppressionActivated(),
         });
         this.cascadeUnsubscribe = this.cascade.onChange((event) => {
@@ -367,9 +368,15 @@ let DetectionCoordinator = class DetectionCoordinator {
     acceptClipVectors(embeddings, embeddingModel, capturedAt) {
         this.eventManager.acceptClipVectors(embeddings, embeddingModel, capturedAt);
     }
+    traceFaceReads(reads) {
+        this.faceReads.push(...reads);
+    }
     acceptFaceVectors(faces, embeddingModel, capturedAt) {
         this.eventManager.acceptFaceVectors(faces, embeddingModel, capturedAt);
     }
+    acceptPersonVectors(persons, embeddingModel, capturedAt) {
+        this.eventManager.acceptPersonVectors(persons, embeddingModel, capturedAt);
+    }
     async dispose() {
         this.heldMoments.clear();
         await this.stopVideoLoop();
@@ -509,6 +516,8 @@ let DetectionCoordinator = class DetectionCoordinator {
                             if (results.thumbnails && results.thumbnails.length > 0) {
                                 snapshot.thumbnails = results.thumbnails;
                             }
+                            if (results.trainingSuggestions)
+                                snapshot.trainingSuggestions = results.trainingSuggestions;
                             await this.attachTrainingFrame(snapshot, analysis);
                             this.eventManager.processResults(snapshot);
                         }
@@ -640,9 +649,6 @@ let DetectionCoordinator = class DetectionCoordinator {
     get motionResolution() {
         return this.config.detectionSettings.motion.resolution;
     }
-    get targetFps() {
-        return this.frameSource.fps;
-    }
     get videoAspectRatio() {
         const [w, h] = this.config.interfaceSettings.aspectRatio.split(':').map(Number);
         return w / h;
@@ -1299,6 +1305,10 @@ let DetectionCoordinator = class DetectionCoordinator {
         trace.rtp = analysis.rtp;
         trace.src = analysis.role?.replace('-resolution', '');
         trace.attrs = traceAttributes(results);
+        if (this.faceReads.length > 0) {
+            trace.reads = this.faceReads;
+            this.faceReads = [];
+        }
         const witness = [...this.witnessSeen].filter(([, at]) => Math.abs(trace.tMs - at) <= WITNESS_WINDOW_MS).map(([label]) => label);
         if (witness.length > 0)
             trace.witness = witness;
@@ -1322,6 +1332,7 @@ let DetectionCoordinator = class DetectionCoordinator {
         let objectDetections = [];
         let staticDetections = [];
         let trainingExtras = [];
+        let trainingSuggestions = [];
         const results = { timestamp: t0 };
         let trace;
         // while the PTZ repositions: motion and object keep running so the
@@ -1454,10 +1465,11 @@ let DetectionCoordinator = class DetectionCoordinator {
                     results.object = { detected: objectDetections.length > 0, detections: visibleTracks };
                     staticDetections = pipelineResult.staticTracks;
                     trainingExtras = pipelineResult.trainingExtras;
+                    trainingSuggestions = pipelineResult.trainingSuggestions;
                     if (pipelineResult.crossings.length > 0)
                         results.lineCrossings = pipelineResult.crossings;
-                    // incl. extrapolated tracks, otherwise a single missed detector
-                    // frame flips the autotracker into LOST/REACQUIRE churn
+                    // incl. extrapolated tracks, they keep the target's id through a
+                    // missed detector frame
                     this.ptzAutotracker.handleObjectDetections(pipelineResult.tracked);
                     // live tracks anchor the next tick's window; a track that just died
                     // gets one farewell look at its last position
@@ -1508,7 +1520,8 @@ let DetectionCoordinator = class DetectionCoordinator {
             return;
         if (objectDetections.length > 0) {
             const images = this.hqUpgrade ?? analysis;
-            await this.runSecondariesAndThumbnails(images, objectDetections, results);
+            const stillPersons = staticDetections.filter((d) => d.label === 'person');
+            await this.runSecondariesAndThumbnails(images, objectDetections, results, stillPersons);
             if (!this.loopRunning)
                 return;
             await this.captureAttributeMoment(results, images, t0);
@@ -1526,6 +1539,9 @@ let DetectionCoordinator = class DetectionCoordinator {
             snapshot.staticObjects = staticDetections;
         if (trainingExtras.length > 0)
             snapshot.trainingExtras = trainingExtras;
+        const suggestions = [...trainingSuggestions, ...(results.trainingSuggestions ?? [])];
+        if (suggestions.length > 0)
+            snapshot.trainingSuggestions = suggestions;
         if (results.thumbnails && results.thumbnails.length > 0) {
             snapshot.thumbnails = results.thumbnails;
         }
@@ -1567,13 +1583,13 @@ let DetectionCoordinator = class DetectionCoordinator {
             this.thumbnailer.upgradeEventThumbnailAsync();
         }
     }
-    async runSecondariesAndThumbnails(analysis, objectDetections, results) {
+    async runSecondariesAndThumbnails(analysis, objectDetections, results, stillPersons = []) {
         if (objectDetections.length === 0)
             return;
         this.perf.objects += objectDetections.length;
         this.perf.framesWithObjects++;
         const secondaryStart = Date.now();
-        await this.secondaries.detect(analysis.frame, analysis.scaler, objectDetections, results);
+        await this.secondaries.detect(analysis.frame, analysis.scaler, objectDetections, results, stillPersons);
         this.perf.secondaryMs += Date.now() - secondaryStart;
         try {
             const jpegStart = Date.now();
@@ -1667,26 +1683,30 @@ let DetectionCoordinator = class DetectionCoordinator {
     async attachTrainingFrame(snapshot, analysis) {
         if (!this.eventManager.hasActiveEvent() && !this.snapshotWillStartEvent(snapshot))
             return;
-        // moment-style score per subject, an edge-clipped subject makes a poor
-        // sample; the sink sums only the boxes the scene memory does not know yet
+        // the sink sums only the boxes the scene memory does not know yet
+        const value = (label, confidence, box) => trainingValue(label, confidence, this.pipeline.threshold(label), box);
         const subjects = snapshot.objects
             .filter((d) => isTrainingSubject(d))
-            .map((d) => ({ label: d.label, box: d.box, score: d.confidence * Math.sqrt(d.box.width * d.box.height) * (touchesFrameEdge(d.box) ? 0.5 : 1) }));
-        // flat bonus: faces and plates are the scarce labels but their boxes are too
-        // small for the area term; fresh only, a buffered stale result describes
-        // an older frame and must not lift this one
+            .map((d) => ({ label: d.label, box: d.box, score: value(d.label, d.confidence, d.box) }));
+        // flat bonus: faces and plates are the scarce labels; fresh only, a
+        // buffered stale result describes an older frame and must not lift this one
         const now = Date.now();
         const facesFresh = snapshot.facesAt !== undefined && now - snapshot.facesAt <= SECONDARY_FRESH_MS;
         const platesFresh = snapshot.platesAt !== undefined && now - snapshot.platesAt <= SECONDARY_FRESH_MS;
         for (const face of facesFresh ? snapshot.faces : []) {
             if (face.box && !isFullFrameBox(face.box))
-                subjects.push({ label: 'face', box: face.box, score: TRAINING_ATTRIBUTE_BONUS * (touchesFrameEdge(face.box) ? 0.5 : 1) });
+                subjects.push({ label: 'face', box: face.box, score: TRAINING_ATTRIBUTE_BONUS });
         }
         for (const plate of platesFresh ? snapshot.plates : []) {
             if (plate.box && !isFullFrameBox(plate.box))
-                subjects.push({ label: 'license_plate', box: plate.box, score: TRAINING_ATTRIBUTE_BONUS * (touchesFrameEdge(plate.box) ? 0.5 : 1) });
+                subjects.push({ label: 'license_plate', box: plate.box, score: TRAINING_ATTRIBUTE_BONUS });
+        }
+        for (const hint of snapshot.trainingSuggestions ?? []) {
+            if (!isFullFrameBox(hint.box)) {
+                subjects.push({ label: hint.label, box: hint.box, score: TRAINING_SUGGESTION_WEIGHT * value(hint.label, hint.confidence, hint.box), hint: true });
+            }
         }
-        if (subjects.length === 0)
+        if (!subjects.some((s) => !s.hint))
             return;
         if (!this.eventManager.wantsTrainingFrame(subjects))
             return;
@@ -1697,6 +1717,7 @@ let DetectionCoordinator = class DetectionCoordinator {
             if (jpeg) {
                 snapshot.trainingFrame = jpeg;
                 snapshot.trainingSubjects = subjects;
+                snapshot.trainingDetector = this.plugins.get(SensorType.Object)?.pluginId;
             }
         }
         catch (error) {
@@ -1827,7 +1848,7 @@ let DetectionCoordinator = class DetectionCoordinator {
     }
     async captureMoment(subject, score, trigger, tracked, analysis, at) {
         const box = { x: subject.x, y: subject.y, width: subject.width, height: subject.height };
-        const moving = tracked.filter((d) => (d.trackSpeed ?? 0) >= MOMENT_MOVING_SPEED).map((d) => d.box);
+        const moving = tracked.filter(isMovingTrack).map((d) => d.box);
         const target = { subject: box, base: unionBox([box, ...moving]), hint: directionOf(subject.velocityX, subject.velocityY) };
         const rendered = await this.renderMoment(target, analysis);
         if (!rendered)
@@ -2112,7 +2133,9 @@ let DetectionCoordinator = class DetectionCoordinator {
         const assist = this.plugins.get(SensorType.ObjectAssist);
         try {
             const inferStart = Date.now();
-            const boxed = await this.assistDetections(assist, frame, scaler, reported);
+            // the plugin delivers from its floor: only boxes above the user threshold
+            // may replace the camera's report
+            const boxed = this.pipeline.aboveThreshold(await this.assistDetections(assist, frame, scaler, reported));
             this.perf.assistMs += Date.now() - inferStart;
             this.perf.assistCount++;
             const reportedLabels = new Set(reported.map((d) => d.label.toLowerCase()));