@camera.ui/server 2.2.2 → 2.2.4

package/dist/server/src/camera/decoder/detection-coordinator.js

@@ -40,6 +40,8 @@ const TRAINING_FRAME_MAX_WIDTH = 1280;
 const TRAINING_ATTRIBUTE_BONUS = 0.2;
 const TRAINING_FRAME_QUALITY = 80;
 const MOMENT_EVENTS = new Set(['objectEntered', 'objectWoke', 'objectRecovered', 'bestShotUpdated']);
+const WITNESS_WINDOW_MS = 3000;
+const HELD_MOMENT_MS = 4000;
 const MOMENT_MOVING_SPEED = 0.05;
 const MOMENT_ATTRIBUTE_MIN_AREA = 600;
 const DEFAULT_CASCADE_TIMEOUT = 10;
@@ -77,6 +79,10 @@ let DetectionCoordinator = class DetectionCoordinator {
     platesSeen = new Set();
     classifierLabels = new Map();
     feedingSensors = new Map();
+    witnessSensors = new Set();
+    // label -> when a witness last reported it, for the trace
+    witnessSeen = new Map();
+    heldMoments = new Map();
     dwell = new DwellManager();
     privacy;
     // debugging
@@ -223,6 +229,8 @@ let DetectionCoordinator = class DetectionCoordinator {
         return this.config.detectionSettings;
     }
     getPerfSnapshot() {
+        // decodes outside the loop (external reports, pictures) count as well
+        this.collectDecodeStats();
         const { timings, ...counters } = this.perf.snapshot();
         const detectors = {};
         for (const type of DETECTOR_METRIC_TYPES) {
@@ -247,8 +255,19 @@ let DetectionCoordinator = class DetectionCoordinator {
         };
     }
     resetPerf() {
+        // what was decoded before the reset must not land in the new period
+        this.frameSource.takeDecodeStats();
+        this.thumbnailer.takeHqDecodeStats();
         this.perf.reset();
     }
+    collectDecodeStats() {
+        const decode = this.frameSource.takeDecodeStats();
+        this.perf.decodeMs += decode.ms;
+        this.perf.decodedFrames += decode.frames;
+        const main = this.thumbnailer.takeHqDecodeStats();
+        this.perf.mainDecodeMs += main.ms;
+        this.perf.mainDecodedFrames += main.frames;
+    }
     pauseForBenchmark(paused) {
         this.benchmarkRunning = paused;
     }
@@ -334,6 +353,7 @@ let DetectionCoordinator = class DetectionCoordinator {
         }
     }
     async dispose() {
+        this.heldMoments.clear();
         await this.stopVideoLoop();
         await this.frameSource.stop();
         await this.audioLoop.stop();
@@ -349,6 +369,12 @@ let DetectionCoordinator = class DetectionCoordinator {
         this.currentDetectionState = {};
     }
     async onSensorAdded(sensor) {
+        if (sensor.role === 'witness') {
+            this.feedingSensors.delete(sensor.sensorId);
+            this.witnessSensors.add(sensor.sensorId);
+            return;
+        }
+        this.witnessSensors.delete(sensor.sensorId);
         if (DETECTION_SENSOR_TYPES.has(sensor.sensorType)) {
             this.feedingSensors.set(sensor.sensorId, sensor.sensorType);
         }
@@ -364,6 +390,7 @@ let DetectionCoordinator = class DetectionCoordinator {
         }
     }
     async onSensorRemoved(sensorId) {
+        this.witnessSensors.delete(sensorId);
         this.classifierLabels.delete(sensorId);
         const sensorType = this.feedingSensors.get(sensorId);
         if (sensorType === SensorType.Face)
@@ -392,7 +419,12 @@ let DetectionCoordinator = class DetectionCoordinator {
         this.ptzAutotracker.setCapabilities(sensorId, capabilities);
     }
     async reportSensorWrite(sensorId, sensorType, properties) {
-        // the registry only announces the plugin assigned for this type, everyone else reports into the void
+        if (sensorType === SensorType.Object && this.witnessSensors.has(sensorId)) {
+            this.recordWitness(properties);
+            return;
+        }
+        // the registry announces the plugin assigned for this type and the
+        // witnesses, everyone else reports into the void
         if (!this.feedingSensors.has(sensorId))
             return;
         // a detector reports its spec again once its models finished loading
@@ -677,6 +709,25 @@ let DetectionCoordinator = class DetectionCoordinator {
             ...(result.decibels !== undefined ? { decibels: result.decibels } : {}),
         });
     }
+    recordWitness(properties) {
+        if (properties.detected !== true)
+            return;
+        const detections = properties.detections ?? [];
+        const labels = new Set();
+        for (const detection of detections) {
+            const label = detection.label?.toLowerCase();
+            if (label && this.pipeline.objectLabelAllowed(label))
+                labels.add(label);
+        }
+        if (labels.size === 0)
+            return;
+        const at = Date.now();
+        for (const label of labels) {
+            this.pipeline.attest(label, at);
+            this.witnessSeen.set(label, at);
+        }
+        this.logger.trace(`[witness] ${[...labels].join(', ')}`);
+    }
     writeSensorProperties(sensorId, properties) {
         const msg = {
             sensorId,
@@ -1087,6 +1138,7 @@ let DetectionCoordinator = class DetectionCoordinator {
         let firstFrame = true;
         while (this.loopRunning) {
             const tickStart = Date.now();
+            let onMainStream = false;
             const snap = await this.frameSource.nextFrame(lastFrameId);
             if (!snap)
                 break; // source stopped
@@ -1115,6 +1167,7 @@ let DetectionCoordinator = class DetectionCoordinator {
                 // motion always reads the low stream: switching its input resolution
                 // would reset the background model on every transition
                 const analysis = await this.acquireAnalysisFrame(snap);
+                onMainStream = analysis.isMainStream;
                 // debugging
                 detectionRecord.setFrame(analysis.isMainStream ? 'main' : 'low', analysis.frame.width, analysis.frame.height, analysis.rtp, analysis.role);
                 try {
@@ -1147,9 +1200,7 @@ let DetectionCoordinator = class DetectionCoordinator {
                     // ignore
                 }
             }
-            const decode = this.frameSource.takeDecodeStats();
-            this.perf.decodeMs += decode.ms;
-            this.perf.decodedFrames += decode.frames;
+            this.collectDecodeStats();
             // activity, not stream choice: cameras without main-stream analysis still count as active
             if (this.worldSpans.size > 0 || this.eventManager.hasActiveSegment())
                 this.perf.activeTicks++;
@@ -1159,7 +1210,12 @@ let DetectionCoordinator = class DetectionCoordinator {
             const remaining = (this.mainStreamActive ? ACTIVE_TICK_MS : IDLE_TICK_MS) - (Date.now() - tickStart);
             if (remaining > 0)
                 await sleep(remaining);
-            this.perf.loopMs += Date.now() - tickStart;
+            const tickMs = Date.now() - tickStart;
+            this.perf.loopMs += tickMs;
+            if (onMainStream) {
+                this.perf.mainTicks++;
+                this.perf.mainLoopMs += tickMs;
+            }
         }
         await this.frameSource.detach();
         this.frameScaler.clearCache();
@@ -1167,12 +1223,9 @@ let DetectionCoordinator = class DetectionCoordinator {
     }
     async acquireAnalysisFrame(snap) {
         if (this.mainStreamActive) {
-            const t0 = Date.now();
             try {
                 const main = await this.thumbnailer.acquireHqFrame(0);
                 if (main) {
-                    this.perf.mainFrames++;
-                    this.perf.mainDecodeMs += Date.now() - t0;
                     return { frame: main.frame, scaler: main.scaler, isMainStream: true, rtp: main.rtp, role: this.thumbnailer.mainStreamRole };
                 }
             }
@@ -1185,12 +1238,9 @@ let DetectionCoordinator = class DetectionCoordinator {
     async acquireOpeningHqFrame() {
         if (!this.mainStreamAvailable)
             return undefined;
-        const t0 = Date.now();
         try {
             const main = await this.thumbnailer.acquireHqFrame(0);
             if (main) {
-                this.perf.mainFrames++;
-                this.perf.mainDecodeMs += Date.now() - t0;
                 return { frame: main.frame, scaler: main.scaler, isMainStream: true, rtp: main.rtp, role: this.thumbnailer.mainStreamRole };
             }
         }
@@ -1225,6 +1275,9 @@ let DetectionCoordinator = class DetectionCoordinator {
         trace.rtp = analysis.rtp;
         trace.src = analysis.role?.replace('-resolution', '');
         trace.attrs = traceAttributes(results);
+        const witness = [...this.witnessSeen].filter(([, at]) => Math.abs(trace.tMs - at) <= WITNESS_WINDOW_MS).map(([label]) => label);
+        if (witness.length > 0)
+            trace.witness = witness;
         // debugging
         detectionRecord.tick({ ...trace });
         return trace;
@@ -1259,8 +1312,10 @@ let DetectionCoordinator = class DetectionCoordinator {
             const motionDue = t0 - this.lastMotionAt >= MOTION_INTERVAL_MS - TICK_SLACK_MS;
             const motionScaleStart = Date.now();
             const motionFrame = motionDue ? await this.scaleForMotion(motionRawFrame) : undefined;
-            if (motionDue)
-                this.perf.scaleMs += Date.now() - motionScaleStart;
+            if (motionDue) {
+                this.perf.motionScaleMs += Date.now() - motionScaleStart;
+                this.perf.motionScaleCount++;
+            }
             if (!this.loopRunning)
                 return;
             if (motionFrame) {
@@ -1306,7 +1361,8 @@ let DetectionCoordinator = class DetectionCoordinator {
             if (localizeDue && this.localizerWanted) {
                 const scaleStart = Date.now();
                 const gray = await this.scaleForMotionInput(motionRawFrame);
-                this.perf.scaleMs += Date.now() - scaleStart;
+                this.perf.motionScaleMs += Date.now() - scaleStart;
+                this.perf.motionScaleCount++;
                 if (!this.loopRunning)
                     return;
                 if (gray)
@@ -1344,6 +1400,7 @@ let DetectionCoordinator = class DetectionCoordinator {
                     if (this.updateWorldSpans(pipelineResult) && !analysis.isMainStream) {
                         this.hqUpgrade = await this.acquireOpeningHqFrame();
                     }
+                    await this.holdSightingMoments(pipelineResult, this.hqUpgrade ?? analysis, t0);
                     await this.captureMoments(pipelineResult, this.hqUpgrade ?? analysis, t0);
                     // open spans and a lingering segment keep the cascade armed: the
                     // world can only observe absence or a reappearance while it still
@@ -1648,6 +1705,35 @@ let DetectionCoordinator = class DetectionCoordinator {
         }
         return opened;
     }
+    async holdSightingMoments(result, analysis, at) {
+        for (const [trackId, held] of this.heldMoments) {
+            if (at - held.capturedAt > HELD_MOMENT_MS)
+                this.heldMoments.delete(trackId);
+        }
+        for (const id of result.removed)
+            this.heldMoments.delete(id);
+        if (this.witnessSensors.size === 0)
+            return;
+        const confirmed = new Set(result.created);
+        for (const sighting of result.sightings) {
+            // confirmed in the same tick: the regular moment cuts this frame anyway
+            if (confirmed.has(sighting.trackId))
+                continue;
+            const box = { x: sighting.x, y: sighting.y, width: sighting.width, height: sighting.height };
+            const rendered = await this.renderMoment({ subject: box, base: box }, analysis);
+            if (!rendered)
+                continue;
+            const score = sighting.confidence * Math.sqrt(sighting.width * sighting.height);
+            this.heldMoments.set(sighting.trackId, {
+                strip: rendered.strip,
+                card: rendered.card,
+                capturedAt: at,
+                score,
+                rank: MOMENT_RANK_OBJECT,
+                stream: analysis.isMainStream ? 'main' : 'low',
+            });
+        }
+    }
     async captureMoments(result, analysis, at) {
         let subject;
         let bestScore = -1;
@@ -1672,6 +1758,16 @@ let DetectionCoordinator = class DetectionCoordinator {
         // one frame per tick, not one per object: they share the picture
         if (!subject)
             return;
+        // a track confirmed through a witness after its only sighting: the
+        // current frame no longer shows it, the picture held at the sighting does
+        const held = this.heldMoments.get(subject.trackId);
+        this.heldMoments.delete(subject.trackId);
+        if (held && trigger === 'objectEntered' && subject.lastSeenMs < at) {
+            if (!this.eventManager.wantsMoment(held.rank, held.score, held.stream, at))
+                return;
+            this.eventManager.offerMoment({ ...held, capturedAt: at, shownAt: held.capturedAt });
+            return;
+        }
         if (!this.eventManager.wantsMoment(MOMENT_RANK_OBJECT, bestScore, analysis.isMainStream ? 'main' : 'low', at))
             return;
         await this.captureMoment(subject, bestScore, trigger, result.tracked, analysis, at);
@@ -1921,7 +2017,9 @@ let DetectionCoordinator = class DetectionCoordinator {
             const perWindow = await Promise.all(windows.map((window) => this.detectInWindow(objectPlugin, analysis.frame, analysis.scaler, window, inputSpec)));
             this.perf.objectMs += Date.now() - inferStart;
             this.perf.objectCount++;
-            return mergeWindowDetections(perWindow.flat());
+            for (const window of perWindow)
+                this.perf.scaleMs += window.cropMs;
+            return mergeWindowDetections(perWindow.flatMap((window) => window.detections));
         }
         const scaleStart = Date.now();
         const objectFrame = await this.scaleForObject(analysis.frame, analysis.scaler);
@@ -1937,11 +2035,11 @@ let DetectionCoordinator = class DetectionCoordinator {
     async detectInWindow(plugin, frame, scaler, window, spec) {
         const scaleStart = Date.now();
         const cropped = await scaler.cropToSpec(frame, window, spec);
-        this.perf.scaleMs += Date.now() - scaleStart;
+        const cropMs = Date.now() - scaleStart;
         if (!cropped)
-            return [];
+            return { detections: [], cropMs };
         const result = await PromiseTimeout(plugin.proxy.detectObjects(scaler.toVideoFrameData(cropped.padded, 'model')), DETECT_TIMEOUT_MS, undefined, `Object detection timed out after ${DETECT_TIMEOUT_MS}ms`);
-        return FrameScaler.undoLetterbox(ensureDetectionBoxes(result.detections), cropped.geometry);
+        return { detections: FrameScaler.undoLetterbox(ensureDetectionBoxes(result.detections), cropped.geometry), cropMs };
     }
     async scaleFrameForPlugin(rawFrame, plugin, scaler = this.frameScaler) {
         const inputSpec = plugin.modelSpec?.input;
@@ -1990,7 +2088,6 @@ let DetectionCoordinator = class DetectionCoordinator {
             const found = boxed.filter((d) => reportedLabels.size === 0 || reportedLabels.has(d.label.toLowerCase()));
             if (found.length === 0)
                 return { detections: reported, assisted: false };
-            this.perf.objects += found.length;
             return { detections: found, assisted: true };
         }
         catch (error) {
@@ -2013,7 +2110,7 @@ let DetectionCoordinator = class DetectionCoordinator {
             if (windows.length > 0) {
                 detectionRecord.zoom({ kind: 'assist', anchors, windows });
                 const perWindow = await Promise.all(windows.map((window) => this.detectInWindow(assist, frame, scaler, window, inputSpec)));
-                return mergeWindowDetections(perWindow.flat());
+                return mergeWindowDetections(perWindow.flatMap((window) => window.detections));
             }
         }
         detectionRecord.tick({