@camera.ui/server 2.1.2 → 2.1.4

package/dist/server/src/camera/decoder/event-manager.js

@@ -1,10 +1,16 @@
 import { randomUUID } from 'node:crypto';
 import { NamespaceManager } from '../../rpc/namespaces.js';
 import { boxIntersectsPolygon } from '../utils/filter.js';
+import { detectionRecord } from './debug/detection-record.js';
+import { leanEvent, NvrSink } from './nvr-sink.js';
 import { MAX_UNTRACKED_PLATES, normalizePlateText, PlateVoteTracker } from './plate-vote.js';
-import { STATIONARY_SPEED_THRESHOLD } from './stationary-suppressor.js';
+export const MOMENT_RANK_OBJECT = 0;
+export const MOMENT_RANK_ATTRIBUTE = 1;
 const UPDATE_THROTTLE_MS = 1000;
+const MOMENT_IMPROVEMENT = 1.25;
+const CLIP_ATTRIBUTE = 'clip';
 const MIN_MOVING_SPEED = 0.05;
+const STATIONARY_SPEED_THRESHOLD = 0.002;
 function jpegInfo(jpeg) {
     let i = 2;
     while (i + 9 < jpeg.length) {
@@ -32,28 +38,35 @@ export class DetectionEventManager {
     proxy;
     logger;
     static MAX_PLATE_THUMBNAILS = 16;
+    static SEGMENT_LINGER_MS = 10_000;
     activeEvent = null;
     activeSegment = null;
-    segmentTimer = null;
+    // first/last box center per track within the current segment, so the NVR
+    // can reason about where a subject entered and left the frame
+    segmentTrackPaths = new Map();
     segmentIndex = 0;
     lastPublishTime = 0;
-    sceneThumbnail = null;
-    sceneThumbnailShipped = false;
-    detectionLabelThumbnails = new Map();
+    shippedSceneAt = 0;
     attributeThumbnails = new Map();
+    heldAttributes = [];
+    shippedAttributes = new Map();
+    segmentMoment;
+    pendingMoment;
+    eventThumbnailAt = 0;
+    segmentClosePendingSince = null;
+    lingerTimer = null;
+    onSegmentClosedCallback;
     eventThumbnail = null;
     needsEventThumbnail = false;
-    eventBestShot = null;
-    eventBestShotPushed = false;
-    eventBestShotDirty = false;
     segmentFaceTrackIds = new Map();
     segmentPlateAttrIndex = new Map();
     segmentPlateReads = new Set();
     eventPlateVotes = new Map();
     plateVotingActive = true;
     segmentClassifierTrackIds = new Map();
-    segmentClipTrackIds = new Set();
+    segmentClipLabels = new Map();
     eventSubject;
+    nvr;
     onEventEndCallback;
     constructor(cameraId, proxy, logger) {
         this.cameraId = cameraId;
@@ -61,10 +74,35 @@ export class DetectionEventManager {
         this.logger = logger;
         const ns = NamespaceManager.detectionEventNamespaces(cameraId);
         this.eventSubject = ns.detectionEventSubject;
+        this.nvr = new NvrSink(proxy, logger);
+    }
+    updateNvrRpc(namespace) {
+        this.nvr.setNamespace(namespace);
     }
     onEventEnd(callback) {
         this.onEventEndCallback = callback;
     }
+    onSegmentClosed(callback) {
+        this.onSegmentClosedCallback = callback;
+    }
+    offerMoment(moment) {
+        // the tick cuts the moment before it opens the segment, so the very picture
+        // that opens a span would be the one dropped; hold it for that opening
+        if (!this.activeSegment) {
+            // a tick can offer the object shot and then the face shot; within the tick
+            // they compete by rank, a newer tick always displaces the older pending one
+            if (this.beatsMoment(moment.rank, moment.score, moment.stream, this.incumbentMoment(moment.capturedAt))) {
+                this.pendingMoment = moment;
+            }
+            // debugging
+            this.recordMomentVerdict(moment, 'pending');
+            return;
+        }
+        this.takeMoment(moment);
+    }
+    wantsMoment(rank, score, stream, at) {
+        return this.beatsMoment(rank, score, stream, this.incumbentMoment(at));
+    }
     processResults(data) {
         const now = data.timestamp;
         const triggers = this.extractTriggers(data, now);
@@ -80,6 +118,7 @@ export class DetectionEventManager {
         // publish immediately, the UI shouldn't wait for the next throttled update
         if (data.eventThumbnail && this.needsEventThumbnail) {
             this.eventThumbnail = data.eventThumbnail;
+            this.eventThumbnailAt = now;
             this.needsEventThumbnail = false;
             this.publishEventThumbnail();
         }
@@ -95,11 +134,17 @@ export class DetectionEventManager {
                 this.publishUpdateThrottled();
             }
         }
-        if (hasDetections && data.hasCascadeTrigger) {
+        if (data.hasOpenSpans) {
+            this.segmentClosePendingSince = null;
+            if (this.lingerTimer) {
+                clearTimeout(this.lingerTimer);
+                this.lingerTimer = null;
+            }
             if (!this.activeSegment) {
-                this.openSegment(data, now);
+                if (hasDetections)
+                    this.openSegment(data, now);
             }
-            else {
+            else if (hasDetections) {
                 this.enrichSegment(data, now);
                 if (data.thumbnails && data.thumbnails.length > 0) {
                     this.updateThumbnails(data.thumbnails);
@@ -108,40 +153,48 @@ export class DetectionEventManager {
                 this.updateTypes();
                 this.publishSegmentThrottled('segment-update');
             }
-            this.resetSegmentTimer(data.segmentTimeout);
         }
+        else if (this.activeSegment && this.segmentClosePendingSince === null) {
+            // all spans closed: linger before deciding, a reopening span continues
+            // this segment (detector flicker is not a new visit); lastSeen stays on
+            // the last real detection, timer-driven so tickless cameras close too
+            this.segmentClosePendingSince = now;
+            this.lingerTimer = setTimeout(() => {
+                this.lingerTimer = null;
+                if (this.segmentClosePendingSince === null)
+                    return;
+                this.segmentClosePendingSince = null;
+                this.closeSegment();
+                this.onSegmentClosedCallback?.();
+            }, DetectionEventManager.SEGMENT_LINGER_MS);
+        }
+    }
+    hasActiveSegment() {
+        return this.activeSegment !== null;
     }
     forceEndActiveEvent() {
         if (this.activeEvent)
             this.endEvent();
     }
     destroy() {
-        if (this.segmentTimer) {
-            clearTimeout(this.segmentTimer);
-            this.segmentTimer = null;
-        }
         if (this.activeEvent)
             this.endEvent();
     }
-    publishEventThumbnail(jpeg, source = 'scene') {
+    publishEventThumbnail(jpeg, capturedAt) {
         if (!this.activeEvent)
             return;
-        // a late scene re-shoot (HQ warm-up) must not undo an object crop
-        if (source === 'scene' && this.eventBestShotPushed)
-            return;
         if (jpeg) {
             this.eventThumbnail = jpeg;
+            this.eventThumbnailAt = capturedAt ?? Date.now();
             this.needsEventThumbnail = false;
         }
         if (!this.eventThumbnail)
             return;
         this.logger.trace(`[event] thumbnail id=${this.activeEvent.id.slice(0, 8)} ${jpegInfo(this.eventThumbnail)}`);
         this.activeEvent.segments = [];
-        this.activeEvent.thumbnail = this.eventThumbnail;
-        this.publish('update');
-        this.activeEvent.thumbnail = undefined;
-        if (this.activeSegment && !this.sceneThumbnail) {
-            this.sceneThumbnail = { jpeg: this.eventThumbnail, score: 0, area: 0, onEdge: false, hasAttribute: false };
+        this.activeEvent.thumbnailAt = this.eventThumbnailAt || undefined;
+        this.publish('update', { scene: this.eventThumbnail });
+        if (this.activeSegment && !this.segmentMoment) {
             this.publishSegment('segment-update');
         }
     }
@@ -166,39 +219,29 @@ export class DetectionEventManager {
         this.segmentIndex = 0;
         this.needsEventThumbnail = true;
         this.eventThumbnail = null;
-        this.eventBestShot = null;
-        this.eventBestShotPushed = false;
-        this.eventBestShotDirty = false;
+        this.eventThumbnailAt = 0;
         if (data.eventThumbnail) {
             this.eventThumbnail = data.eventThumbnail;
+            this.eventThumbnailAt = now;
             this.needsEventThumbnail = false;
         }
         this.updateTypes();
-        // pre-captured thumbnail rides inline in the start message, otherwise the
-        // next coordinator tick delivers it via update
+        // a pre-captured scene ships with the start message, otherwise the next
+        // coordinator tick delivers it via update
         if (this.eventThumbnail) {
-            this.activeEvent.thumbnail = this.eventThumbnail;
+            this.activeEvent.thumbnailAt = this.eventThumbnailAt || undefined;
         }
-        this.publish('start');
-        if (this.activeEvent)
-            this.activeEvent.thumbnail = undefined;
+        this.publish('start', this.eventThumbnail ? { scene: this.eventThumbnail } : undefined);
         const triggerTypes = [...new Set(triggers.map((t) => t.type))].join(',');
         const thumbInfo = this.eventThumbnail ? jpegInfo(this.eventThumbnail) : 'pending';
         this.logger.log(`[event] start id=${this.activeEvent.id.slice(0, 8)} triggers=[${triggerTypes}] types=[${this.activeEvent.types.join(',')}] thumb=${thumbInfo}`);
-        if (this.hasNewDetections(data) && data.hasCascadeTrigger) {
+        if (this.hasNewDetections(data) && data.hasOpenSpans) {
             this.openSegment(data, now);
-            this.resetSegmentTimer(data.segmentTimeout);
         }
     }
     endEvent() {
         if (!this.activeEvent)
             return;
-        if (this.eventBestShotDirty && this.eventBestShot) {
-            this.publishEventThumbnail(this.eventBestShot.jpeg, 'best-shot');
-        }
-        this.eventBestShot = null;
-        this.eventBestShotPushed = false;
-        this.eventBestShotDirty = false;
         this.closeSegment();
         this.activeEvent.state = 'ended';
         this.activeEvent.endTime = Date.now();
@@ -211,11 +254,12 @@ export class DetectionEventManager {
         this.logger.log(`[event] end id=${this.activeEvent.id.slice(0, 8)} duration=${duration}ms segments=${this.segmentIndex} types=[${this.activeEvent.types.join(',')}]`);
         this.activeEvent = null;
         this.activeSegment = null;
-        this.eventPlateVotes.clear();
-        if (this.segmentTimer) {
-            clearTimeout(this.segmentTimer);
-            this.segmentTimer = null;
+        this.segmentClosePendingSince = null;
+        if (this.lingerTimer) {
+            clearTimeout(this.lingerTimer);
+            this.lingerTimer = null;
         }
+        this.eventPlateVotes.clear();
         if (this.onEventEndCallback) {
             this.onEventEndCallback();
         }
@@ -229,7 +273,15 @@ export class DetectionEventManager {
             detections: [],
             attributes: [],
         };
+        this.segmentTrackPaths = new Map();
         this.enrichSegment(data, now);
+        // the tick cuts its moment before it opens the segment, and both carry the
+        // tick's stamp, so an older one belongs to a tick that opened nothing
+        const pending = this.pendingMoment;
+        this.pendingMoment = undefined;
+        if (pending?.capturedAt === now) {
+            this.takeMoment(pending);
+        }
         if (data.thumbnails && data.thumbnails.length > 0) {
             this.updateThumbnails(data.thumbnails);
         }
@@ -250,7 +302,7 @@ export class DetectionEventManager {
             return;
         }
         this.segmentPlateAttrIndex.set(attrKey, this.activeSegment.attributes.length);
-        this.activeSegment.attributes.push({ type: 'license_plate', label, confidence, parentTrackId });
+        this.pushAttribute({ type: 'license_plate', label, confidence, parentTrackId });
     }
     flushPlateFallbacks() {
         if (!this.activeSegment || !this.plateVotingActive)
@@ -272,16 +324,13 @@ export class DetectionEventManager {
     closeSegment() {
         if (!this.activeSegment || !this.activeEvent)
             return;
-        this.activeSegment.lastSeen = Date.now();
+        // lastSeen stays on the last real detection: the linger only delays the
+        // decision to close, it is not part of what was seen
         this.flushPlateFallbacks();
         this.publishSegment('segment-end');
         this.clearSegmentThumbnails();
         this.segmentIndex++;
         this.activeSegment = null;
-        if (this.segmentTimer) {
-            clearTimeout(this.segmentTimer);
-            this.segmentTimer = null;
-        }
     }
     enrichTriggers(triggers, now) {
         if (!this.activeEvent)
@@ -304,10 +353,21 @@ export class DetectionEventManager {
             return;
         this.activeSegment.lastSeen = now;
         if (data.objects.length > 0) {
-            // upsert per label, keep best score/box/trackId and maxCount
             const labelCounts = new Map();
             for (const obj of data.objects) {
                 const t = obj;
+                if (t.trackId !== undefined && obj.box) {
+                    const cx = obj.box.x + obj.box.width / 2;
+                    const cy = obj.box.y + obj.box.height / 2;
+                    const path = this.segmentTrackPaths.get(t.trackId);
+                    if (path) {
+                        path.exitX = cx;
+                        path.exitY = cy;
+                    }
+                    else {
+                        this.segmentTrackPaths.set(t.trackId, { enterX: cx, enterY: cy, exitX: cx, exitY: cy });
+                    }
+                }
                 const entry = labelCounts.get(obj.label);
                 if (entry) {
                     entry.count++;
@@ -344,6 +404,13 @@ export class DetectionEventManager {
                     this.activeSegment.detections.push({ label, score: bestScore, maxCount: count, box: bestBox, trackId: bestTrackId, moving });
                 }
             }
+            for (const d of this.activeSegment.detections) {
+                if (d.trackId === undefined)
+                    continue;
+                const path = this.segmentTrackPaths.get(d.trackId);
+                if (path)
+                    d.path = path;
+            }
             // boxes and polygons are both normalized 0-1, direct overlap test
             if (data.detectionZones && data.detectionZones.length > 0) {
                 const zonesSet = new Set(this.activeSegment.zones ?? []);
@@ -366,33 +433,26 @@ export class DetectionEventManager {
                 continue;
             if (face.identity) {
                 if (!this.activeSegment.attributes.some((a) => a.type === 'face' && a.label === face.identity)) {
-                    this.activeSegment.attributes.push({ type: 'face', label: face.identity, parentTrackId: face.parentTrackId });
+                    this.pushAttribute({ type: 'face', label: face.identity, parentTrackId: face.parentTrackId });
                 }
             }
             else if (face.embedding?.length) {
-                if (face.parentTrackId !== undefined) {
-                    const existingIdx = this.segmentFaceTrackIds.get(face.parentTrackId);
-                    if (existingIdx !== undefined) {
-                        const existing = this.activeSegment.attributes[existingIdx];
-                        if (existing && face.confidence > (existing.confidence ?? 0)) {
-                            existing.confidence = face.confidence;
-                            existing.embedding = face.embedding;
-                            existing.embeddingModel = data.faceEmbeddingModel;
-                            existing.thumbnail = face.thumbnail;
-                        }
-                        continue;
+                // one slot per person, and one shared slot for faces the tracker could
+                // not attach to anyone: without a track there is nothing to tell two
+                // strangers apart by, and a slot per sighting would file the same face
+                // into the index on every tick
+                const bucket = face.parentTrackId !== undefined ? `t${face.parentTrackId}` : 'untracked';
+                const existingIdx = this.segmentFaceTrackIds.get(bucket);
+                if (existingIdx !== undefined) {
+                    const existing = this.activeSegment.attributes[existingIdx];
+                    if (existing && face.confidence > (existing.confidence ?? 0)) {
+                        existing.confidence = face.confidence;
+                        this.heldAttributes[existingIdx] = { thumbnail: face.thumbnail, embedding: face.embedding, embeddingModel: data.faceEmbeddingModel };
                     }
-                    this.segmentFaceTrackIds.set(face.parentTrackId, this.activeSegment.attributes.length);
+                    continue;
                 }
-                this.activeSegment.attributes.push({
-                    type: 'face',
-                    label: 'unknown',
-                    confidence: face.confidence,
-                    embedding: face.embedding,
-                    embeddingModel: data.faceEmbeddingModel,
-                    thumbnail: face.thumbnail,
-                    parentTrackId: face.parentTrackId,
-                });
+                this.segmentFaceTrackIds.set(bucket, this.activeSegment.attributes.length);
+                this.pushAttribute({ type: 'face', label: 'unknown', confidence: face.confidence, parentTrackId: face.parentTrackId }, { thumbnail: face.thumbnail, embedding: face.embedding, embeddingModel: data.faceEmbeddingModel });
             }
         }
         this.plateVotingActive = data.plateVoting !== false;
@@ -408,7 +468,7 @@ export class DetectionEventManager {
                         existing.confidence = plate.confidence;
                 }
                 else {
-                    this.activeSegment.attributes.push({ type: 'license_plate', label, confidence: plate.confidence, parentTrackId: plate.parentTrackId });
+                    this.pushAttribute({ type: 'license_plate', label, confidence: plate.confidence, parentTrackId: plate.parentTrackId });
                 }
                 continue;
             }
@@ -456,24 +516,28 @@ export class DetectionEventManager {
             else if (this.activeSegment.attributes.some((a) => a.type === cls.attribute && a.label === cls.subAttribute)) {
                 continue;
             }
-            this.activeSegment.attributes.push({ type: cls.attribute, label: cls.subAttribute, confidence: cls.confidence, parentTrackId: cls.parentTrackId });
+            this.pushAttribute({ type: cls.attribute, label: cls.subAttribute, confidence: cls.confidence, parentTrackId: cls.parentTrackId });
         }
         for (const clip of data.clips) {
             if (clip.embedding?.length) {
-                const clipKey = clip.parentTrackId !== undefined ? `${clip.parentTrackId}:${clip.label}` : clip.label;
-                if (this.segmentClipTrackIds.has(clipKey))
+                // one embedding per subject actually present, not per track: a
+                // re-numbered track is the same person and would otherwise land in the
+                // search index again, while two real people must keep both appearances
+                const stored = this.segmentClipLabels.get(clip.label) ?? 0;
+                const present = this.activeSegment.detections.find((d) => d.label === clip.label)?.maxCount ?? 1;
+                if (stored >= Math.max(1, present))
                     continue;
-                this.segmentClipTrackIds.add(clipKey);
-                this.activeSegment.attributes.push({
-                    type: 'clip',
-                    label: clip.label,
-                    clipEmbedding: clip.embedding,
-                    clipEmbeddingModel: data.clipEmbeddingModel,
-                    parentTrackId: clip.parentTrackId,
-                });
+                this.segmentClipLabels.set(clip.label, stored + 1);
+                this.pushAttribute({ type: 'clip', label: clip.label, parentTrackId: clip.parentTrackId }, { clipEmbedding: clip.embedding, clipEmbeddingModel: data.clipEmbeddingModel });
             }
         }
     }
+    pushAttribute(attribute, held) {
+        if (!this.activeSegment)
+            return;
+        this.activeSegment.attributes.push(attribute);
+        this.heldAttributes[this.activeSegment.attributes.length - 1] = held ?? {};
+    }
     extractTriggers(data, now) {
         const triggers = [];
         if (data.motion?.detected) {
@@ -517,7 +581,7 @@ export class DetectionEventManager {
     updateTypes() {
         if (!this.activeEvent)
             return;
-        const types = new Set();
+        const types = new Set(this.activeEvent.types);
         for (const t of this.activeEvent.triggers)
             types.add(t.type);
         // activeEvent.segments is only filled during publishSegment, include the
@@ -526,57 +590,26 @@ export class DetectionEventManager {
         for (const seg of segs) {
             for (const d of seg.detections)
                 types.add(d.label);
-            for (const attr of seg.attributes)
+            for (const attr of seg.attributes) {
+                if (attr.type === CLIP_ATTRIBUTE)
+                    continue;
                 types.add(attr.type);
+            }
         }
         this.activeEvent.types = [...types];
     }
     hasNewDetections(data) {
         return data.objects.length > 0 || data.faces.length > 0 || data.plates.length > 0 || data.classifiers.length > 0;
     }
-    resetSegmentTimer(timeoutSeconds) {
-        if (this.segmentTimer)
-            clearTimeout(this.segmentTimer);
-        this.segmentTimer = setTimeout(() => {
-            this.segmentTimer = null;
-            this.closeSegment();
-        }, timeoutSeconds * 1000);
-    }
     updateThumbnails(thumbnails) {
         for (const thumb of thumbnails) {
             const label = thumb.label;
-            if (label.startsWith('face:') || label.startsWith('plate:') || label.startsWith('class:')) {
-                this.updateBestCandidate(this.attributeThumbnails, label, thumb);
-                // a moving vehicle produces a distinct plate crop almost every frame, cap retention
-                if (label.startsWith('plate:'))
-                    this.prunePlateThumbnails();
-            }
-            else {
-                const hasAttr = thumbnails.some((t) => t.label.startsWith('face:') || t.label.startsWith('plate:') || t.label.startsWith('class:'));
-                const candidate = {
-                    jpeg: thumb.jpeg,
-                    score: thumb.score,
-                    area: thumb.area,
-                    onEdge: thumb.onEdge,
-                    hasAttribute: hasAttr,
-                    trackId: thumb.trackId,
-                    speed: thumb.speed,
-                };
-                if (!this.sceneThumbnail || this.isBetterScene(candidate, this.sceneThumbnail)) {
-                    this.sceneThumbnail = candidate;
-                }
-                if (!this.eventBestShot || this.isBetterScene(candidate, this.eventBestShot)) {
-                    this.eventBestShot = candidate;
-                    if (this.eventBestShotPushed) {
-                        this.eventBestShotDirty = true;
-                    }
-                    else {
-                        this.eventBestShotPushed = true;
-                        this.publishEventThumbnail(candidate.jpeg, 'best-shot');
-                    }
-                }
-                this.updateBestCandidate(this.detectionLabelThumbnails, label, thumb);
-            }
+            if (!label.startsWith('face:') && !label.startsWith('plate:') && !label.startsWith('class:'))
+                continue;
+            this.updateBestCandidate(this.attributeThumbnails, label, thumb);
+            // a moving vehicle produces a distinct plate crop almost every frame, cap retention
+            if (label.startsWith('plate:'))
+                this.prunePlateThumbnails();
         }
     }
     prunePlateThumbnails() {
@@ -605,7 +638,6 @@ export class DetectionEventManager {
         }
         const cm = DetectionEventManager.isMoving(candidate);
         const currM = DetectionEventManager.isMoving(current);
-        // prefer moving objects over stationary
         if (cm && !currM) {
             this.logger.trace(`Thumbnail ${key}: prefer moving track#${candidate.trackId} (speed=${candidate.speed?.toFixed(4)}) over static track#${current.trackId}`);
             map.set(key, candidate);
@@ -617,109 +649,165 @@ export class DetectionEventManager {
             map.set(key, candidate);
         }
     }
-    isBetterScene(candidate, current) {
-        if (candidate.hasAttribute && !current.hasAttribute)
-            return true;
-        const cm = DetectionEventManager.isMoving(candidate);
-        const currM = DetectionEventManager.isMoving(current);
-        if (cm && !currM) {
-            this.logger.trace(`Scene thumbnail: prefer moving track#${candidate.trackId} (speed=${candidate.speed?.toFixed(4)}) over static track#${current.trackId}`);
-            return true;
-        }
-        if (!cm && currM)
-            return false;
-        if (candidate.onEdge && !current.onEdge)
-            return false;
-        if (candidate.score > current.score * 1.05)
-            return true;
-        if (candidate.area > current.area * 1.1)
-            return true;
-        return false;
-    }
     static isMoving(c) {
         return (c.speed ?? 0) >= MIN_MOVING_SPEED;
     }
-    injectSegmentThumbnails() {
-        if (!this.activeSegment)
-            return;
-        if (this.sceneThumbnail) {
-            this.activeSegment.thumbnail = this.sceneThumbnail.jpeg;
-        }
-        for (const detection of this.activeSegment.detections) {
-            const detCand = this.detectionLabelThumbnails.get(detection.label);
-            if (!detCand)
-                continue;
-            if (detCand.jpeg === this.sceneThumbnail?.jpeg)
-                continue;
-            detection.thumbnail = detCand.jpeg;
-        }
-        for (const attr of this.activeSegment.attributes) {
-            if (attr.thumbnail)
-                continue;
-            const keys = [`face:${attr.label}`, `plate:${attr.label}`, `class:${attr.label}`];
-            const directKey = `${attr.type}:${attr.label}`;
-            if (!keys.includes(directKey))
-                keys.unshift(directKey);
-            for (const key of keys) {
-                const cand = this.attributeThumbnails.get(key);
-                if (cand) {
-                    attr.thumbnail = cand.jpeg;
-                    break;
+    segmentAttachments(withEmbeddings) {
+        const attachments = {};
+        const moment = this.segmentMoment;
+        if (moment && moment.capturedAt !== this.shippedSceneAt) {
+            attachments.strip = moment.strip;
+            attachments.card = moment.card;
+            this.shippedSceneAt = moment.capturedAt;
+        }
+        const attributes = this.activeSegment?.attributes ?? [];
+        const crops = [];
+        let anyCrop = false;
+        for (const [index, attribute] of attributes.entries()) {
+            const held = this.heldAttributes[index];
+            const crop = held?.thumbnail ?? this.attributeThumbnailFor(attribute);
+            if (crop?.length && this.shippedAttributes.get(index) !== crop) {
+                this.shippedAttributes.set(index, crop);
+                crops.push(crop);
+                anyCrop = true;
+            }
+            else {
+                crops.push(undefined);
+            }
+            // embeddings are write-only into the face and clip indexes, and the
+            // recorder reads them once when the segment closes
+            if (withEmbeddings && held) {
+                if (held.embedding) {
+                    attribute.embedding = held.embedding;
+                    attribute.embeddingModel = held.embeddingModel;
+                }
+                if (held.clipEmbedding) {
+                    attribute.clipEmbedding = held.clipEmbedding;
+                    attribute.clipEmbeddingModel = held.clipEmbeddingModel;
                 }
             }
         }
+        if (anyCrop)
+            attachments.attributes = crops;
+        return attachments;
+    }
+    attributeThumbnailFor(attribute) {
+        const keys = [`face:${attribute.label}`, `plate:${attribute.label}`, `class:${attribute.label}`];
+        const directKey = `${attribute.type}:${attribute.label}`;
+        if (!keys.includes(directKey))
+            keys.unshift(directKey);
+        for (const key of keys) {
+            const candidate = this.attributeThumbnails.get(key);
+            if (candidate)
+                return candidate.jpeg;
+        }
+        return undefined;
+    }
+    takeMoment(moment) {
+        if (!this.beatsMoment(moment.rank, moment.score, moment.stream, this.segmentMoment)) {
+            // debugging
+            this.recordMomentVerdict(moment, 'rejected');
+            return;
+        }
+        // debugging
+        this.recordMomentVerdict(moment, 'kept');
+        this.segmentMoment = moment;
     }
-    stripSegmentThumbnails() {
-        if (!this.activeSegment)
+    heldMomentInfo() {
+        const held = this.segmentMoment;
+        if (!held)
+            return undefined;
+        return { capturedAt: held.capturedAt, score: held.score, rank: held.rank, stream: held.stream };
+    }
+    incumbentMoment(at) {
+        if (this.activeSegment)
+            return this.segmentMoment;
+        return this.pendingMoment?.capturedAt === at ? this.pendingMoment : undefined;
+    }
+    beatsMoment(rank, score, stream, held) {
+        if (!held)
+            return true;
+        if (rank !== held.rank)
+            return rank > held.rank;
+        // the score is normalized, so it cannot see that a main-stream crop is the
+        // same framing with several times the pixels: that one only has to tie
+        if (stream === 'main' && held.stream === 'low')
+            return score > held.score;
+        return score > held.score * MOMENT_IMPROVEMENT;
+    }
+    // debugging
+    recordMomentVerdict(moment, verdict) {
+        if (!detectionRecord.active)
             return;
-        this.activeSegment.thumbnail = undefined;
-        for (const det of this.activeSegment.detections)
-            det.thumbnail = undefined;
-        for (const attr of this.activeSegment.attributes)
-            attr.thumbnail = undefined;
+        detectionRecord.momentVerdict({
+            verdict,
+            capturedAt: moment.capturedAt,
+            score: moment.score,
+            rank: moment.rank,
+            stream: moment.stream,
+            held: this.heldMomentInfo(),
+            eventId: this.activeEvent?.id.slice(0, 8),
+            segment: this.activeSegment ? this.segmentIndex : undefined,
+        });
     }
     clearSegmentThumbnails() {
-        this.sceneThumbnail = null;
-        this.sceneThumbnailShipped = false;
-        this.detectionLabelThumbnails.clear();
+        this.shippedSceneAt = 0;
+        this.segmentMoment = undefined;
+        this.pendingMoment = undefined;
         this.attributeThumbnails.clear();
+        this.heldAttributes = [];
+        this.shippedAttributes.clear();
         this.segmentFaceTrackIds.clear();
         this.segmentPlateAttrIndex.clear();
         this.segmentPlateReads.clear();
         this.segmentClassifierTrackIds.clear();
-        this.segmentClipTrackIds.clear();
+        this.segmentClipLabels.clear();
     }
-    publish(type) {
+    publish(type, attachments) {
         if (!this.activeEvent)
             return;
-        this.proxy.publish(this.eventSubject, { type, data: this.activeEvent });
+        const message = { type, data: leanEvent(this.activeEvent) };
+        this.proxy.publish(this.eventSubject, message);
+        this.nvr.send(type, this.activeEvent, attachments);
+        // debugging
+        this.recordMessage(type, attachments);
         this.lastPublishTime = Date.now();
     }
+    // debugging
+    recordMessage(type, attachments) {
+        if (!detectionRecord.active || !this.activeEvent)
+            return;
+        const at = this.segmentMoment?.capturedAt ?? this.activeEvent.lastUpdate;
+        const segment = this.activeSegment;
+        const attributes = segment?.attributes ?? [];
+        detectionRecord.message({
+            type,
+            eventId: this.activeEvent.id.slice(0, 8),
+            segment: segment ? this.segmentIndex : undefined,
+            types: this.activeEvent.types,
+            recorderReachable: this.nvr.connected,
+            detections: segment?.detections.map((d) => ({ label: d.label, score: d.score, count: d.maxCount, track: d.trackId, moving: d.moving })),
+            attributes: attributes.map((a) => ({ type: a.type, label: a.label, confidence: a.confidence, parent: a.parentTrackId })),
+            sent: {
+                scene: attachments?.scene ? detectionRecord.picture(this.eventThumbnailAt || at, 'scene', 'event', attachments.scene) : undefined,
+                strip: attachments?.strip ? detectionRecord.picture(at, 'moment', 'strip', attachments.strip) : undefined,
+                card: attachments?.card ? detectionRecord.picture(at, 'moment', 'card', attachments.card) : undefined,
+                attributes: attachments?.attributes?.map((crop, index) => crop ? detectionRecord.picture(at, 'attr', `${attributes[index]?.type}-${attributes[index]?.label}`, crop) : undefined),
+            },
+            momentStream: this.segmentMoment?.stream,
+            momentAt: this.segmentMoment?.capturedAt,
+        });
+    }
     publishSegment(type) {
         if (!this.activeEvent || !this.activeSegment)
             return;
         this.activeEvent.segments = [this.activeSegment];
         this.activeEvent.segmentIndex = this.segmentIndex;
         this.logSegment(type);
-        if (type === 'segment-end') {
-            this.injectSegmentThumbnails();
-            this.publish(type);
-        }
-        else if (type === 'segment-start') {
-            this.injectSegmentThumbnails();
-            this.publish(type);
-            this.sceneThumbnailShipped = this.sceneThumbnail !== null;
-            this.stripSegmentThumbnails();
-        }
-        else if (!this.sceneThumbnailShipped && this.sceneThumbnail) {
-            this.activeSegment.thumbnail = this.sceneThumbnail.jpeg;
-            this.publish(type);
-            this.activeSegment.thumbnail = undefined;
-            this.sceneThumbnailShipped = true;
-        }
-        else {
-            this.publish(type);
-        }
+        const moment = this.segmentMoment;
+        if (moment)
+            this.activeSegment.thumbnailAt = moment.capturedAt;
+        this.publish(type, this.segmentAttachments(type === 'segment-end'));
     }
     publishSegmentThrottled(type) {
         const now = Date.now();
@@ -749,17 +837,16 @@ export class DetectionEventManager {
         }
         if (seg.attributes.length > 0) {
             const groups = new Map();
-            for (const a of seg.attributes) {
+            for (const [index, a] of seg.attributes.entries()) {
+                const held = this.heldAttributes[index];
                 const list = groups.get(a.type) ?? [];
                 let info = a.label;
                 if (a.confidence)
                     info += `(${a.confidence.toFixed(2)})`;
-                if (a.type === 'face' && a.embedding?.length)
+                if (held?.embedding?.length || held?.clipEmbedding?.length)
                     info += '+emb';
-                if (a.type === 'face' && a.thumbnail)
+                if (held?.thumbnail ?? this.attributeThumbnailFor(a))
                     info += '+thumb';
-                if (a.type === 'clip' && a.clipEmbedding?.length)
-                    info += '+emb';
                 list.push(info);
                 groups.set(a.type, list);
             }
@@ -782,10 +869,10 @@ export class DetectionEventManager {
             parts.push(`zones=[${seg.zones.join(',')}]`);
         }
         const thumbParts = [];
-        if (this.sceneThumbnail)
-            thumbParts.push(`scene:${jpegInfo(this.sceneThumbnail.jpeg)}`);
-        for (const [label, cand] of this.detectionLabelThumbnails) {
-            thumbParts.push(cand.jpeg === this.sceneThumbnail?.jpeg ? `${label}:=scene` : `${label}:${jpegInfo(cand.jpeg)}`);
+        const moment = this.segmentMoment;
+        if (moment) {
+            const shows = moment.rank >= MOMENT_RANK_ATTRIBUTE ? 'attr' : 'obj';
+            thumbParts.push(`moment(${shows},${moment.stream}):${jpegInfo(moment.strip)}${moment.card ? `+card:${jpegInfo(moment.card)}` : ''}`);
         }
         for (const [label, cand] of this.attributeThumbnails)
             thumbParts.push(`${label}:${jpegInfo(cand.jpeg)}`);