@camera.ui/server 2.1.9 → 2.1.10

package/dist/server/src/camera/decoder/detection-coordinator.js

@@ -19,11 +19,13 @@ import { AudioDetectionLoop } from './audio-loop.js';
 import { CascadeManager } from './cascade-manager.js';
 import { detectionRecord } from './debug/detection-record.js';
 import { DetectionPipeline } from './detection-pipeline.js';
+import { clusterBoxes, DetectionWindow, mergeWindowDetections, MOTION_PAD, planWindowsOnce, TRACK_PAD } from './detection-window.js';
 import { DwellManager } from './dwell-manager.js';
 import { DetectionEventManager, MOMENT_RANK_ATTRIBUTE, MOMENT_RANK_OBJECT } from './event-manager.js';
 import { EventThumbnailer } from './event-thumbnailer.js';
 import { FrameScaler } from './frame-scaler.js';
 import { directionBetween, directionOf, MOMENT_FORMATS, MOMENT_QUALITY, momentWindow, unionBox } from './moment-crop.js';
+import { MotionLocalizer } from './motion-localizer.js';
 import { PerfTracker } from './perf-tracker.js';
 import { normalizePlateText } from './plate-vote.js';
 import { isVideoInputSpec, modelIdentity, PluginRegistry } from './plugin-registry.js';
@@ -74,6 +76,13 @@ let DetectionCoordinator = class DetectionCoordinator {
     perf = new PerfTracker();
     benchmarkRunning = false;
     hqUpgrade;
+    localizer;
+    lastLocalizeAt = 0;
+    anchorBoxes = [];
+    trackAnchorBoxes = [];
+    farewellBoxes = [];
+    lastTrackedById = new Map();
+    detectionWindow = new DetectionWindow();
     mainStreamActive = false;
     idleSince = 0;
     lastMotionAt = 0;
@@ -115,6 +124,7 @@ let DetectionCoordinator = class DetectionCoordinator {
         if (config.streamHot)
             this.frameSource.start();
         this.frameScaler = new FrameScaler(null, logger, this.privacy);
+        this.localizer = new MotionLocalizer();
         this.pipeline = new DetectionPipeline(this.pipelineZones(), config.detectionSettings);
         this.secondaries = new SecondaryStage(this, this.plugins, this.pipeline, this.frameScaler, this.proxy, this.perf, logger);
         if (config.zones.lines.length > 0) {
@@ -167,6 +177,7 @@ let DetectionCoordinator = class DetectionCoordinator {
         this.cascadeUnsubscribe = this.cascade.onChange((event) => {
             if (event.type === 'activated') {
                 this.startAdHocVideoLoopIfNeeded();
+                this.seedLocalizerReference();
             }
             else {
                 this.stopAdHocVideoLoopIfIdle();
@@ -194,6 +205,7 @@ let DetectionCoordinator = class DetectionCoordinator {
                 if (!this.dwell.hasActive() && this.worldSpans.size === 0 && !this.eventManager.hasActiveSegment()) {
                     this.eventManager.forceEndActiveEvent();
                 }
+                this.stopAdHocVideoLoopIfIdle();
             }
         });
     }
@@ -390,6 +402,9 @@ let DetectionCoordinator = class DetectionCoordinator {
             this.ptzAutotracker.handlePresenceDetections(externalDetections);
             if (this.ptzAutotracker.suppressionActive)
                 return;
+            detectionRecord.tick({
+                externalObject: { detected: filtered.detected === true, count: (filtered.detections ?? []).length },
+            });
             this.ingestDetectionResult(SensorType.Object, sensorId, filtered);
             this.eventManager.processResults(this.buildSnapshot());
             // re-shoot the event thumbnail: smart-camera reports arrive after the
@@ -403,8 +418,10 @@ let DetectionCoordinator = class DetectionCoordinator {
             if (filtered.detected !== true)
                 return;
             // assist alone is enough: its boxes upgrade zones, bboxes and thumbnails
-            // even when no face/plate/clip secondary consumes them
-            if (!this.plugins.hasFrameBasedSecondary() && !this.plugins.hasEligibleObjectAssist())
+            // even when no face/plate/clip secondary consumes them; without either,
+            // the embedded diff's anchors can still localize a moment crop
+            const hasAnchors = this.anchorBoxes.length > 0 || this.farewellBoxes.length > 0;
+            if (!this.plugins.hasFrameBasedSecondary() && !this.plugins.hasEligibleObjectAssist() && !hasAnchors)
                 return;
             if (this.processingExternalSecondary)
                 return; // previous RPC still running
@@ -416,42 +433,60 @@ let DetectionCoordinator = class DetectionCoordinator {
                     return;
                 try {
                     const results = { timestamp: Date.now() };
-                    const objects = await this.runObjectAssist(handle.frame, rawExternal);
-                    this.ingestAssistedObjects(sensorId, objects);
-                    if (objects.assisted) {
-                        // real boxes: crop the object like the frame pipeline, so face and
-                        // plate secondaries run on the subject instead of the whole scene
-                        // (processExternal only adapts the shape, external boxes get no real tracking)
-                        const objectDetections = this.pipeline.processExternal(objects.detections);
-                        await this.runSecondariesAndThumbnails({ frame: handle.frame, scaler: this.frameScaler, isMainStream: false }, objectDetections, results);
-                        this.ingestResultsForAllSecondaries(results);
-                        const snapshot = this.buildSnapshot();
-                        if (results.thumbnails && results.thumbnails.length > 0) {
-                            snapshot.thumbnails = results.thumbnails;
-                        }
-                        this.eventManager.processResults(snapshot);
-                    }
-                    else {
-                        const sceneJpeg = await this.secondaries.detectFullFrame(handle.frame, results);
-                        try {
-                            results.thumbnails = await this.secondaries.generateThumbnails(handle.frame, this.frameScaler, results);
-                        }
-                        catch (error) {
-                            this.logger.error('Thumbnail generation error:', error);
-                        }
-                        this.ingestResultsForAllSecondaries(results);
-                        const snapshot = this.buildSnapshot();
-                        if (results.thumbnails && results.thumbnails.length > 0) {
-                            snapshot.thumbnails = results.thumbnails;
+                    // anchors may have switched analysis to the main stream: the assist
+                    // windows and secondary crops should see those native pixels
+                    const analysis = await this.acquireAnalysisFrame(handle.frame);
+                    try {
+                        const objects = await this.runObjectAssist(analysis.frame, rawExternal, analysis.scaler);
+                        this.ingestAssistedObjects(sensorId, objects);
+                        if (objects.assisted) {
+                            // real boxes: crop the object like the frame pipeline, so face and
+                            // plate secondaries run on the subject instead of the whole scene
+                            // (processExternal only adapts the shape, external boxes get no real tracking)
+                            const objectDetections = this.pipeline.processExternal(objects.detections);
+                            detectionRecord.tick({ assistProcessed: { count: objectDetections.length, bufferedObjects: this.currentDetectionState.object?.detections?.length ?? 0 } });
+                            await this.runSecondariesAndThumbnails(analysis, objectDetections, results);
+                            await this.captureExternalMoment(objectDetections, analysis, results.timestamp);
+                            this.ingestResultsForAllSecondaries(results);
+                            const snapshot = this.buildSnapshot();
+                            detectionRecord.tick({ assistSnapshot: { objects: snapshot.objects.length } });
+                            if (results.thumbnails && results.thumbnails.length > 0) {
+                                snapshot.thumbnails = results.thumbnails;
+                            }
+                            this.eventManager.processResults(snapshot);
                         }
-                        if (sceneJpeg)
-                            snapshot.eventThumbnail = sceneJpeg;
-                        const wantsEventThumb = this.eventManager.needsThumbnail() || this.snapshotWillStartEvent(snapshot);
-                        this.eventManager.processResults(snapshot);
-                        if (wantsEventThumb) {
-                            this.thumbnailer.upgradeEventThumbnailAsync();
+                        else {
+                            // no assist boxes: the diff's motion clusters are the only
+                            // localization, still better than a full-scene card
+                            const anchorDetections = this.anchorMomentDetections(rawExternal);
+                            if (anchorDetections.length > 0) {
+                                await this.captureExternalMoment(anchorDetections, analysis, results.timestamp);
+                            }
+                            const sceneJpeg = await this.secondaries.detectFullFrame(analysis.frame, results);
+                            try {
+                                results.thumbnails = await this.secondaries.generateThumbnails(analysis.frame, analysis.scaler, results);
+                            }
+                            catch (error) {
+                                this.logger.error('Thumbnail generation error:', error);
+                            }
+                            this.ingestResultsForAllSecondaries(results);
+                            const snapshot = this.buildSnapshot();
+                            if (results.thumbnails && results.thumbnails.length > 0) {
+                                snapshot.thumbnails = results.thumbnails;
+                            }
+                            if (sceneJpeg)
+                                snapshot.eventThumbnail = sceneJpeg;
+                            const wantsEventThumb = this.eventManager.needsThumbnail() || this.snapshotWillStartEvent(snapshot);
+                            this.eventManager.processResults(snapshot);
+                            if (wantsEventThumb) {
+                                this.thumbnailer.upgradeEventThumbnailAsync();
+                            }
                         }
                     }
+                    finally {
+                        if (analysis.frame !== handle.frame)
+                            analysis.frame[Symbol.dispose]?.();
+                    }
                 }
                 finally {
                     await handle[Symbol.asyncDispose]();
@@ -511,18 +546,28 @@ let DetectionCoordinator = class DetectionCoordinator {
     hasPlugin(sensorType) {
         return this.plugins.has(sensorType);
     }
-    externalObjectSpanActive() {
-        // the registry holds frame-based providers only; a smart-camera object
-        // sensor (Reolink AI, ONVIF) lives in the feeding set, and its dwell is
-        // what keeps the span open between its report edges
-        if (this.plugins.get(SensorType.Object)?.requiresFrames)
-            return false;
-        for (const [sensorId, sensorType] of this.feedingSensors) {
-            if (sensorType === SensorType.Object && this.dwell.isActive(sensorId))
+    get windowAnchorsWanted() {
+        return this.plugins.get(SensorType.Object)?.requiresFrames === true || this.plugins.hasEligibleObjectAssist();
+    }
+    get localizerWanted() {
+        // the registry only holds frame-based providers; a smart-camera object
+        // sensor (Reolink AI, ONVIF) feeds externally and needs the anchors just
+        // as much — for assist zoom windows and for framing its moment pictures
+        if (this.plugins.has(SensorType.Object) || this.plugins.hasEligibleObjectAssist())
+            return true;
+        for (const sensorType of this.feedingSensors.values()) {
+            if (sensorType === SensorType.Object)
                 return true;
         }
         return false;
     }
+    get windowWantsMainStream() {
+        if (!this.cascade.isActive)
+            return false;
+        if (this.anchorBoxes.length === 0 && this.trackAnchorBoxes.length === 0)
+            return false;
+        return this.windowAnchorsWanted;
+    }
     get cascadeEnabled() {
         return this.config.detectionSettings.cascadeDetection !== false;
     }
@@ -539,7 +584,22 @@ let DetectionCoordinator = class DetectionCoordinator {
         const [w, h] = this.config.interfaceSettings.aspectRatio.split(':').map(Number);
         return w / h;
     }
+    externalObjectSpanActive() {
+        // the registry holds frame-based providers only; a smart-camera object
+        // sensor (Reolink AI, ONVIF) lives in the feeding set, and its dwell is
+        // what keeps the span open between its report edges
+        if (this.plugins.get(SensorType.Object)?.requiresFrames)
+            return false;
+        for (const [sensorId, sensorType] of this.feedingSensors) {
+            if (sensorType === SensorType.Object && this.dwell.isActive(sensorId))
+                return true;
+        }
+        return false;
+    }
     handleCascadeDeactivated() {
+        this.anchorBoxes = [];
+        this.farewellBoxes = [];
+        this.detectionWindow.reset();
         // clear cascade-gated buffer slots, keep motion/audio
         const cs = this.currentDetectionState;
         cs.object = undefined;
@@ -965,18 +1025,26 @@ let DetectionCoordinator = class DetectionCoordinator {
         }
     }
     startAdHocVideoLoopIfNeeded() {
-        if (!this.loopRunning && this.plugins.needsAdHocLoop()) {
+        // localizerWanted covers what the registry cannot see: an object assist or
+        // an external object sensor needs the loop for the localizer's anchors
+        if (!this.loopRunning && (this.plugins.needsAdHocLoop() || this.localizerWanted)) {
             this.adHocVideoLoop = true;
             this.logger.debug('Starting ad-hoc video loop for sensor cascade trigger');
             this.startVideoLoop();
         }
     }
     stopAdHocVideoLoopIfIdle() {
-        if (this.adHocVideoLoop && !this.plugins.shouldVideoBeActive()) {
-            this.adHocVideoLoop = false;
-            this.logger.debug('Stopping ad-hoc video loop — cascade ended');
-            this.stopVideoLoop();
-        }
+        if (!this.adHocVideoLoop || this.plugins.shouldVideoBeActive())
+            return;
+        // the localizer and assist still need frames while the event runs;
+        // stopping on the cascade edge alone churned the main-stream decoder
+        // open/closed mid-event, which starves the hw device (vaapi surface sync
+        // failures) right when the next report arrives
+        if (this.cascade.isActive || this.dwell.hasActive() || this.eventManager.hasActiveSegment())
+            return;
+        this.adHocVideoLoop = false;
+        this.logger.debug('Stopping ad-hoc video loop — idle');
+        this.stopVideoLoop();
     }
     async runDetectionLoop() {
         // the source reconnects on its own, nextFrame simply waits it out
@@ -1009,6 +1077,8 @@ let DetectionCoordinator = class DetectionCoordinator {
                     this.frameScaler.updateHardwareContext(this.frameSource.hardwareContext);
                     this.frameScaler.clearCache();
                     this.thumbnailer.sync(this.mainStreamAnalysisWanted);
+                    this.localizer.reset();
+                    this.detectionWindow.reset();
                 }
                 // motion always reads the low stream: switching its input resolution
                 // would reset the background model on every transition
@@ -1106,7 +1176,7 @@ let DetectionCoordinator = class DetectionCoordinator {
         return undefined;
     }
     updateStreamState() {
-        if (this.worldSpans.size > 0 || this.eventManager.hasActiveSegment()) {
+        if (this.worldSpans.size > 0 || this.eventManager.hasActiveSegment() || this.windowWantsMainStream) {
             this.idleSince = 0;
             if (!this.mainStreamActive && this.mainStreamAvailable) {
                 this.mainStreamActive = true;
@@ -1141,6 +1211,7 @@ let DetectionCoordinator = class DetectionCoordinator {
         const t0 = Date.now();
         let motionDetected = false;
         let objectDetections = [];
+        let staticDetections = [];
         const results = { timestamp: t0 };
         // while the PTZ repositions: motion and object keep running so the
         // background model and track ids survive the pan, but nothing below
@@ -1160,6 +1231,8 @@ let DetectionCoordinator = class DetectionCoordinator {
                 return;
             if (motionFrame) {
                 this.lastMotionAt = t0;
+                if (this.localizerWanted)
+                    this.feedLocalizer(motionFrame, t0, ptzSuppressed);
                 try {
                     const motionInferStart = Date.now();
                     const result = await motionPlugin.proxy.detectMotion(motionFrame);
@@ -1193,6 +1266,17 @@ let DetectionCoordinator = class DetectionCoordinator {
             // no frame-based motion plugin, the cascade is the only motion-equivalent signal
             motionDetected = true;
             results.cascadeTriggered = true;
+            // the localizer scales its own gray here, nobody else needed one
+            const localizeDue = t0 - this.lastLocalizeAt >= MOTION_INTERVAL_MS - TICK_SLACK_MS;
+            if (localizeDue && this.localizerWanted) {
+                const scaleStart = Date.now();
+                const gray = await this.scaleForMotionInput(motionRawFrame);
+                this.perf.scaleMs += Date.now() - scaleStart;
+                if (!this.loopRunning)
+                    return;
+                if (gray)
+                    this.feedLocalizer(gray, t0, ptzSuppressed);
+            }
         }
         // detected is re-derived after zone filtering, the plugin's own flag is ignored
         if (results.motion && motionPlugin) {
@@ -1206,25 +1290,16 @@ let DetectionCoordinator = class DetectionCoordinator {
         const objectPlugin = this.plugins.get(SensorType.Object);
         const shouldDetectObjects = objectPlugin?.requiresFrames === true && (ptzSuppressed || !this.cascadeEnabled || this.cascade.isActive);
         if (shouldDetectObjects) {
-            const scaleStart = Date.now();
-            const objectFrame = await this.scaleForObject(analysis.frame, analysis.scaler);
-            this.perf.scaleMs += Date.now() - scaleStart;
-            if (!this.loopRunning)
-                return;
-            if (objectFrame) {
-                try {
-                    const inferStart = Date.now();
-                    const result = await PromiseTimeout(objectPlugin.proxy.detectObjects(objectFrame.model), DETECT_TIMEOUT_MS, undefined, `Object detection timed out after ${DETECT_TIMEOUT_MS}ms`);
-                    this.perf.objectMs += Date.now() - inferStart;
-                    this.perf.objectCount++;
-                    if (!this.loopRunning)
-                        return;
+            try {
+                const detected = await this.runObjectDetection(objectPlugin, analysis, results.motion?.detections ?? []);
+                if (!this.loopRunning)
+                    return;
+                if (detected) {
                     this.trackDetectionCadence();
                     // run the pipeline even on empty frames so Norfair advances its
                     // Kalman state; the pose delta keeps predictions stable across pans
                     const poseDelta = this.ptzAutotracker.consumePoseDelta();
                     const postStart = Date.now();
-                    const detected = FrameScaler.undoLetterbox(ensureDetectionBoxes(result.detections), objectFrame.geometry);
                     const pipelineResult = this.pipeline.process(detected, poseDelta);
                     this.perf.postMs += Date.now() - postStart;
                     // the tick that opens a span analysed the low stream (the switch lands
@@ -1258,25 +1333,35 @@ let DetectionCoordinator = class DetectionCoordinator {
                         return cx > -0.1 && cx < 1.1 && cy > -0.1 && cy < 1.1;
                     });
                     objectDetections = visibleTracks.filter((t) => !t.trackLost);
-                    results.object = { ...result, detections: visibleTracks };
+                    results.object = { detected: objectDetections.length > 0, detections: visibleTracks };
+                    staticDetections = pipelineResult.staticTracks;
                     if (pipelineResult.crossings.length > 0)
                         results.lineCrossings = pipelineResult.crossings;
                     // incl. extrapolated tracks, otherwise a single missed detector
                     // frame flips the autotracker into LOST/REACQUIRE churn
                     this.ptzAutotracker.handleObjectDetections(pipelineResult.tracked);
-                }
-                catch (error) {
-                    if (!this.loopRunning || isNoRespondersError(error))
-                        return;
-                    this.logger.error('Object detection error:', error);
+                    // live tracks anchor the next tick's window; a track that just died
+                    // gets one farewell look at its last position
+                    this.trackAnchorBoxes = objectDetections.map((t) => t.box);
+                    for (const id of pipelineResult.removed) {
+                        const last = this.lastTrackedById.get(id);
+                        if (last)
+                            this.farewellBoxes.push(last);
+                    }
+                    this.lastTrackedById = new Map(pipelineResult.tracked.filter((t) => t.trackId !== undefined).map((t) => [t.trackId, t.box]));
                 }
             }
+            catch (error) {
+                if (!this.loopRunning || isNoRespondersError(error))
+                    return;
+                this.logger.error('Object detection error:', error);
+            }
         }
         // tracker and autotracker are fed, everything event-facing stays quiet
         if (ptzSuppressed) {
             // object boxes stay live in the UI through the move, bbox publish only
             if (objectPlugin && results.object) {
-                this.writeSensorProperties(objectPlugin.sensorId, { detections: results.object.detections });
+                this.writeSensorProperties(objectPlugin.sensorId, { detections: results.object.detections, staticDetections });
             }
             // keep an already-running object dwell alive so a chase doesn't end its
             // own event, but never activate it from suppressed frames
@@ -1295,6 +1380,7 @@ let DetectionCoordinator = class DetectionCoordinator {
             this.ingestDetectionResult(SensorType.Object, objectPlugin.sensorId, {
                 detected: objectDetections.length > 0,
                 detections: results.object.detections,
+                staticDetections,
             });
         }
         if (!this.loopRunning)
@@ -1493,6 +1579,41 @@ let DetectionCoordinator = class DetectionCoordinator {
             return;
         await this.captureMoment(subject, bestScore, trigger, result.tracked, analysis, at);
     }
+    anchorMomentDetections(reported) {
+        const anchors = [...this.anchorBoxes, ...this.farewellBoxes];
+        if (anchors.length === 0 || reported.length === 0)
+            return [];
+        const subject = reported.find((d) => d.label !== 'motion') ?? reported[0];
+        const clusters = clusterBoxes(anchors);
+        const largest = Math.max(...clusters.map((b) => b.width * b.height));
+        // specks would drag the base union across the frame
+        return clusters.filter((b) => b.width * b.height >= largest * 0.2).map((b) => ({ label: subject.label, confidence: subject.confidence, box: b }));
+    }
+    async captureExternalMoment(detections, analysis, at) {
+        let subject;
+        let bestScore = 0;
+        for (const detection of detections) {
+            const box = detection.box;
+            if (!box)
+                continue;
+            const score = detection.confidence * Math.sqrt(box.width * box.height);
+            if (score > bestScore) {
+                bestScore = score;
+                subject = detection;
+            }
+        }
+        if (!subject)
+            return;
+        if (!this.eventManager.wantsMoment(MOMENT_RANK_OBJECT, bestScore, 'low', at))
+            return;
+        const target = { subject: subject.box, base: unionBox(detections.map((d) => d.box).filter(Boolean)) };
+        const rendered = await this.renderMoment(target, analysis);
+        if (!rendered)
+            return;
+        this.eventManager.offerMoment({ strip: rendered.strip, card: rendered.card, capturedAt: at, score: bestScore, rank: MOMENT_RANK_OBJECT, stream: 'low' });
+        // debugging
+        await this.recordMoment(target, rendered, analysis, `${subject.label}-external`, bestScore, 'external', at);
+    }
     async captureMoment(subject, score, trigger, tracked, analysis, at) {
         const box = { x: subject.x, y: subject.y, width: subject.width, height: subject.height };
         const moving = tracked.filter((d) => (d.trackSpeed ?? 0) >= MOMENT_MOVING_SPEED).map((d) => d.box);
@@ -1527,7 +1648,6 @@ let DetectionCoordinator = class DetectionCoordinator {
         // legible one frames the shot, the rest stay their own attribute tiles
         let best;
         let bestScore = -1;
-        let bestCard;
         for (const candidate of candidates) {
             const cut = results.thumbnails?.find((t) => t.label === candidate.label);
             // a subject cut off by the frame edge makes a poor card
@@ -1542,20 +1662,21 @@ let DetectionCoordinator = class DetectionCoordinator {
             if (score > bestScore) {
                 bestScore = score;
                 best = candidate;
-                bestCard = cut?.jpeg;
             }
         }
         // ask before cutting: the same face rides along on every tick of a segment
         // and re-encoding for a picture that loses anyway is pure cost
         if (!best || !this.eventManager.wantsMoment(MOMENT_RANK_ATTRIBUTE, bestScore, analysis.isMainStream ? 'main' : 'low', at))
             return;
+        // the card frames the whole person with the face as its center, not the
+        // bare face box: the zoomed face already exists as an attribute tile, and
+        // a misdetected "face" zoomed to card size ruins the event picture
         const target = {
             subject: best.box,
+            base: best.parentBox ?? best.box,
             hint: best.parentBox ? directionBetween(best.box, best.parentBox) : undefined,
         };
-        // the attribute tile was cut from this frame with this very target, so the
-        // card is already encoded; only the 16:9 companion is still missing
-        const rendered = await this.renderMoment(target, analysis, bestCard);
+        const rendered = await this.renderMoment(target, analysis);
         if (!rendered)
             return;
         const stream = analysis.isMainStream ? 'main' : 'low';
@@ -1619,10 +1740,42 @@ let DetectionCoordinator = class DetectionCoordinator {
         const motionPlugin = this.plugins.get(SensorType.Motion);
         if (!motionPlugin?.requiresFrames)
             return undefined;
+        return this.scaleForMotionInput(rawFrame);
+    }
+    async scaleForMotionInput(rawFrame) {
         const maxWidth = MOTION_WIDTH_MAP[this.motionResolution];
         const scaled = await this.frameScaler.scaleProportional(rawFrame, maxWidth, 'gray');
         return scaled ? this.frameScaler.toVideoFrameData(scaled) : undefined;
     }
+    feedLocalizer(gray, t0, ptzSuppressed) {
+        this.lastLocalizeAt = t0;
+        const data = Buffer.isBuffer(gray.data) ? gray.data : Buffer.from(gray.data);
+        const boxes = this.localizer.localize({ data, width: gray.width, height: gray.height });
+        // during ego-motion the diff is garbage, feeding keeps the reference alive
+        this.anchorBoxes = ptzSuppressed ? [] : boxes;
+    }
+    async seedLocalizerReference() {
+        if (!this.localizerWanted || !this.frameSource.decodeOldestKeyframe)
+            return;
+        try {
+            const frame = await this.frameSource.decodeOldestKeyframe();
+            if (!frame)
+                return;
+            try {
+                const gray = await this.scaleForMotionInput(frame);
+                if (gray) {
+                    const data = Buffer.isBuffer(gray.data) ? gray.data : Buffer.from(gray.data);
+                    this.localizer.seedReference({ data, width: gray.width, height: gray.height });
+                }
+            }
+            finally {
+                frame[Symbol.dispose]?.();
+            }
+        }
+        catch (error) {
+            this.logger.debug('Localizer reference seed failed:', error);
+        }
+    }
     trackDetectionCadence() {
         const now = Date.now();
         const previous = this.lastObjectCallAt;
@@ -1656,6 +1809,43 @@ let DetectionCoordinator = class DetectionCoordinator {
             return undefined;
         return this.scaleFrameForPlugin(rawFrame, objectPlugin, scaler);
     }
+    async runObjectDetection(objectPlugin, analysis, pluginMotionBoxes) {
+        const inputSpec = objectPlugin.modelSpec?.input;
+        const motionAnchors = [...this.anchorBoxes, ...pluginMotionBoxes.map((d) => d.box).filter((box) => box !== undefined), ...this.farewellBoxes];
+        this.farewellBoxes = [];
+        const windows = isVideoInputSpec(inputSpec)
+            ? this.detectionWindow.plan(motionAnchors, this.trackAnchorBoxes, analysis.frame.width, analysis.frame.height, inputSpec.width)
+            : null;
+        if (windows && windows.length > 0) {
+            this.perf.zoomTicks++;
+            this.perf.zoomWindows += windows.length;
+            detectionRecord.zoom({ kind: 'object', anchors: motionAnchors, tracks: this.trackAnchorBoxes, windows });
+            const inferStart = Date.now();
+            const perWindow = await Promise.all(windows.map((window) => this.detectInWindow(objectPlugin, analysis.frame, analysis.scaler, window, inputSpec)));
+            this.perf.objectMs += Date.now() - inferStart;
+            this.perf.objectCount++;
+            return mergeWindowDetections(perWindow.flat());
+        }
+        const scaleStart = Date.now();
+        const objectFrame = await this.scaleForObject(analysis.frame, analysis.scaler);
+        this.perf.scaleMs += Date.now() - scaleStart;
+        if (!objectFrame || !this.loopRunning)
+            return undefined;
+        const inferStart = Date.now();
+        const result = await PromiseTimeout(objectPlugin.proxy.detectObjects(objectFrame.model), DETECT_TIMEOUT_MS, undefined, `Object detection timed out after ${DETECT_TIMEOUT_MS}ms`);
+        this.perf.objectMs += Date.now() - inferStart;
+        this.perf.objectCount++;
+        return FrameScaler.undoLetterbox(ensureDetectionBoxes(result.detections), objectFrame.geometry);
+    }
+    async detectInWindow(plugin, frame, scaler, window, spec) {
+        const scaleStart = Date.now();
+        const cropped = await scaler.cropToSpec(frame, window, spec);
+        this.perf.scaleMs += Date.now() - scaleStart;
+        if (!cropped)
+            return [];
+        const result = await PromiseTimeout(plugin.proxy.detectObjects(scaler.toVideoFrameData(cropped.padded, 'model')), DETECT_TIMEOUT_MS, undefined, `Object detection timed out after ${DETECT_TIMEOUT_MS}ms`);
+        return FrameScaler.undoLetterbox(ensureDetectionBoxes(result.detections), cropped.geometry);
+    }
     async scaleFrameForPlugin(rawFrame, plugin, scaler = this.frameScaler) {
         const inputSpec = plugin.modelSpec?.input;
         if (!isVideoInputSpec(inputSpec))
@@ -1672,27 +1862,34 @@ let DetectionCoordinator = class DetectionCoordinator {
     ingestAssistedObjects(sensorId, objects) {
         if (!objects.assisted)
             return;
+        // the assist returns raw detector output; the frame path dedupes via NMS
+        // before ingest, so this path must merge too or near-identical candidate
+        // boxes inflate the object count
+        const merged = this.pipeline.runMergeAndZoneFilter(objects.detections);
         const refiltered = this.applyExternalDetectionFilters(SensorType.Object, {
-            detected: objects.detections.length > 0,
-            detections: objects.detections,
+            detected: merged.length > 0,
+            detections: merged,
+        });
+        detectionRecord.tick({
+            assistIngest: {
+                raw: objects.detections.map((d) => ({ label: d.label, score: d.confidence, box: d.box })),
+                merged: merged.length,
+                kept: (refiltered.detections ?? []).length,
+            },
         });
         this.ingestDetectionResult(SensorType.Object, sensorId, refiltered);
     }
-    async runObjectAssist(frame, reported) {
+    async runObjectAssist(frame, reported, scaler = this.frameScaler) {
         if (!this.plugins.hasEligibleObjectAssist()) {
             return { detections: reported, assisted: false };
         }
         const assist = this.plugins.get(SensorType.ObjectAssist);
-        const scaled = await this.scaleFrameForPlugin(frame, assist);
-        if (!scaled)
-            return { detections: reported, assisted: false };
         try {
             const inferStart = Date.now();
-            const result = await PromiseTimeout(assist.proxy.detectObjects(scaled.model), DETECT_TIMEOUT_MS, undefined, `Object assist timed out after ${DETECT_TIMEOUT_MS}ms`);
+            const boxed = await this.assistDetections(assist, frame, scaler, reported);
             this.perf.assistMs += Date.now() - inferStart;
             this.perf.assistCount++;
             const reportedLabels = new Set(reported.map((d) => d.label.toLowerCase()));
-            const boxed = FrameScaler.undoLetterbox(ensureDetectionBoxes(result?.detections ?? []), scaled.geometry);
             const found = boxed.filter((d) => reportedLabels.size === 0 || reportedLabels.has(d.label.toLowerCase()));
             if (found.length === 0)
                 return { detections: reported, assisted: false };
@@ -1706,6 +1903,31 @@ let DetectionCoordinator = class DetectionCoordinator {
             return { detections: reported, assisted: false };
         }
     }
+    async assistDetections(assist, frame, scaler, reported) {
+        const inputSpec = assist.modelSpec?.input;
+        const reportedBoxes = reported.map((d) => d.box).filter((box) => box !== undefined);
+        // a coordinate-less report still has the localizer's diff anchors: zoom
+        // the assist into the moving regions instead of letterboxing the whole
+        // frame, which shrinks the subject to nothing on a main-stream frame
+        const anchors = reportedBoxes.length > 0 ? reportedBoxes : [...this.anchorBoxes, ...this.farewellBoxes];
+        const pad = reportedBoxes.length > 0 ? TRACK_PAD : MOTION_PAD;
+        if (isVideoInputSpec(inputSpec) && anchors.length > 0) {
+            const windows = planWindowsOnce(anchors, pad, frame.width, frame.height, inputSpec.width);
+            if (windows.length > 0) {
+                detectionRecord.zoom({ kind: 'assist', anchors, windows });
+                const perWindow = await Promise.all(windows.map((window) => this.detectInWindow(assist, frame, scaler, window, inputSpec)));
+                return mergeWindowDetections(perWindow.flat());
+            }
+        }
+        detectionRecord.tick({
+            assistFullFrame: { anchors: anchors.length, videoSpec: isVideoInputSpec(inputSpec), loop: this.loopRunning, cascade: this.cascade.isActive },
+        });
+        const scaled = await this.scaleFrameForPlugin(frame, assist, scaler);
+        if (!scaled)
+            return [];
+        const result = await PromiseTimeout(assist.proxy.detectObjects(scaled.model), DETECT_TIMEOUT_MS, undefined, `Object assist timed out after ${DETECT_TIMEOUT_MS}ms`);
+        return FrameScaler.undoLetterbox(ensureDetectionBoxes(result?.detections ?? []), scaled.geometry);
+    }
 };
 __decorate([
     RPCMethod,