@xenon-device-management/xenon 1.11.1 → 1.11.2

package/lib/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@xenon-device-management/xenon",
-    "version": "1.11.1",
+    "version": "1.11.2",
     "description": "Xenon - Intelligent Mobile Infrastructure. A self-healing device orchestration platform for Appium.",
     "main": "./lib/src/index.js",
     "exports": {

package/lib/src/device-managers/android/AndroidStreamService.js

@@ -128,6 +128,29 @@ let AndroidStreamService = class AndroidStreamService {
     shouldIdleCapture(session) {
         return session.status === 'running' && session.viewerCount === 0;
     }
+    /**
+     * Log a stall once per episode, whichever path notices it first.
+     *
+     * Both the capture loop's catch and the MJPEG writer call this. The writer
+     * matters most: ending responses on a stall drops the viewer count to zero,
+     * which makes `shouldIdleCapture` true, which stops the loop attempting
+     * further captures — so the catch may never run again. A live cable-pull run
+     * hit exactly that and logged nothing at all.
+     *
+     * Returns whether this call did the logging, so the caller can decide what to
+     * say instead.
+     */
+    announceStallOnce(session, now, lastError) {
+        if (!session.captureHealth.takeStallAnnouncement(now))
+            return false;
+        const seconds = Math.round(session.captureHealth.failingForMs(now) / 1000);
+        logger_1.default.warn(`[${session.udid}] Device has not answered screencap for ${seconds}s ` +
+            `(${session.captureHealth.consecutiveFailures} consecutive failures` +
+            `${lastError ? `, last: ${lastError}` : ''}). Stream clients are being disconnected ` +
+            'rather than served a frozen frame; the stream restarts on the next request once the ' +
+            'device recovers.');
+        return true;
+    }
     captureLoop(udid, session) {
         return __awaiter(this, void 0, void 0, function* () {
             logger_1.default.info(`[${udid}] Background capture loop started.`);
@@ -143,8 +166,11 @@ let AndroidStreamService = class AndroidStreamService {
                     yield new Promise((r) => setTimeout(r, 150));
                     continue;
                 }
+                // Hoisted out of the try so the catch can date a failure from when the
+                // attempt began: a 15s ADB timeout means the device had already been
+                // silent for 15s by the time we reach the catch.
+                const startTime = Date.now();
                 try {
-                    const startTime = Date.now();
                     // High-Speed Binary Snapshot
                     const screenshot = yield DeviceLockManager_1.deviceLock.acquire(udid, () => __awaiter(this, void 0, void 0, function* () {
                         return yield new Promise((resolve, reject) => {
@@ -197,6 +223,7 @@ let AndroidStreamService = class AndroidStreamService {
                                 logger_1.default.info(`[${udid}] First Android frame successfully captured and converted.`);
                             }
                             session.latestFrameTimestamp = Date.now();
+                            session.captureHealth.recordSuccess();
                         }
                     }
                     else {
@@ -207,7 +234,11 @@ let AndroidStreamService = class AndroidStreamService {
                     yield new Promise((r) => setTimeout(r, delay));
                 }
                 catch (e) {
-                    logger_1.default.debug(`[${udid}] Capture failure: ${e.message}`);
+                    const now = Date.now();
+                    const { consecutive } = session.captureHealth.recordFailure(startTime, now);
+                    if (!this.announceStallOnce(session, now, e.message)) {
+                        logger_1.default.debug(`[${udid}] Capture failure (${consecutive} consecutive): ${e.message}`);
+                    }
                     yield new Promise((r) => setTimeout(r, 1000));
                 }
             }
@@ -228,6 +259,7 @@ let AndroidStreamService = class AndroidStreamService {
                         status: existing.status,
                         serverListening: ((_a = existing.server) === null || _a === void 0 ? void 0 : _a.listening) === true,
                         hasFrame: existing.latestFrame !== undefined,
+                        captureStalled: existing.captureHealth.isStalled(Date.now()),
                     });
                     if (decision.reuse) {
                         if (existing)
@@ -263,6 +295,7 @@ let AndroidStreamService = class AndroidStreamService {
                             viewerCount: 0,
                             adbPath,
                             adbHostArgs,
+                            captureHealth: new androidStreamReuse_1.CaptureHealth(),
                         };
                         this.sessions.set(udid, candidate);
                         try {
@@ -433,6 +466,26 @@ let AndroidStreamService = class AndroidStreamService {
                         !res.writable) {
                         return;
                     }
+                    // The device has gone silent, so `latestFrame` is now a still image.
+                    // End the response instead of rewriting it forever: ffmpeg finalises
+                    // the mp4 with the real footage it captured, and browser clients fall
+                    // into their normal stream-retry — which re-enters startStream, whose
+                    // health check restarts the stream. Serving on would have produced a
+                    // recording that looks valid and is a photograph (issue #200).
+                    const nowMs = Date.now();
+                    if (session.captureHealth.isStalled(nowMs)) {
+                        // Announce from here too: this path is often the only one that gets
+                        // to notice, because the disconnect below is what idles the capture
+                        // loop in the first place.
+                        this.announceStallOnce(session, nowMs);
+                        try {
+                            res.end();
+                        }
+                        catch (_a) {
+                            /* ignore */
+                        }
+                        return;
+                    }
                     if (session.latestFrame) {
                         try {
                             res.write('--BoundaryString\r\n');
@@ -441,7 +494,7 @@ let AndroidStreamService = class AndroidStreamService {
                             res.write(session.latestFrame);
                             res.write('\r\n');
                         }
-                        catch (_a) {
+                        catch (_b) {
                             return;
                         }
                     }

package/lib/src/device-managers/android/androidStreamReuse.js

@@ -20,6 +20,7 @@
  * maps its live session onto the view below.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.CaptureHealth = exports.CAPTURE_STALL_MS = void 0;
 exports.decideAndroidStreamReuse = decideAndroidStreamReuse;
 function decideAndroidStreamReuse(session) {
     if (!session)
@@ -34,5 +35,94 @@ function decideAndroidStreamReuse(session) {
         return { reuse: false, reason: 'server-closed' };
     if (!session.hasFrame)
         return { reuse: false, reason: 'no-frame' };
+    // Ordered after hasFrame: "never captured anything" and "captured, then went
+    // silent" are different problems, and the reason ends up in a log line.
+    if (session.captureStalled)
+        return { reuse: false, reason: 'capture-stalled' };
     return { reuse: true };
 }
+/**
+ * How long the device may go without producing a frame before anything we still
+ * hold is treated as a still image rather than a live feed.
+ *
+ * Deliberately generous. A single failed `screencap`, a device briefly held by
+ * the interaction lock, or one 15s ADB timeout must not tear down a live
+ * preview or an in-flight recording — the cure for that would be the
+ * over-eager-restart failure mode #198 was careful to avoid.
+ */
+exports.CAPTURE_STALL_MS = 10000;
+/**
+ * Tracks whether the device is still answering `screencap`.
+ *
+ * Why this exists (issue #200): the capture loop swallows capture errors and
+ * loops on `while (status === 'running' || status === 'starting')`, so a device
+ * that goes away — unplugged, adb killed, reboot — never changes the session's
+ * status. `latestFrame` is only ever assigned, never cleared, so the MJPEG
+ * server happily rewrote the last good JPEG every 60ms: a frozen preview, and a
+ * recording that ffprobe calls healthy but which is a still photograph. That is
+ * worse than the 0-byte failure of #194/#196, because it looks valid.
+ *
+ * The measure is time-since-the-device-last-answered, not a failure count: a
+ * fast `ADB Exit 1` and a 15s `ADB Timeout` are wildly different amounts of
+ * frozen video for the same count. It is also immune to the idle path —
+ * `shouldIdleCapture` skips capture entirely when nobody is watching, so an
+ * idle-but-healthy session records neither successes nor failures and simply
+ * stays in whatever state it was left in.
+ */
+class CaptureHealth {
+    constructor() {
+        this.consecutive = 0;
+        this.announced = false;
+    }
+    /** A frame was captured — the device is answering. */
+    recordSuccess() {
+        this.failingSince = undefined;
+        this.consecutive = 0;
+        this.announced = false;
+    }
+    /**
+     * A capture attempt threw.
+     *
+     * `attemptStartedAt` (not `now`) dates the stall, because a 15s ADB timeout
+     * means the device had already been silent for 15s by the time we got here.
+     */
+    recordFailure(attemptStartedAt, now) {
+        if (this.failingSince === undefined)
+            this.failingSince = attemptStartedAt;
+        this.consecutive += 1;
+        return { consecutive: this.consecutive, stalled: this.isStalled(now) };
+    }
+    /**
+     * Claim the right to announce this stall episode: true for the first caller
+     * that observes the session stalled, false for every caller after it until a
+     * successful frame resets the episode.
+     *
+     * Why a claim rather than a flag computed inside `recordFailure`: ending
+     * client responses on a stall drops `viewerCount` to 0, `shouldIdleCapture`
+     * then stops the capture loop attempting anything more, and so the failure
+     * that would have crossed the threshold never happens — the catch never gets
+     * to warn. A live cable-pull run showed exactly that: 10 failures spanning
+     * 9.214s against a 10s threshold, then silence, so a device that had vanished
+     * produced no warning at all. Either path can claim it now, and neither
+     * double-logs.
+     */
+    takeStallAnnouncement(now) {
+        if (!this.isStalled(now) || this.announced)
+            return false;
+        this.announced = true;
+        return true;
+    }
+    /** Consecutive failed capture attempts in the current episode, for logging. */
+    get consecutiveFailures() {
+        return this.consecutive;
+    }
+    /** Capture has been failing long enough that any cached frame is a still. */
+    isStalled(now) {
+        return this.failingSince !== undefined && now - this.failingSince >= exports.CAPTURE_STALL_MS;
+    }
+    /** How long the device has been silent, for logging. 0 when healthy. */
+    failingForMs(now) {
+        return this.failingSince === undefined ? 0 : now - this.failingSince;
+    }
+}
+exports.CaptureHealth = CaptureHealth;

package/lib/src/services/VideoPipelineService.js

@@ -380,26 +380,60 @@ let VideoPipelineService = class VideoPipelineService {
                     logger_1.default.error(`[VideoPipeline] FFMPEG Error [${sessionId}]: ${msg}`);
                 }
             });
-            ffmpegProc.on('exit', (code) => {
-                if (code !== 0 && code !== null) {
-                    logger_1.default.warn(`[VideoPipeline] FFMPEG for ${sessionId} exited with code ${code}`);
-                }
-                this.activeRecordings.delete(sessionId);
-            });
+            ffmpegProc.on('exit', (code) => this.handleProcessExit(sessionId, code, options.onExit));
             this.activeRecordings.set(sessionId, ffmpegProc);
             this.recordingPaths.set(sessionId, outputPath);
         });
     }
+    /**
+     * A recording's ffmpeg exited. Reports it to `onExit` only when nobody asked
+     * for it.
+     *
+     * `stopRecording` drops the handle *before* awaiting the exit, so a still
+     * present entry is the signal that this exit was not requested — the source
+     * ended or ffmpeg died. Getting that backwards would run the caller's
+     * source-ended path on every ordinary Stop as well.
+     *
+     * Split out from the listener so the decision is testable without spawning a
+     * process.
+     */
+    handleProcessExit(sessionId, code, onExit) {
+        var _a;
+        if (code !== 0 && code !== null) {
+            logger_1.default.warn(`[VideoPipeline] FFMPEG for ${sessionId} exited with code ${code}`);
+        }
+        const unrequested = this.activeRecordings.delete(sessionId);
+        if (!unrequested || !onExit)
+            return;
+        logger_1.default.info(`[VideoPipeline] FFMPEG for ${sessionId} exited on its own (${code === null ? 'signal' : `code ${code}`})`);
+        try {
+            onExit(code);
+        }
+        catch (err) {
+            logger_1.default.warn(`[VideoPipeline] onExit handler threw for ${sessionId}: ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+        }
+    }
     /**
      * Stop recording and return the relative asset path
      */
     stopRecording(sessionId) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a;
+            var _a, _b;
             const proc = this.activeRecordings.get(sessionId);
             const recordedPath = this.recordingPaths.get(sessionId);
             if (!proc) {
                 logger_1.default.info(`[VideoPipeline] No active recording process for ${sessionId}, returning stored path if any.`);
+                // ffmpeg already exited on its own. The file still needs the faststart
+                // remux the normal stop path performs, or it stays a fragmented mp4 that
+                // some players will not scrub.
+                if (recordedPath) {
+                    try {
+                        yield remuxToStandardMp4(recordedPath);
+                    }
+                    catch (err) {
+                        logger_1.default.warn(`[VideoPipeline] Remux failed for ${sessionId} (leaving fMP4): ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+                    }
+                }
                 const relativePath = recordedPath
                     ? path_1.default.relative(config_1.config.sessionAssetsPath, recordedPath)
                     : null;
@@ -407,15 +441,17 @@ let VideoPipelineService = class VideoPipelineService {
                 return relativePath;
             }
             logger_1.default.info(`[VideoPipeline] Stopping recording for ${sessionId}`);
-            yield waitForFfmpegExit(proc, `recording ${sessionId}`);
+            // Drop the handle before awaiting: the exit that follows is one we asked
+            // for, and the exit listener must not report it as a source-initiated end.
             this.activeRecordings.delete(sessionId);
+            yield waitForFfmpegExit(proc, `recording ${sessionId}`);
             this.recordingPaths.delete(sessionId);
             if (recordedPath) {
                 try {
                     yield remuxToStandardMp4(recordedPath);
                 }
                 catch (err) {
-                    logger_1.default.warn(`[VideoPipeline] Remux failed for ${sessionId} (leaving fMP4): ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+                    logger_1.default.warn(`[VideoPipeline] Remux failed for ${sessionId} (leaving fMP4): ${(_b = err === null || err === void 0 ? void 0 : err.message) !== null && _b !== void 0 ? _b : err}`);
                 }
             }
             return recordedPath ? path_1.default.relative(config_1.config.sessionAssetsPath, recordedPath) : null;

package/lib/src/services/recording/RecordingOrchestrator.js

@@ -69,6 +69,7 @@ const device_service_1 = require("../../data-service/device-service");
 const device_store_1 = require("../../data-service/device-store");
 const manualLock_1 = require("./manualLock");
 const ensureMjpegForRecording_1 = require("./ensureMjpegForRecording");
+const probeDuration_1 = require("./probeDuration");
 const attributes_1 = require("../telemetry/attributes");
 const logger_1 = __importDefault(require("../../logger"));
 const ArtifactStore_1 = require("../artifacts/ArtifactStore");
@@ -127,7 +128,13 @@ class RecordingError extends Error {
 exports.RecordingError = RecordingError;
 let RecordingOrchestrator = class RecordingOrchestrator {
     constructor(deps = {}) {
-        var _a, _b, _c, _d, _e, _f, _g, _h;
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+        /**
+         * Recording ids currently being closed out. Both the user-initiated stop and
+         * an ffmpeg self-exit finalize rows, and they can land at the same instant —
+         * this keeps exactly one of them doing it.
+         */
+        this.finalizing = new Set();
         this._deps = {
             busyPrecheck: (_a = deps.busyPrecheck) !== null && _a !== void 0 ? _a : typedi_1.Container.get(busy_precheck_1.BusyPrecheck),
             store: (_b = deps.store) !== null && _b !== void 0 ? _b : typedi_1.Container.get(recording_store_1.RecordingStore),
@@ -138,6 +145,7 @@ let RecordingOrchestrator = class RecordingOrchestrator {
         this.blockDeviceFn = (_f = deps.blockDeviceFn) !== null && _f !== void 0 ? _f : device_service_1.blockDevice;
         this.unblockDeviceFn = (_g = deps.unblockDeviceFn) !== null && _g !== void 0 ? _g : device_service_1.unblockDevice;
         this.ensureMjpegPortFn = (_h = deps.ensureMjpegPortFn) !== null && _h !== void 0 ? _h : ensureMjpegForRecording_1.ensureMjpegForRecording;
+        this.probeDurationMsFn = (_j = deps.probeDurationMsFn) !== null && _j !== void 0 ? _j : probeDuration_1.probeVideoDurationMs;
     }
     get busyPrecheck() {
         return this._deps.busyPrecheck;
@@ -265,6 +273,9 @@ let RecordingOrchestrator = class RecordingOrchestrator {
                         udid,
                         outputPath: filePath,
                         mjpegPort,
+                        // ffmpeg can end before anyone presses Stop — most often because the
+                        // device stopped answering and the MJPEG server disconnected it.
+                        onExit: (code) => void this.handleSourceEnded(id, code),
                     });
                     recordings.push({ id, udid, status: 'RECORDING' });
                 }
@@ -341,6 +352,125 @@ let RecordingOrchestrator = class RecordingOrchestrator {
             return { groupId, recordings, startedAt, compositeEnabled };
         });
     }
+    /**
+     * Close out one recording: stop its ffmpeg, size the file, decide the terminal
+     * status, persist it, release the gate slot and the device lock, and record
+     * the metrics. Shared by the user-initiated stop path and by
+     * handleSourceEnded, so the two can never drift.
+     */
+    finalizeRecording(r_1) {
+        return __awaiter(this, arguments, void 0, function* (r, opts = {}) {
+            var _a, _b, _c, _d, _e;
+            // Both entry points land here and this is where the instruments are used;
+            // handleSourceEnded has no reason to know they are lazily created.
+            ensureOtelInstruments();
+            let status = 'STOPPED';
+            let failReason;
+            try {
+                yield this.videoPipeline.stopRecording(r.id);
+            }
+            catch (err) {
+                status = 'FAILED';
+                failReason = (_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : 'ffmpeg_stop_failed';
+            }
+            // How long the video actually is, not how long the session was open. Those
+            // agree only while capture keeps up: a recording whose device died at 26s
+            // and was stopped 5 minutes later reported 5m36s for a 35s file (#204).
+            // Wall-clock is not lost — started_at and ended_at are both persisted, so
+            // it stays derivable, while the media duration is not recoverable later
+            // without re-reading the file. Probed after stopRecording so the faststart
+            // remux has happened and the header carries a duration; falls back to
+            // wall-clock whenever the probe can't answer.
+            const wallClockMs = r.started_at ? Date.now() - new Date(r.started_at).getTime() : undefined;
+            let durationMs = wallClockMs;
+            try {
+                const probed = yield this.probeDurationMsFn(r.file_path);
+                if (probed !== undefined)
+                    durationMs = probed;
+            }
+            catch (err) {
+                recLog.debug(`duration probe failed for ${r.id}, using wall-clock: ${(_b = err === null || err === void 0 ? void 0 : err.message) !== null && _b !== void 0 ? _b : err}`);
+            }
+            let sizeBytes;
+            try {
+                sizeBytes = fs.statSync(r.file_path).size;
+            }
+            catch (_f) {
+                // File may not exist on FAIL; leave size undefined.
+            }
+            if (status === 'STOPPED' && (sizeBytes === undefined || sizeBytes < MIN_PLAYABLE_MP4_BYTES)) {
+                status = 'FAILED';
+                failReason = 'empty_or_corrupt_mp4';
+                recLog.warn(`Recording ${r.id} for ${r.device_udid} produced unusable file ` +
+                    `(size=${sizeBytes !== null && sizeBytes !== void 0 ? sizeBytes : 'missing'} bytes)`);
+            }
+            // A recording whose source ended still produced valid video — just less of
+            // it than was asked for. Keeping it STOPPED leaves the mp4 downloadable;
+            // recording why it ended keeps "this is shorter than you expected" visible
+            // in the data rather than only in the log.
+            if (opts.endedBySource && status === 'STOPPED') {
+                failReason = 'source_ended';
+            }
+            try {
+                yield this.store.finalize(r.id, { status, durationMs, sizeBytes, failReason });
+            }
+            catch (err) {
+                recLog.warn(`finalize failed for ${r.id}: ${err === null || err === void 0 ? void 0 : err.message}`);
+            }
+            this.gate.release(r.id);
+            yield this.releaseLockIfNotInheritedFromMosaic(r.device_udid, (_c = r.device_host) !== null && _c !== void 0 ? _c : '127.0.0.1');
+            if (durationMs !== undefined) {
+                durationHistogram.record(durationMs, {
+                    outcome: status === 'STOPPED' ? attributes_1.OUTCOME.SUCCESS : attributes_1.OUTCOME.FAILURE,
+                });
+            }
+            if (status === 'FAILED') {
+                failuresCounter.add(1, { fail_reason: failReason !== null && failReason !== void 0 ? failReason : 'ffmpeg_stop_failed' });
+            }
+            // Fix E: warm the annotated-mp4 cache off the critical path so a later
+            // Download is served from cache instead of an on-demand burn-in. Only
+            // for finalized recordings that actually carry annotations.
+            if (status === 'STOPPED' && ((_e = (_d = r.annotations) === null || _d === void 0 ? void 0 : _d.length) !== null && _e !== void 0 ? _e : 0) > 0) {
+                this.prewarmAnnotatedRender(r.id);
+            }
+            return { id: r.id, udid: r.device_udid, status, durationMs, sizeBytes };
+        });
+    }
+    /**
+     * The ffmpeg for one recording exited without us asking it to.
+     *
+     * The usual cause is the source ending: the device stopped answering, so the
+     * MJPEG server disconnected its clients (see AndroidStreamService and issue
+     * #200) and ffmpeg saw end-of-input. The mp4 is valid, just shorter than
+     * requested. Nothing else reconciles the row, so without this it sits at
+     * RECORDING — no ended_at, no duration, no size — until a manual Stop or
+     * until recoverOnBoot marks it FAILED on the next server restart, which is
+     * both late and wrong for a recording that ended cleanly. See issue #203.
+     */
+    handleSourceEnded(recordingId, code) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            if (this.finalizing.has(recordingId))
+                return; // a stop is already on it
+            this.finalizing.add(recordingId);
+            try {
+                const r = yield this.store.findById(recordingId);
+                if (!r || r.status !== 'RECORDING')
+                    return;
+                recLog.warn(`Recording ${recordingId} for ${r.device_udid}: ffmpeg exited on its own ` +
+                    `(${code === null ? 'signal' : `code ${code}`}); finalizing it here rather than ` +
+                    'leaving the row open until someone presses Stop.');
+                const one = yield this.finalizeRecording(r, { endedBySource: true });
+                this.eventMgr.emitRecordingStopped({ groupId: r.group_id, recordings: [one] });
+            }
+            catch (err) {
+                recLog.warn(`source-ended finalize failed for ${recordingId}: ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+            }
+            finally {
+                this.finalizing.delete(recordingId);
+            }
+        });
+    }
     /**
      * Stop all recordings in a group. Each recording is finalized independently
      * (one ffmpeg failure does not abort the others), the gate slot is released,
@@ -348,7 +478,7 @@ let RecordingOrchestrator = class RecordingOrchestrator {
      */
     stop(groupId) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c, _d, _e, _f;
+            var _a, _b;
             ensureOtelInstruments();
             const span = api_1.trace.getTracer('xenon.recording').startSpan('xenon.recording.stop', {
                 attributes: { [attributes_1.ATTR.RECORDING_GROUP_ID]: groupId },
@@ -367,52 +497,14 @@ let RecordingOrchestrator = class RecordingOrchestrator {
                 const recordings = yield this.store.listGroup(groupId);
                 const out = [];
                 for (const r of recordings) {
-                    let status = 'STOPPED';
-                    let failReason;
-                    try {
-                        yield this.videoPipeline.stopRecording(r.id);
-                    }
-                    catch (err) {
-                        status = 'FAILED';
-                        failReason = (_b = err === null || err === void 0 ? void 0 : err.message) !== null && _b !== void 0 ? _b : 'ffmpeg_stop_failed';
-                    }
-                    const durationMs = r.started_at ? Date.now() - new Date(r.started_at).getTime() : undefined;
-                    let sizeBytes;
-                    try {
-                        sizeBytes = fs.statSync(r.file_path).size;
-                    }
-                    catch (_g) {
-                        // File may not exist on FAIL; leave size undefined.
-                    }
-                    if (status === 'STOPPED' &&
-                        (sizeBytes === undefined || sizeBytes < MIN_PLAYABLE_MP4_BYTES)) {
-                        status = 'FAILED';
-                        failReason = 'empty_or_corrupt_mp4';
-                        recLog.warn(`Recording ${r.id} for ${r.device_udid} produced unusable file ` +
-                            `(size=${sizeBytes !== null && sizeBytes !== void 0 ? sizeBytes : 'missing'} bytes)`);
-                    }
+                    // Claim it so an ffmpeg exit landing mid-stop doesn't finalize the same
+                    // row from handleSourceEnded at the same time.
+                    this.finalizing.add(r.id);
                     try {
-                        yield this.store.finalize(r.id, { status, durationMs, sizeBytes, failReason });
-                    }
-                    catch (err) {
-                        recLog.warn(`finalize failed for ${r.id}: ${err === null || err === void 0 ? void 0 : err.message}`);
-                    }
-                    this.gate.release(r.id);
-                    yield this.releaseLockIfNotInheritedFromMosaic(r.device_udid, (_c = r.device_host) !== null && _c !== void 0 ? _c : '127.0.0.1');
-                    if (durationMs !== undefined) {
-                        durationHistogram.record(durationMs, {
-                            outcome: status === 'STOPPED' ? attributes_1.OUTCOME.SUCCESS : attributes_1.OUTCOME.FAILURE,
-                        });
-                    }
-                    if (status === 'FAILED') {
-                        failuresCounter.add(1, { fail_reason: failReason !== null && failReason !== void 0 ? failReason : 'ffmpeg_stop_failed' });
+                        out.push(yield this.finalizeRecording(r));
                     }
-                    out.push({ id: r.id, udid: r.device_udid, status, durationMs, sizeBytes });
-                    // Fix E: warm the annotated-mp4 cache off the critical path so a later
-                    // Download is served from cache instead of an on-demand burn-in. Only
-                    // for finalized recordings that actually carry annotations.
-                    if (status === 'STOPPED' && ((_e = (_d = r.annotations) === null || _d === void 0 ? void 0 : _d.length) !== null && _e !== void 0 ? _e : 0) > 0) {
-                        this.prewarmAnnotatedRender(r.id);
+                    finally {
+                        this.finalizing.delete(r.id);
                     }
                 }
                 this.eventMgr.emitRecordingStopped({ groupId, recordings: out });
@@ -421,7 +513,7 @@ let RecordingOrchestrator = class RecordingOrchestrator {
             }
             catch (err) {
                 span.recordException(err);
-                span.setStatus({ code: api_1.SpanStatusCode.ERROR, message: (_f = err === null || err === void 0 ? void 0 : err.message) !== null && _f !== void 0 ? _f : 'unknown' });
+                span.setStatus({ code: api_1.SpanStatusCode.ERROR, message: (_b = err === null || err === void 0 ? void 0 : err.message) !== null && _b !== void 0 ? _b : 'unknown' });
                 throw err;
             }
             finally {

package/lib/src/services/recording/annotation-render.js

@@ -61,6 +61,7 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const recording_store_1 = require("./recording-store");
 const ffmpegPath_1 = require("../../helpers/ffmpegPath");
+const probeDuration_1 = require("./probeDuration");
 const logger_1 = __importDefault(require("../../logger"));
 const renderLog = logger_1.default.scope('AnnotationRender');
 // All ffmpeg spawns go through the bundled binary (resolveFfmpegPath) — a bare
@@ -332,63 +333,14 @@ let AnnotationRenderService = class AnnotationRenderService {
         return Math.max(0, durationSec - LATE_ANNOTATION_MARGIN_SEC);
     }
     /**
-     * Probe a video's duration (seconds) using the bundled ffmpeg (no ffprobe
-     * dependency): read the container header first, then fall back to a decode
-     * pass for fragmented mp4s (e.g. a failed remux) whose header carries no
-     * duration. Undefined on any failure — the clamp then degrades to unclamped.
+     * Probe a video's duration (seconds). Lives in probeDuration.ts now — the
+     * recording-stop path needs the same number to persist a real duration
+     * instead of wall-clock (issue #204), and two copies of a two-stage ffmpeg
+     * probe would drift.
      */
     probeDurationSec(filePath) {
         return __awaiter(this, void 0, void 0, function* () {
-            const fromHeader = yield this.ffmpegHeaderDurationSec(filePath);
-            if (fromHeader !== undefined)
-                return fromHeader;
-            return this.ffmpegDecodeDurationSec(filePath);
-        });
-    }
-    /** Duration from ffmpeg's "Duration: HH:MM:SS.ss" header line (fast path). */
-    ffmpegHeaderDurationSec(filePath) {
-        return new Promise((resolve) => {
-            var _a;
-            try {
-                // `ffmpeg -i <file>` with no output prints stream info then exits
-                // non-zero; we only want the Duration line on stderr.
-                const p = this.spawnFfmpeg(['-hide_banner', '-i', filePath], `probe:${path.basename(filePath)}`);
-                let stderr = '';
-                (_a = p.stderr) === null || _a === void 0 ? void 0 : _a.on('data', (d) => (stderr += d.toString()));
-                p.on('error', () => resolve(undefined));
-                p.on('close', () => {
-                    const m = stderr.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/);
-                    if (!m)
-                        return resolve(undefined);
-                    const sec = Number(m[1]) * 3600 + Number(m[2]) * 60 + parseFloat(m[3]);
-                    resolve(Number.isFinite(sec) && sec > 0 ? sec : undefined);
-                });
-            }
-            catch (_b) {
-                resolve(undefined);
-            }
-        });
-    }
-    /** Duration by decoding to null — reliable for fragmented mp4 with no header. */
-    ffmpegDecodeDurationSec(filePath) {
-        return new Promise((resolve) => {
-            var _a;
-            try {
-                const p = this.spawnFfmpeg(['-hide_banner', '-nostats', '-i', filePath, '-f', 'null', '-progress', 'pipe:1', '-'], `probe-decode:${path.basename(filePath)}`);
-                let out = '';
-                (_a = p.stdout) === null || _a === void 0 ? void 0 : _a.on('data', (d) => (out += d.toString()));
-                p.on('error', () => resolve(undefined));
-                p.on('close', () => {
-                    const matches = [...out.matchAll(/out_time_us=(\d+)/g)];
-                    if (!matches.length)
-                        return resolve(undefined);
-                    const sec = Number(matches[matches.length - 1][1]) / 1e6;
-                    resolve(Number.isFinite(sec) && sec > 0 ? sec : undefined);
-                });
-            }
-            catch (_b) {
-                resolve(undefined);
-            }
+            return (0, probeDuration_1.probeVideoDurationSec)(filePath);
         });
     }
     sanitizeColor(c) {