@xenon-device-management/xenon 1.11.0 → 1.11.2

package/lib/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@xenon-device-management/xenon",
-    "version": "1.11.0",
+    "version": "1.11.2",
     "description": "Xenon - Intelligent Mobile Infrastructure. A self-healing device orchestration platform for Appium.",
     "main": "./lib/src/index.js",
     "exports": {

package/lib/src/app/routers/control.js

@@ -832,20 +832,18 @@ router.get('/:udid/stream', (req, res) => __awaiter(void 0, void 0, void 0, func
         }
     }
     else {
-        // Android auto-start
-        const androidStreamService = typedi_1.Container.get(AndroidStreamService_1.default);
-        const session = androidStreamService.getStreamStatus(udid);
-        if (session && session.status === 'running') {
-            mjpegPort = session.mjpegPort;
+        // Android auto-start. Always go through startStream: it health-checks an
+        // existing session before reusing it (decideAndroidStreamReuse) and dedupes
+        // concurrent starts. The short-circuit that used to live here — hand back
+        // any session marked 'running' — bypassed both, re-introducing the very
+        // stale-port bug the service now guards against. Reuse stays just as cheap;
+        // the health check is an in-process property read, not a request.
+        try {
+            const result = yield typedi_1.Container.get(AndroidStreamService_1.default).startStream(udid);
+            mjpegPort = result.mjpegPort;
         }
-        else {
-            try {
-                const result = yield androidStreamService.startStream(udid);
-                mjpegPort = result.mjpegPort;
-            }
-            catch (err) {
-                return res.status(503).send({ error: 'Android stream failed', message: err.message });
-            }
+        catch (err) {
+            return res.status(503).send({ error: 'Android stream failed', message: err.message });
         }
     }
     if (!mjpegPort) {

package/lib/src/device-managers/android/AndroidStreamService.js

@@ -66,6 +66,7 @@ const ResourceIsolationService_1 = require("../../services/ResourceIsolationServ
 const PortAllocator_1 = require("../../services/PortAllocator");
 const ffmpegPath_1 = require("../../helpers/ffmpegPath");
 const singleFlight_1 = require("../../helpers/singleFlight");
+const androidStreamReuse_1 = require("./androidStreamReuse");
 // JPEG quality for the in-process sharp encoder (0-100). ~78 approximates the
 // old ffmpeg `-q:v 8` (mjpeg quantizer scale) — the low-lag/quality knob.
 const JPEG_QUALITY = 78;
@@ -127,6 +128,29 @@ let AndroidStreamService = class AndroidStreamService {
     shouldIdleCapture(session) {
         return session.status === 'running' && session.viewerCount === 0;
     }
+    /**
+     * Log a stall once per episode, whichever path notices it first.
+     *
+     * Both the capture loop's catch and the MJPEG writer call this. The writer
+     * matters most: ending responses on a stall drops the viewer count to zero,
+     * which makes `shouldIdleCapture` true, which stops the loop attempting
+     * further captures — so the catch may never run again. A live cable-pull run
+     * hit exactly that and logged nothing at all.
+     *
+     * Returns whether this call did the logging, so the caller can decide what to
+     * say instead.
+     */
+    announceStallOnce(session, now, lastError) {
+        if (!session.captureHealth.takeStallAnnouncement(now))
+            return false;
+        const seconds = Math.round(session.captureHealth.failingForMs(now) / 1000);
+        logger_1.default.warn(`[${session.udid}] Device has not answered screencap for ${seconds}s ` +
+            `(${session.captureHealth.consecutiveFailures} consecutive failures` +
+            `${lastError ? `, last: ${lastError}` : ''}). Stream clients are being disconnected ` +
+            'rather than served a frozen frame; the stream restarts on the next request once the ' +
+            'device recovers.');
+        return true;
+    }
     captureLoop(udid, session) {
         return __awaiter(this, void 0, void 0, function* () {
             logger_1.default.info(`[${udid}] Background capture loop started.`);
@@ -142,8 +166,11 @@ let AndroidStreamService = class AndroidStreamService {
                     yield new Promise((r) => setTimeout(r, 150));
                     continue;
                 }
+                // Hoisted out of the try so the catch can date a failure from when the
+                // attempt began: a 15s ADB timeout means the device had already been
+                // silent for 15s by the time we reach the catch.
+                const startTime = Date.now();
                 try {
-                    const startTime = Date.now();
                     // High-Speed Binary Snapshot
                     const screenshot = yield DeviceLockManager_1.deviceLock.acquire(udid, () => __awaiter(this, void 0, void 0, function* () {
                         return yield new Promise((resolve, reject) => {
@@ -196,6 +223,7 @@ let AndroidStreamService = class AndroidStreamService {
                                 logger_1.default.info(`[${udid}] First Android frame successfully captured and converted.`);
                             }
                             session.latestFrameTimestamp = Date.now();
+                            session.captureHealth.recordSuccess();
                         }
                     }
                     else {
@@ -206,7 +234,11 @@ let AndroidStreamService = class AndroidStreamService {
                     yield new Promise((r) => setTimeout(r, delay));
                 }
                 catch (e) {
-                    logger_1.default.debug(`[${udid}] Capture failure: ${e.message}`);
+                    const now = Date.now();
+                    const { consecutive } = session.captureHealth.recordFailure(startTime, now);
+                    if (!this.announceStallOnce(session, now, e.message)) {
+                        logger_1.default.debug(`[${udid}] Capture failure (${consecutive} consecutive): ${e.message}`);
+                    }
                     yield new Promise((r) => setTimeout(r, 1000));
                 }
             }
@@ -216,11 +248,27 @@ let AndroidStreamService = class AndroidStreamService {
     startStream(udid) {
         return __awaiter(this, void 0, void 0, function* () {
             const performStartup = () => __awaiter(this, void 0, void 0, function* () {
-                var _a, _b;
+                var _a, _b, _c;
                 try {
+                    // A session is handed back only when it is actually serving. A closed
+                    // server, or one that went live without ever capturing a frame, was
+                    // otherwise reused indefinitely and every consumer — recording included
+                    // — got a port that yields nothing. See androidStreamReuse / issue #196.
                     const existing = this.sessions.get(udid);
-                    if (existing && (existing.status === 'running' || existing.status === 'starting')) {
-                        return { mjpegPort: existing.mjpegPort };
+                    const decision = (0, androidStreamReuse_1.decideAndroidStreamReuse)(existing && {
+                        status: existing.status,
+                        serverListening: ((_a = existing.server) === null || _a === void 0 ? void 0 : _a.listening) === true,
+                        hasFrame: existing.latestFrame !== undefined,
+                        captureStalled: existing.captureHealth.isStalled(Date.now()),
+                    });
+                    if (decision.reuse) {
+                        if (existing)
+                            return { mjpegPort: existing.mjpegPort };
+                    }
+                    else if (existing) {
+                        logger_1.default.warn(`[${udid}] Stream session on port ${existing.mjpegPort} is not usable ` +
+                            `(${decision.reason}); discarding it and starting fresh.`);
+                        yield this.disposeSession(udid, existing);
                     }
                     const device = yield device_store_1.DeviceStoreFactory.getStore().findDevice({ udid });
                     if (!device)
@@ -247,6 +295,7 @@ let AndroidStreamService = class AndroidStreamService {
                             viewerCount: 0,
                             adbPath,
                             adbHostArgs,
+                            captureHealth: new androidStreamReuse_1.CaptureHealth(),
                         };
                         this.sessions.set(udid, candidate);
                         try {
@@ -256,12 +305,12 @@ let AndroidStreamService = class AndroidStreamService {
                             break;
                         }
                         catch (bindErr) {
-                            logger_1.default.warn(`[${udid}] MJPEG bind failed on ${mjpegPort} (attempt ${attempt + 1}): ${(_a = bindErr === null || bindErr === void 0 ? void 0 : bindErr.message) !== null && _a !== void 0 ? _a : bindErr}`);
+                            logger_1.default.warn(`[${udid}] MJPEG bind failed on ${mjpegPort} (attempt ${attempt + 1}): ${(_b = bindErr === null || bindErr === void 0 ? void 0 : bindErr.message) !== null && _b !== void 0 ? _b : bindErr}`);
                             this.sessions.delete(udid);
                             try {
-                                (_b = candidate.server) === null || _b === void 0 ? void 0 : _b.close();
+                                (_c = candidate.server) === null || _c === void 0 ? void 0 : _c.close();
                             }
-                            catch (_c) {
+                            catch (_d) {
                                 /* ignore */
                             }
                             yield portAllocator.release(mjpegPort).catch(() => undefined);
@@ -318,6 +367,37 @@ let AndroidStreamService = class AndroidStreamService {
             return this.startFlight.run(udid, performStartup);
         });
     }
+    /**
+     * Release a dead session's resources so a fresh one can replace it.
+     *
+     * Deliberately *not* stopStream(): that also unblocks the device when the
+     * lock is a manual one, which would drop the caller's own hold on the device
+     * halfway through a restart. Only stream-owned resources are touched here.
+     */
+    disposeSession(udid, session) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            // The capture loop and every writeFrame loop poll this, so flipping it
+            // first stops them before the server goes away.
+            session.status = 'stopped';
+            try {
+                (_a = session.server) === null || _a === void 0 ? void 0 : _a.close();
+            }
+            catch (_b) {
+                /* ignore */
+            }
+            session.server = null;
+            // Only drop our own entry — never evict a newer session for this udid.
+            if (this.sessions.get(udid) === session)
+                this.sessions.delete(udid);
+            try {
+                yield typedi_1.Container.get(PortAllocator_1.PortAllocator).release(session.mjpegPort);
+            }
+            catch (e) {
+                logger_1.default.warn(`[${udid}] Failed to release mjpeg port lease while restarting stream: ${e}`);
+            }
+        });
+    }
     stopStream(udid) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
@@ -386,6 +466,26 @@ let AndroidStreamService = class AndroidStreamService {
                         !res.writable) {
                         return;
                     }
+                    // The device has gone silent, so `latestFrame` is now a still image.
+                    // End the response instead of rewriting it forever: ffmpeg finalises
+                    // the mp4 with the real footage it captured, and browser clients fall
+                    // into their normal stream-retry — which re-enters startStream, whose
+                    // health check restarts the stream. Serving on would have produced a
+                    // recording that looks valid and is a photograph (issue #200).
+                    const nowMs = Date.now();
+                    if (session.captureHealth.isStalled(nowMs)) {
+                        // Announce from here too: this path is often the only one that gets
+                        // to notice, because the disconnect below is what idles the capture
+                        // loop in the first place.
+                        this.announceStallOnce(session, nowMs);
+                        try {
+                            res.end();
+                        }
+                        catch (_a) {
+                            /* ignore */
+                        }
+                        return;
+                    }
                     if (session.latestFrame) {
                         try {
                             res.write('--BoundaryString\r\n');
@@ -394,7 +494,7 @@ let AndroidStreamService = class AndroidStreamService {
                             res.write(session.latestFrame);
                             res.write('\r\n');
                         }
-                        catch (_a) {
+                        catch (_b) {
                             return;
                         }
                     }

package/lib/src/device-managers/android/androidStreamReuse.js

@@ -0,0 +1,128 @@
+"use strict";
+/**
+ * Decide whether an existing Android MJPEG stream session can be handed back as
+ * is, or has to be torn down and restarted.
+ *
+ * Why this exists: `startStream` used to return `existing.mjpegPort` for any
+ * session marked 'running' *or* 'starting', with nothing verifying that a
+ * server was still bound to it. A session whose HTTP server had closed, or one
+ * that went live without ever capturing a frame (startStream warns and
+ * continues after a 5s first-frame wait), was therefore reused indefinitely —
+ * every caller, `ensureMjpegForRecording` included, got a port that serves
+ * nothing, and ffmpeg left a 0-byte mp4 behind. Same silent symptom as #194,
+ * reached by a different route. See issue #196.
+ *
+ * This is the Android analogue of `resolveIosMjpegPort` (#191), minus the async
+ * probe: the MJPEG server is ours and in-process, so its liveness is a property
+ * read rather than a loopback request.
+ *
+ * Kept free of TypeDI and `http` so the decision is unit-testable; the service
+ * maps its live session onto the view below.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CaptureHealth = exports.CAPTURE_STALL_MS = void 0;
+exports.decideAndroidStreamReuse = decideAndroidStreamReuse;
+function decideAndroidStreamReuse(session) {
+    if (!session)
+        return { reuse: false, reason: 'no-session' };
+    // 'starting' is deliberately excluded: the bind loop registers the candidate
+    // before awaiting the listen, so that status means "a port exists" and not
+    // "a port serves". Concurrent callers join the in-flight start via
+    // SingleFlight (#195) instead of being handed a half-bound port here.
+    if (session.status !== 'running')
+        return { reuse: false, reason: 'not-running' };
+    if (!session.serverListening)
+        return { reuse: false, reason: 'server-closed' };
+    if (!session.hasFrame)
+        return { reuse: false, reason: 'no-frame' };
+    // Ordered after hasFrame: "never captured anything" and "captured, then went
+    // silent" are different problems, and the reason ends up in a log line.
+    if (session.captureStalled)
+        return { reuse: false, reason: 'capture-stalled' };
+    return { reuse: true };
+}
+/**
+ * How long the device may go without producing a frame before anything we still
+ * hold is treated as a still image rather than a live feed.
+ *
+ * Deliberately generous. A single failed `screencap`, a device briefly held by
+ * the interaction lock, or one 15s ADB timeout must not tear down a live
+ * preview or an in-flight recording — the cure for that would be the
+ * over-eager-restart failure mode #198 was careful to avoid.
+ */
+exports.CAPTURE_STALL_MS = 10000;
+/**
+ * Tracks whether the device is still answering `screencap`.
+ *
+ * Why this exists (issue #200): the capture loop swallows capture errors and
+ * loops on `while (status === 'running' || status === 'starting')`, so a device
+ * that goes away — unplugged, adb killed, reboot — never changes the session's
+ * status. `latestFrame` is only ever assigned, never cleared, so the MJPEG
+ * server happily rewrote the last good JPEG every 60ms: a frozen preview, and a
+ * recording that ffprobe calls healthy but which is a still photograph. That is
+ * worse than the 0-byte failure of #194/#196, because it looks valid.
+ *
+ * The measure is time-since-the-device-last-answered, not a failure count: a
+ * fast `ADB Exit 1` and a 15s `ADB Timeout` are wildly different amounts of
+ * frozen video for the same count. It is also immune to the idle path —
+ * `shouldIdleCapture` skips capture entirely when nobody is watching, so an
+ * idle-but-healthy session records neither successes nor failures and simply
+ * stays in whatever state it was left in.
+ */
+class CaptureHealth {
+    constructor() {
+        this.consecutive = 0;
+        this.announced = false;
+    }
+    /** A frame was captured — the device is answering. */
+    recordSuccess() {
+        this.failingSince = undefined;
+        this.consecutive = 0;
+        this.announced = false;
+    }
+    /**
+     * A capture attempt threw.
+     *
+     * `attemptStartedAt` (not `now`) dates the stall, because a 15s ADB timeout
+     * means the device had already been silent for 15s by the time we got here.
+     */
+    recordFailure(attemptStartedAt, now) {
+        if (this.failingSince === undefined)
+            this.failingSince = attemptStartedAt;
+        this.consecutive += 1;
+        return { consecutive: this.consecutive, stalled: this.isStalled(now) };
+    }
+    /**
+     * Claim the right to announce this stall episode: true for the first caller
+     * that observes the session stalled, false for every caller after it until a
+     * successful frame resets the episode.
+     *
+     * Why a claim rather than a flag computed inside `recordFailure`: ending
+     * client responses on a stall drops `viewerCount` to 0, `shouldIdleCapture`
+     * then stops the capture loop attempting anything more, and so the failure
+     * that would have crossed the threshold never happens — the catch never gets
+     * to warn. A live cable-pull run showed exactly that: 10 failures spanning
+     * 9.214s against a 10s threshold, then silence, so a device that had vanished
+     * produced no warning at all. Either path can claim it now, and neither
+     * double-logs.
+     */
+    takeStallAnnouncement(now) {
+        if (!this.isStalled(now) || this.announced)
+            return false;
+        this.announced = true;
+        return true;
+    }
+    /** Consecutive failed capture attempts in the current episode, for logging. */
+    get consecutiveFailures() {
+        return this.consecutive;
+    }
+    /** Capture has been failing long enough that any cached frame is a still. */
+    isStalled(now) {
+        return this.failingSince !== undefined && now - this.failingSince >= exports.CAPTURE_STALL_MS;
+    }
+    /** How long the device has been silent, for logging. 0 when healthy. */
+    failingForMs(now) {
+        return this.failingSince === undefined ? 0 : now - this.failingSince;
+    }
+}
+exports.CaptureHealth = CaptureHealth;

package/lib/src/services/VideoPipelineService.js

@@ -380,26 +380,60 @@ let VideoPipelineService = class VideoPipelineService {
                     logger_1.default.error(`[VideoPipeline] FFMPEG Error [${sessionId}]: ${msg}`);
                 }
             });
-            ffmpegProc.on('exit', (code) => {
-                if (code !== 0 && code !== null) {
-                    logger_1.default.warn(`[VideoPipeline] FFMPEG for ${sessionId} exited with code ${code}`);
-                }
-                this.activeRecordings.delete(sessionId);
-            });
+            ffmpegProc.on('exit', (code) => this.handleProcessExit(sessionId, code, options.onExit));
             this.activeRecordings.set(sessionId, ffmpegProc);
             this.recordingPaths.set(sessionId, outputPath);
         });
     }
+    /**
+     * A recording's ffmpeg exited. Reports it to `onExit` only when nobody asked
+     * for it.
+     *
+     * `stopRecording` drops the handle *before* awaiting the exit, so a still
+     * present entry is the signal that this exit was not requested — the source
+     * ended or ffmpeg died. Getting that backwards would run the caller's
+     * source-ended path on every ordinary Stop as well.
+     *
+     * Split out from the listener so the decision is testable without spawning a
+     * process.
+     */
+    handleProcessExit(sessionId, code, onExit) {
+        var _a;
+        if (code !== 0 && code !== null) {
+            logger_1.default.warn(`[VideoPipeline] FFMPEG for ${sessionId} exited with code ${code}`);
+        }
+        const unrequested = this.activeRecordings.delete(sessionId);
+        if (!unrequested || !onExit)
+            return;
+        logger_1.default.info(`[VideoPipeline] FFMPEG for ${sessionId} exited on its own (${code === null ? 'signal' : `code ${code}`})`);
+        try {
+            onExit(code);
+        }
+        catch (err) {
+            logger_1.default.warn(`[VideoPipeline] onExit handler threw for ${sessionId}: ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+        }
+    }
     /**
      * Stop recording and return the relative asset path
      */
     stopRecording(sessionId) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a;
+            var _a, _b;
             const proc = this.activeRecordings.get(sessionId);
             const recordedPath = this.recordingPaths.get(sessionId);
             if (!proc) {
                 logger_1.default.info(`[VideoPipeline] No active recording process for ${sessionId}, returning stored path if any.`);
+                // ffmpeg already exited on its own. The file still needs the faststart
+                // remux the normal stop path performs, or it stays a fragmented mp4 that
+                // some players will not scrub.
+                if (recordedPath) {
+                    try {
+                        yield remuxToStandardMp4(recordedPath);
+                    }
+                    catch (err) {
+                        logger_1.default.warn(`[VideoPipeline] Remux failed for ${sessionId} (leaving fMP4): ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+                    }
+                }
                 const relativePath = recordedPath
                     ? path_1.default.relative(config_1.config.sessionAssetsPath, recordedPath)
                     : null;
@@ -407,15 +441,17 @@ let VideoPipelineService = class VideoPipelineService {
                 return relativePath;
             }
             logger_1.default.info(`[VideoPipeline] Stopping recording for ${sessionId}`);
-            yield waitForFfmpegExit(proc, `recording ${sessionId}`);
+            // Drop the handle before awaiting: the exit that follows is one we asked
+            // for, and the exit listener must not report it as a source-initiated end.
             this.activeRecordings.delete(sessionId);
+            yield waitForFfmpegExit(proc, `recording ${sessionId}`);
             this.recordingPaths.delete(sessionId);
             if (recordedPath) {
                 try {
                     yield remuxToStandardMp4(recordedPath);
                 }
                 catch (err) {
-                    logger_1.default.warn(`[VideoPipeline] Remux failed for ${sessionId} (leaving fMP4): ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+                    logger_1.default.warn(`[VideoPipeline] Remux failed for ${sessionId} (leaving fMP4): ${(_b = err === null || err === void 0 ? void 0 : err.message) !== null && _b !== void 0 ? _b : err}`);
                 }
             }
             return recordedPath ? path_1.default.relative(config_1.config.sessionAssetsPath, recordedPath) : null;