@xenon-device-management/xenon 1.11.1 → 1.12.0

package/lib/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@xenon-device-management/xenon",
-    "version": "1.11.1",
+    "version": "1.12.0",
     "description": "Xenon - Intelligent Mobile Infrastructure. A self-healing device orchestration platform for Appium.",
     "main": "./lib/src/index.js",
     "exports": {

package/lib/schema.json

@@ -224,6 +224,21 @@
             "default": true,
             "description": "When true, the cleanup job also deletes session video recordings and screenshots from disk (not just DB rows)."
         },
+        "recordingCleanupDays": {
+            "type": "number",
+            "default": 30,
+            "description": "Live Devices recordings older than this many days are purged (DB row + files) by the cleanup job. Runs on `buildCleanupSchedule`."
+        },
+        "recordingCleanupMaxCount": {
+            "type": "number",
+            "default": 100,
+            "description": "Maximum number of recordings to retain. Oldest-first eviction beyond this cap regardless of `recordingCleanupDays`."
+        },
+        "recordingFailedCleanupDays": {
+            "type": "number",
+            "default": 2,
+            "description": "Failed recordings hold no playable file, so they are purged on this much shorter window than `recordingCleanupDays`."
+        },
         "sessionHeartbeatIntervalMs": {
             "type": "number",
             "default": 30000,
@@ -290,6 +305,9 @@
         "buildCleanupMaxCount",
         "buildCleanupSchedule",
         "deleteBuildAssets",
+        "recordingCleanupDays",
+        "recordingCleanupMaxCount",
+        "recordingFailedCleanupDays",
         "sessionHeartbeatIntervalMs",
         "enableJsonLogging"
     ],

package/lib/src/device-managers/android/AndroidStreamService.js

@@ -128,6 +128,29 @@ let AndroidStreamService = class AndroidStreamService {
     shouldIdleCapture(session) {
         return session.status === 'running' && session.viewerCount === 0;
     }
+    /**
+     * Log a stall once per episode, whichever path notices it first.
+     *
+     * Both the capture loop's catch and the MJPEG writer call this. The writer
+     * matters most: ending responses on a stall drops the viewer count to zero,
+     * which makes `shouldIdleCapture` true, which stops the loop attempting
+     * further captures — so the catch may never run again. A live cable-pull run
+     * hit exactly that and logged nothing at all.
+     *
+     * Returns whether this call did the logging, so the caller can decide what to
+     * say instead.
+     */
+    announceStallOnce(session, now, lastError) {
+        if (!session.captureHealth.takeStallAnnouncement(now))
+            return false;
+        const seconds = Math.round(session.captureHealth.failingForMs(now) / 1000);
+        logger_1.default.warn(`[${session.udid}] Device has not answered screencap for ${seconds}s ` +
+            `(${session.captureHealth.consecutiveFailures} consecutive failures` +
+            `${lastError ? `, last: ${lastError}` : ''}). Stream clients are being disconnected ` +
+            'rather than served a frozen frame; the stream restarts on the next request once the ' +
+            'device recovers.');
+        return true;
+    }
     captureLoop(udid, session) {
         return __awaiter(this, void 0, void 0, function* () {
             logger_1.default.info(`[${udid}] Background capture loop started.`);
@@ -143,8 +166,11 @@ let AndroidStreamService = class AndroidStreamService {
                     yield new Promise((r) => setTimeout(r, 150));
                     continue;
                 }
+                // Hoisted out of the try so the catch can date a failure from when the
+                // attempt began: a 15s ADB timeout means the device had already been
+                // silent for 15s by the time we reach the catch.
+                const startTime = Date.now();
                 try {
-                    const startTime = Date.now();
                     // High-Speed Binary Snapshot
                     const screenshot = yield DeviceLockManager_1.deviceLock.acquire(udid, () => __awaiter(this, void 0, void 0, function* () {
                         return yield new Promise((resolve, reject) => {
@@ -197,6 +223,7 @@ let AndroidStreamService = class AndroidStreamService {
                                 logger_1.default.info(`[${udid}] First Android frame successfully captured and converted.`);
                             }
                             session.latestFrameTimestamp = Date.now();
+                            session.captureHealth.recordSuccess();
                         }
                     }
                     else {
@@ -207,7 +234,11 @@ let AndroidStreamService = class AndroidStreamService {
                     yield new Promise((r) => setTimeout(r, delay));
                 }
                 catch (e) {
-                    logger_1.default.debug(`[${udid}] Capture failure: ${e.message}`);
+                    const now = Date.now();
+                    const { consecutive } = session.captureHealth.recordFailure(startTime, now);
+                    if (!this.announceStallOnce(session, now, e.message)) {
+                        logger_1.default.debug(`[${udid}] Capture failure (${consecutive} consecutive): ${e.message}`);
+                    }
                     yield new Promise((r) => setTimeout(r, 1000));
                 }
             }
@@ -228,6 +259,7 @@ let AndroidStreamService = class AndroidStreamService {
                         status: existing.status,
                         serverListening: ((_a = existing.server) === null || _a === void 0 ? void 0 : _a.listening) === true,
                         hasFrame: existing.latestFrame !== undefined,
+                        captureStalled: existing.captureHealth.isStalled(Date.now()),
                     });
                     if (decision.reuse) {
                         if (existing)
@@ -263,6 +295,7 @@ let AndroidStreamService = class AndroidStreamService {
                             viewerCount: 0,
                             adbPath,
                             adbHostArgs,
+                            captureHealth: new androidStreamReuse_1.CaptureHealth(),
                         };
                         this.sessions.set(udid, candidate);
                         try {
@@ -433,6 +466,26 @@ let AndroidStreamService = class AndroidStreamService {
                         !res.writable) {
                         return;
                     }
+                    // The device has gone silent, so `latestFrame` is now a still image.
+                    // End the response instead of rewriting it forever: ffmpeg finalises
+                    // the mp4 with the real footage it captured, and browser clients fall
+                    // into their normal stream-retry — which re-enters startStream, whose
+                    // health check restarts the stream. Serving on would have produced a
+                    // recording that looks valid and is a photograph (issue #200).
+                    const nowMs = Date.now();
+                    if (session.captureHealth.isStalled(nowMs)) {
+                        // Announce from here too: this path is often the only one that gets
+                        // to notice, because the disconnect below is what idles the capture
+                        // loop in the first place.
+                        this.announceStallOnce(session, nowMs);
+                        try {
+                            res.end();
+                        }
+                        catch (_a) {
+                            /* ignore */
+                        }
+                        return;
+                    }
                     if (session.latestFrame) {
                         try {
                             res.write('--BoundaryString\r\n');
@@ -441,7 +494,7 @@ let AndroidStreamService = class AndroidStreamService {
                             res.write(session.latestFrame);
                             res.write('\r\n');
                         }
-                        catch (_a) {
+                        catch (_b) {
                             return;
                         }
                     }

package/lib/src/device-managers/android/androidStreamReuse.js

@@ -20,6 +20,7 @@
  * maps its live session onto the view below.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.CaptureHealth = exports.CAPTURE_STALL_MS = void 0;
 exports.decideAndroidStreamReuse = decideAndroidStreamReuse;
 function decideAndroidStreamReuse(session) {
     if (!session)
@@ -34,5 +35,94 @@ function decideAndroidStreamReuse(session) {
         return { reuse: false, reason: 'server-closed' };
     if (!session.hasFrame)
         return { reuse: false, reason: 'no-frame' };
+    // Ordered after hasFrame: "never captured anything" and "captured, then went
+    // silent" are different problems, and the reason ends up in a log line.
+    if (session.captureStalled)
+        return { reuse: false, reason: 'capture-stalled' };
     return { reuse: true };
 }
+/**
+ * How long the device may go without producing a frame before anything we still
+ * hold is treated as a still image rather than a live feed.
+ *
+ * Deliberately generous. A single failed `screencap`, a device briefly held by
+ * the interaction lock, or one 15s ADB timeout must not tear down a live
+ * preview or an in-flight recording — the cure for that would be the
+ * over-eager-restart failure mode #198 was careful to avoid.
+ */
+exports.CAPTURE_STALL_MS = 10000;
+/**
+ * Tracks whether the device is still answering `screencap`.
+ *
+ * Why this exists (issue #200): the capture loop swallows capture errors and
+ * loops on `while (status === 'running' || status === 'starting')`, so a device
+ * that goes away — unplugged, adb killed, reboot — never changes the session's
+ * status. `latestFrame` is only ever assigned, never cleared, so the MJPEG
+ * server happily rewrote the last good JPEG every 60ms: a frozen preview, and a
+ * recording that ffprobe calls healthy but which is a still photograph. That is
+ * worse than the 0-byte failure of #194/#196, because it looks valid.
+ *
+ * The measure is time-since-the-device-last-answered, not a failure count: a
+ * fast `ADB Exit 1` and a 15s `ADB Timeout` are wildly different amounts of
+ * frozen video for the same count. It is also immune to the idle path —
+ * `shouldIdleCapture` skips capture entirely when nobody is watching, so an
+ * idle-but-healthy session records neither successes nor failures and simply
+ * stays in whatever state it was left in.
+ */
+class CaptureHealth {
+    constructor() {
+        this.consecutive = 0;
+        this.announced = false;
+    }
+    /** A frame was captured — the device is answering. */
+    recordSuccess() {
+        this.failingSince = undefined;
+        this.consecutive = 0;
+        this.announced = false;
+    }
+    /**
+     * A capture attempt threw.
+     *
+     * `attemptStartedAt` (not `now`) dates the stall, because a 15s ADB timeout
+     * means the device had already been silent for 15s by the time we got here.
+     */
+    recordFailure(attemptStartedAt, now) {
+        if (this.failingSince === undefined)
+            this.failingSince = attemptStartedAt;
+        this.consecutive += 1;
+        return { consecutive: this.consecutive, stalled: this.isStalled(now) };
+    }
+    /**
+     * Claim the right to announce this stall episode: true for the first caller
+     * that observes the session stalled, false for every caller after it until a
+     * successful frame resets the episode.
+     *
+     * Why a claim rather than a flag computed inside `recordFailure`: ending
+     * client responses on a stall drops `viewerCount` to 0, `shouldIdleCapture`
+     * then stops the capture loop attempting anything more, and so the failure
+     * that would have crossed the threshold never happens — the catch never gets
+     * to warn. A live cable-pull run showed exactly that: 10 failures spanning
+     * 9.214s against a 10s threshold, then silence, so a device that had vanished
+     * produced no warning at all. Either path can claim it now, and neither
+     * double-logs.
+     */
+    takeStallAnnouncement(now) {
+        if (!this.isStalled(now) || this.announced)
+            return false;
+        this.announced = true;
+        return true;
+    }
+    /** Consecutive failed capture attempts in the current episode, for logging. */
+    get consecutiveFailures() {
+        return this.consecutive;
+    }
+    /** Capture has been failing long enough that any cached frame is a still. */
+    isStalled(now) {
+        return this.failingSince !== undefined && now - this.failingSince >= exports.CAPTURE_STALL_MS;
+    }
+    /** How long the device has been silent, for logging. 0 when healthy. */
+    failingForMs(now) {
+        return this.failingSince === undefined ? 0 : now - this.failingSince;
+    }
+}
+exports.CaptureHealth = CaptureHealth;

package/lib/src/interfaces/IPluginArgs.js

@@ -46,6 +46,9 @@ exports.DefaultPluginArgs = {
     buildCleanupMaxCount: 100,
     buildCleanupSchedule: '0 0 * * *',
     deleteBuildAssets: true,
+    recordingCleanupDays: 30,
+    recordingCleanupMaxCount: 100,
+    recordingFailedCleanupDays: 2,
     sessionHeartbeatIntervalMs: 30000,
     enableJsonLogging: false,
     autowait: {

package/lib/src/services/CleanupService.js

@@ -25,17 +25,22 @@ const logger_1 = __importDefault(require("../logger"));
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const config_1 = require("../config");
+const recordingCleanup_1 = require("./recording/recordingCleanup");
+/** Subdirectory holding composite (multi-device) outputs, keyed by group id. */
+const GROUPS_DIR = '_groups';
 let CleanupService = class CleanupService {
     constructor() {
         this.log = logger_1.default.scope('CleanupService');
     }
     /**
-     * Orchestrates build + session cleanup based on retention policy.
+     * Orchestrates build + session + recording cleanup based on retention policy.
      *
-     * Two phases:
+     * Phases:
      *  1. Purge old builds (by age AND count cap) together with their sessions.
      *  2. Sweep orphan sessions (no build_id) older than the retention window —
      *     otherwise ad-hoc sessions accumulate forever.
+     *  3. Purge expired Live Devices recordings and sweep unreachable recording
+     *     directories (issue #209) — the one asset class nothing used to remove.
      */
     runCleanup(pluginArgs) {
         return __awaiter(this, void 0, void 0, function* () {
@@ -84,6 +89,9 @@ let CleanupService = class CleanupService {
                 //    this, ad-hoc sessions (WebDriver runs with no build capability) are
                 //    never cleaned up — their assets accumulate on disk forever.
                 yield this.purgeOrphanSessions(expirationDate, deleteBuildAssets);
+                // 4. Recordings have their own windows — a failed recording holds no
+                //    playable file, so it need not linger as long as real footage.
+                yield this.purgeRecordings(pluginArgs);
                 this.log.info('✅ Build cleanup completed successfully.');
             }
             catch (err) {
@@ -91,6 +99,119 @@ let CleanupService = class CleanupService {
             }
         });
     }
+    /**
+     * Retention for Live Devices recordings (issue #209).
+     *
+     * Two halves, in order — expiring rows first means their directories are
+     * unreachable by the time the sweep runs, so both are reclaimed in one pass:
+     *  a. delete rows past the age/count policy, and their files;
+     *  b. remove directories no surviving row points into.
+     *
+     * Bookmarks and annotations cascade on the row delete, so no child cleanup
+     * is needed here.
+     */
+    purgeRecordings(pluginArgs) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { recordingCleanupDays = 30, recordingCleanupMaxCount = 100, recordingFailedCleanupDays = 2, } = pluginArgs;
+            const rows = yield prisma_1.prisma.recording.findMany({
+                select: { id: true, status: true, started_at: true, file_path: true, group_id: true },
+            });
+            const expiredIds = (0, recordingCleanup_1.selectExpiredRecordings)({
+                rows,
+                now: Date.now(),
+                days: recordingCleanupDays,
+                maxCount: recordingCleanupMaxCount,
+                failedDays: recordingFailedCleanupDays,
+            });
+            if (expiredIds.length > 0) {
+                const expired = new Set(expiredIds);
+                for (const r of rows) {
+                    if (expired.has(r.id) && r.file_path)
+                        this.unlinkSilently(r.file_path);
+                }
+                yield prisma_1.prisma.recording.deleteMany({ where: { id: { in: expiredIds } } });
+                this.log.info(`Purged ${expiredIds.length} expired recording(s) ` +
+                    `(retention ${recordingCleanupDays}d / max ${recordingCleanupMaxCount}, ` +
+                    `failed ${recordingFailedCleanupDays}d).`);
+            }
+            else {
+                this.log.info('No recordings past their retention window.');
+            }
+            this.sweepOrphanRecordingDirs(rows.filter((r) => !expiredIds.includes(r.id)), rows.length);
+        });
+    }
+    /**
+     * Remove recording directories nothing in the DB can reach.
+     *
+     * Reachability is decided by `file_path`, never by the directory name — see
+     * selectOrphanDirectories for why the name is not a safe key.
+     */
+    sweepOrphanRecordingDirs(survivors, totalRowsRead) {
+        const base = config_1.config.recordingsAssetsPath;
+        let deviceDirs = [];
+        let groupDirs = [];
+        try {
+            if (!fs_1.default.existsSync(base))
+                return;
+            deviceDirs = fs_1.default
+                .readdirSync(base, { withFileTypes: true })
+                .filter((e) => e.isDirectory() && e.name !== GROUPS_DIR)
+                .map((e) => path_1.default.join(base, e.name));
+            const groupsBase = path_1.default.join(base, GROUPS_DIR);
+            if (fs_1.default.existsSync(groupsBase)) {
+                groupDirs = fs_1.default
+                    .readdirSync(groupsBase, { withFileTypes: true })
+                    .filter((e) => e.isDirectory())
+                    .map((e) => path_1.default.join(groupsBase, e.name));
+            }
+        }
+        catch (err) {
+            this.log.warn(`Could not list recording directories: ${err.message}`);
+            return;
+        }
+        if (!(0, recordingCleanup_1.isRecordingSweepSafe)({
+            rowCount: totalRowsRead,
+            dirCount: deviceDirs.length + groupDirs.length,
+        })) {
+            this.log.warn(`Refusing to sweep ${deviceDirs.length + groupDirs.length} recording directories: the ` +
+                'Recording table read returned no rows. That is what a failed or mis-scoped query ' +
+                'looks like, and sweeping on it would delete the whole tree. Sweep manually if the ' +
+                'table really is empty.');
+            return;
+        }
+        const orphans = (0, recordingCleanup_1.selectOrphanDirectories)({
+            deviceDirs,
+            groupDirs,
+            livePaths: survivors.map((r) => r.file_path).filter((p) => !!p),
+            liveGroupIds: survivors.map((r) => r.group_id).filter((g) => !!g),
+        });
+        let removed = 0;
+        for (const dir of orphans) {
+            if (this.removeRecordingDirectory(dir))
+                removed++;
+        }
+        if (removed > 0) {
+            this.log.info(`Swept ${removed} unreachable recording director${removed === 1 ? 'y' : 'ies'}.`);
+        }
+    }
+    /** rm -rf a recording directory, refusing anything outside the tree. */
+    removeRecordingDirectory(dir) {
+        const base = path_1.default.resolve(config_1.config.recordingsAssetsPath);
+        const target = path_1.default.resolve(dir);
+        if (!target.startsWith(base + path_1.default.sep)) {
+            this.log.warn(`Refusing to remove recording dir outside the tree: ${dir}`);
+            return false;
+        }
+        try {
+            fs_1.default.rmSync(target, { recursive: true, force: true });
+            this.log.debug(`Removed orphan recording directory: ${target}`);
+            return true;
+        }
+        catch (err) {
+            this.log.warn(`Failed to remove recording dir ${target}: ${err.message}`);
+            return false;
+        }
+    }
     /**
      * Completely purges a build, its sessions, and all associated metadata/assets.
      */

package/lib/src/services/VideoPipelineService.js

@@ -380,26 +380,60 @@ let VideoPipelineService = class VideoPipelineService {
                     logger_1.default.error(`[VideoPipeline] FFMPEG Error [${sessionId}]: ${msg}`);
                 }
             });
-            ffmpegProc.on('exit', (code) => {
-                if (code !== 0 && code !== null) {
-                    logger_1.default.warn(`[VideoPipeline] FFMPEG for ${sessionId} exited with code ${code}`);
-                }
-                this.activeRecordings.delete(sessionId);
-            });
+            ffmpegProc.on('exit', (code) => this.handleProcessExit(sessionId, code, options.onExit));
             this.activeRecordings.set(sessionId, ffmpegProc);
             this.recordingPaths.set(sessionId, outputPath);
         });
     }
+    /**
+     * A recording's ffmpeg exited. Reports it to `onExit` only when nobody asked
+     * for it.
+     *
+     * `stopRecording` drops the handle *before* awaiting the exit, so a still
+     * present entry is the signal that this exit was not requested — the source
+     * ended or ffmpeg died. Getting that backwards would run the caller's
+     * source-ended path on every ordinary Stop as well.
+     *
+     * Split out from the listener so the decision is testable without spawning a
+     * process.
+     */
+    handleProcessExit(sessionId, code, onExit) {
+        var _a;
+        if (code !== 0 && code !== null) {
+            logger_1.default.warn(`[VideoPipeline] FFMPEG for ${sessionId} exited with code ${code}`);
+        }
+        const unrequested = this.activeRecordings.delete(sessionId);
+        if (!unrequested || !onExit)
+            return;
+        logger_1.default.info(`[VideoPipeline] FFMPEG for ${sessionId} exited on its own (${code === null ? 'signal' : `code ${code}`})`);
+        try {
+            onExit(code);
+        }
+        catch (err) {
+            logger_1.default.warn(`[VideoPipeline] onExit handler threw for ${sessionId}: ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+        }
+    }
     /**
      * Stop recording and return the relative asset path
      */
     stopRecording(sessionId) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a;
+            var _a, _b;
             const proc = this.activeRecordings.get(sessionId);
             const recordedPath = this.recordingPaths.get(sessionId);
             if (!proc) {
                 logger_1.default.info(`[VideoPipeline] No active recording process for ${sessionId}, returning stored path if any.`);
+                // ffmpeg already exited on its own. The file still needs the faststart
+                // remux the normal stop path performs, or it stays a fragmented mp4 that
+                // some players will not scrub.
+                if (recordedPath) {
+                    try {
+                        yield remuxToStandardMp4(recordedPath);
+                    }
+                    catch (err) {
+                        logger_1.default.warn(`[VideoPipeline] Remux failed for ${sessionId} (leaving fMP4): ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+                    }
+                }
                 const relativePath = recordedPath
                     ? path_1.default.relative(config_1.config.sessionAssetsPath, recordedPath)
                     : null;
@@ -407,15 +441,17 @@ let VideoPipelineService = class VideoPipelineService {
                 return relativePath;
             }
             logger_1.default.info(`[VideoPipeline] Stopping recording for ${sessionId}`);
-            yield waitForFfmpegExit(proc, `recording ${sessionId}`);
+            // Drop the handle before awaiting: the exit that follows is one we asked
+            // for, and the exit listener must not report it as a source-initiated end.
             this.activeRecordings.delete(sessionId);
+            yield waitForFfmpegExit(proc, `recording ${sessionId}`);
             this.recordingPaths.delete(sessionId);
             if (recordedPath) {
                 try {
                     yield remuxToStandardMp4(recordedPath);
                 }
                 catch (err) {
-                    logger_1.default.warn(`[VideoPipeline] Remux failed for ${sessionId} (leaving fMP4): ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+                    logger_1.default.warn(`[VideoPipeline] Remux failed for ${sessionId} (leaving fMP4): ${(_b = err === null || err === void 0 ? void 0 : err.message) !== null && _b !== void 0 ? _b : err}`);
                 }
             }
             return recordedPath ? path_1.default.relative(config_1.config.sessionAssetsPath, recordedPath) : null;