@xenon-device-management/xenon 2.13.2 → 2.15.0

package/lib/test/unit/session-device-logs.spec.js

@@ -0,0 +1,347 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const supertest_1 = __importDefault(require("supertest"));
+const stream_1 = require("stream");
+const LogcatStreamService_1 = require("../../src/device-managers/android/LogcatStreamService");
+const PackageResolver_1 = require("../../src/services/logcat/PackageResolver");
+const SessionDeviceLogs_1 = require("../../src/services/logcat/SessionDeviceLogs");
+const deviceLogBook_1 = require("../../src/services/logcat/deviceLogBook");
+const scratch_database_1 = require("../helpers/scratch-database");
+const selector_health_fixture_1 = require("../helpers/selector-health-fixture");
+const pad = (n, w = 2) => String(n).padStart(w, '0');
+// How the Device logs tab reads a line's level for its "Errors only" filter
+// (web log-derive.ts `logLineLevel`, whose own tests hold it to lines in the
+// shape these rows have; the web package can't be imported here).
+const DASHBOARD_THREADTIME = /^\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}\.\d+\s+\d+\s+\d+\s+([VDIWEF])\s/;
+const levelOf = (message) => { var _a, _b; return (_b = (_a = DASHBOARD_THREADTIME.exec(message)) === null || _a === void 0 ? void 0 : _a[1]) !== null && _b !== void 0 ? _b : null; };
+/** A threadtime line logged at `at` by this machine's local clock. */
+function lineAt(at, msg, level = 'I', tag = 'Tag', pid = 4127) {
+    const t = new Date(at);
+    return (`${pad(t.getMonth() + 1)}-${pad(t.getDate())} ${pad(t.getHours())}:${pad(t.getMinutes())}:` +
+        `${pad(t.getSeconds())}.${pad(t.getMilliseconds(), 3)}  ${pid}  ${pid} ${level} ${tag}: ${msg}\n`);
+}
+/** An `adb logcat` child whose output the test writes. */
+function fakeProc() {
+    const stdout = new stream_1.PassThrough();
+    const handlers = {};
+    return {
+        stdout,
+        killed: false,
+        on(ev, cb) {
+            (handlers[ev] || (handlers[ev] = [])).push(cb);
+        },
+        /** The process ends, as when the phone restarts or is unplugged. */
+        exit() {
+            stdout.end();
+            (handlers.close || []).forEach((h) => h());
+        },
+        kill() {
+            this.killed = true;
+        },
+    };
+}
+/** The real stream service, its `adb logcat` children fake. */
+class FakeLogcatStreams extends LogcatStreamService_1.LogcatStreamService {
+    constructor() {
+        super(...arguments);
+        this.procs = [];
+    }
+    spawnLogcat() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const proc = fakeProc();
+            this.procs.push(proc);
+            return proc;
+        });
+    }
+    makeResolver() {
+        return new PackageResolver_1.PackageResolver(() => __awaiter(this, void 0, void 0, function* () { return '  PID NAME\n 4127 com.example.app\n'; }));
+    }
+    get proc() {
+        return this.procs[this.procs.length - 1];
+    }
+}
+class TestDeviceLogs extends SessionDeviceLogs_1.SessionDeviceLogs {
+    constructor(logcat) {
+        super();
+        this.logcat = logcat;
+        this.clock = deviceLogBook_1.UNKNOWN_CLOCK;
+        /** The size of each write, in order. */
+        this.writes = [];
+    }
+    context() {
+        return { pluginArgs: { bindHostOrIp: '127.0.0.1' }, port: 4723, nodeId: 'this-server' };
+    }
+    streams() {
+        return this.logcat;
+    }
+    readClock() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.clock;
+        });
+    }
+    flushIntervalMs() {
+        return 60000;
+    }
+    retryFirstMs() {
+        return 1;
+    }
+    writeLines(sessionId, lines, first) {
+        const _super = Object.create(null, {
+            writeLines: { get: () => super.writeLines }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            this.writes.push(lines.length);
+            yield _super.writeLines.call(this, sessionId, lines, first);
+        });
+    }
+}
+const PHONE = {
+    udid: 'phone-1',
+    platform: 'android',
+    host: 'http://127.0.0.1:4723',
+    nodeId: 'this-server',
+};
+/** Lets readline, the stream's package lookups and the mux deliver what was written. */
+const settle = () => __awaiter(void 0, void 0, void 0, function* () {
+    for (let i = 0; i < 5; i++)
+        yield new Promise((r) => setImmediate(r));
+});
+const until = (check_1, ...args_1) => __awaiter(void 0, [check_1, ...args_1], void 0, function* (check, ms = 2000) {
+    const end = Date.now() + ms;
+    while (!check()) {
+        if (Date.now() > end)
+            throw new Error('timed out');
+        yield new Promise((r) => setTimeout(r, 5));
+    }
+});
+/** The session's Device logs as the session page gets them. */
+function deviceLogs(sessionId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const res = yield (0, supertest_1.default)((0, selector_health_fixture_1.selectorHealthApp)(selector_health_fixture_1.ADMIN)).get(`/session/${sessionId}/logs/device`);
+        (0, chai_1.expect)(res.status, res.text).to.equal(200);
+        return res.body;
+    });
+}
+const textOf = (rows) => rows.map((r) => r.message.replace(/^.*?Tag {5}: /, ''));
+describe("An Android session's Device logs", function () {
+    this.timeout(60000);
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let logcat;
+    let logs;
+    let sessionId;
+    let count = 0;
+    beforeEach(() => __awaiter(this, void 0, void 0, function* () {
+        count += 1;
+        sessionId = `dl-session-${count}`;
+        yield scratch.db.session.create({
+            data: {
+                id: sessionId,
+                device_udid: PHONE.udid,
+                device_platform: 'android',
+                device_version: '10',
+                desired_capabilities: '{}',
+                session_capabilities: '{}',
+                node_id: 'this-server',
+                has_live_video: false,
+            },
+        });
+        logcat = new FakeLogcatStreams();
+        logs = new TestDeviceLogs(logcat);
+    }));
+    afterEach(() => __awaiter(this, void 0, void 0, function* () {
+        yield logs.stop(sessionId);
+        yield logcat.cleanup();
+    }));
+    it('keeps the lines after the first 100, each once, in order, however many commands ran', () => __awaiter(this, void 0, void 0, function* () {
+        const since = Date.now() - 60000;
+        yield logs.start({ sessionId, device: PHONE, since });
+        // History from before the session, as `logcat -T` sends it first.
+        for (let i = 0; i < 20; i++)
+            logcat.proc.stdout.write(lineAt(since - 600000 + i, `old ${i}`));
+        // 250 lines over the run, a batch between each "command".
+        for (let batch = 0; batch < 5; batch++) {
+            for (let i = 0; i < 50; i++) {
+                const n = batch * 50 + i;
+                logcat.proc.stdout.write(lineAt(since + 1000 + n, `line ${n}`));
+            }
+            yield settle();
+        }
+        yield logs.stop(sessionId);
+        const rows = yield deviceLogs(sessionId);
+        (0, chai_1.expect)(textOf(rows)).to.deep.equal(Array.from({ length: 250 }, (_, n) => `line ${n}`));
+    }));
+    it("keeps each line's level and time, so Errors only finds a crash", () => __awaiter(this, void 0, void 0, function* () {
+        const since = Date.now() - 60000;
+        yield logs.start({ sessionId, device: PHONE, since });
+        const crashAt = since + 30123;
+        logcat.proc.stdout.write(lineAt(crashAt - 5, 'Start proc com.example.app', 'I', 'ActivityManager'));
+        logcat.proc.stdout.write(lineAt(crashAt, 'FATAL EXCEPTION: main', 'E', 'AndroidRuntime'));
+        logcat.proc.stdout.write(lineAt(crashAt, 'Process: com.example.app, PID: 4127', 'E', 'AndroidRuntime'));
+        logcat.proc.stdout.write(lineAt(crashAt + 1, 'Sending signal. PID: 4127 SIG: 9', 'I', 'Process'));
+        yield settle();
+        yield logs.stop(sessionId);
+        const rows = yield deviceLogs(sessionId);
+        (0, chai_1.expect)(rows.map((r) => levelOf(r.message))).to.deep.equal(['I', 'E', 'E', 'I']);
+        (0, chai_1.expect)(rows.map((r) => new Date(r.timestamp).getTime())).to.deep.equal([
+            crashAt - 5,
+            crashAt,
+            crashAt,
+            crashAt + 1,
+        ]);
+        (0, chai_1.expect)(rows[1].message).to.match(/ E AndroidRuntime: FATAL EXCEPTION: main$/);
+    }));
+    it("keeps a stack trace's lines in order, though they share a millisecond and a write", () => __awaiter(this, void 0, void 0, function* () {
+        const since = Date.now() - 60000;
+        yield logs.start({ sessionId, device: PHONE, since });
+        const at = since + 10000;
+        const frames = Array.from({ length: 40 }, (_, i) => `\tat com.example.Frame${i}.run(Frame.java:${i})`);
+        for (const frame of frames)
+            logcat.proc.stdout.write(lineAt(at, frame, 'E', 'AndroidRuntime'));
+        yield settle();
+        yield logs.stop(sessionId);
+        const rows = yield deviceLogs(sessionId);
+        (0, chai_1.expect)(rows.map((r) => r.message.replace(/^.*AndroidRuntime: /, ''))).to.deep.equal(frames);
+        // SQLite returns rows of one `createdAt` in the order they were written,
+        // and Postgres in any order: each row's `createdAt` is its own.
+        const created = rows.map((r) => new Date(r.createdAt).getTime());
+        (0, chai_1.expect)(created.every((t, i) => i === 0 || t > created[i - 1]), String(created)).to.equal(true);
+    }));
+    it('writes as the session runs, in batches, not a row at a time', () => __awaiter(this, void 0, void 0, function* () {
+        const since = Date.now() - 60000;
+        yield logs.start({ sessionId, device: PHONE, since });
+        for (let i = 0; i < SessionDeviceLogs_1.WRITE_BATCH + 10; i++)
+            logcat.proc.stdout.write(lineAt(since + i, `line ${i}`));
+        yield settle();
+        // A full batch is written while the session runs, without waiting for
+        // its end or the interval (a minute here).
+        const written = () => scratch.db.log.count({ where: { session_id: sessionId } });
+        const end = Date.now() + 2000;
+        while ((yield written()) < SessionDeviceLogs_1.WRITE_BATCH && Date.now() < end)
+            yield settle();
+        (0, chai_1.expect)(yield written()).to.be.at.least(SessionDeviceLogs_1.WRITE_BATCH);
+        (0, chai_1.expect)(logs.isRecording(sessionId)).to.equal(true);
+        yield logs.stop(sessionId);
+        (0, chai_1.expect)(yield written()).to.equal(SessionDeviceLogs_1.WRITE_BATCH + 10);
+        (0, chai_1.expect)(logs.writes).to.have.length(2);
+        (0, chai_1.expect)(Math.max(...logs.writes)).to.equal(SessionDeviceLogs_1.WRITE_BATCH);
+    }));
+    it('picks up again after the phone restarts, without saving its history twice', () => __awaiter(this, void 0, void 0, function* () {
+        const since = Date.now() - 60000;
+        yield logs.start({ sessionId, device: PHONE, since });
+        const first = logcat.proc;
+        first.stdout.write(lineAt(since + 1000, 'before 1'));
+        first.stdout.write(lineAt(since + 2000, 'before 2'));
+        yield settle();
+        first.exit();
+        yield until(() => logcat.procs.length === 2);
+        // The new stream starts with history the session already has, then a
+        // line of the same millisecond it hadn't seen, then new lines.
+        logcat.proc.stdout.write(lineAt(since + 1000, 'before 1'));
+        logcat.proc.stdout.write(lineAt(since + 2000, 'before 2'));
+        logcat.proc.stdout.write(lineAt(since + 2000, 'also at 2'));
+        logcat.proc.stdout.write(lineAt(since + 9000, 'after'));
+        yield settle();
+        yield logs.stop(sessionId);
+        (0, chai_1.expect)(textOf(yield deviceLogs(sessionId))).to.deep.equal([
+            'before 1',
+            'before 2',
+            "Xenon: The phone's log was interrupted. Lines may be missing from here.",
+            'also at 2',
+            'after',
+        ]);
+    }));
+    it("shares the phone's stream with the Logs viewer, and never stops it", () => __awaiter(this, void 0, void 0, function* () {
+        const since = Date.now() - 60000;
+        const mux = yield logcat.start(PHONE.udid);
+        const viewer = [];
+        mux.addClient((r) => viewer.push(r.message), () => true);
+        yield logs.start({ sessionId, device: PHONE, since });
+        (0, chai_1.expect)(logcat.procs).to.have.length(1);
+        logcat.proc.stdout.write(lineAt(since + 1, 'shared'));
+        yield settle();
+        yield logs.stop(sessionId);
+        (0, chai_1.expect)(viewer).to.deep.equal(['shared']);
+        (0, chai_1.expect)(logcat.proc.killed).to.equal(false);
+        (0, chai_1.expect)(logcat.getMultiplexer(PHONE.udid)).to.equal(mux);
+        (0, chai_1.expect)(textOf(yield deviceLogs(sessionId))).to.deep.equal(['shared']);
+    }));
+    it('keeps the stream from its idle stop while the session runs, and lets it go after', () => __awaiter(this, void 0, void 0, function* () {
+        yield logs.start({ sessionId, device: PHONE, since: Date.now() });
+        const mux = logcat.getMultiplexer(PHONE.udid);
+        (0, chai_1.expect)(mux.emptySince).to.equal(undefined);
+        yield logs.stop(sessionId);
+        (0, chai_1.expect)(mux.emptySince).to.be.a('number');
+    }));
+    it(`holds to ${deviceLogBook_1.DEVICE_LOG_LINE_LIMIT} lines, then ${deviceLogBook_1.DEVICE_LOG_ERROR_LIMIT} errors`, () => __awaiter(this, void 0, void 0, function* () {
+        const since = Date.now() - 600000;
+        yield logs.start({ sessionId, device: PHONE, since });
+        const total = deviceLogBook_1.DEVICE_LOG_LINE_LIMIT + 3000;
+        let chunk = '';
+        for (let i = 0; i < total; i++) {
+            // Past the limit, five lines in six are errors: 2,500 errors, 500 others.
+            const level = i >= deviceLogBook_1.DEVICE_LOG_LINE_LIMIT && i % 6 !== 0 ? 'E' : 'D';
+            chunk += lineAt(since + 1000 + i, `line ${i}`, level);
+            if (i % 1000 === 999) {
+                logcat.proc.stdout.write(chunk);
+                chunk = '';
+                yield settle();
+            }
+        }
+        yield settle();
+        yield logs.stop(sessionId);
+        const rows = yield deviceLogs(sessionId);
+        (0, chai_1.expect)(rows).to.have.length(deviceLogBook_1.DEVICE_LOG_LINE_LIMIT + 1 + deviceLogBook_1.DEVICE_LOG_ERROR_LIMIT + 1);
+        (0, chai_1.expect)(textOf(rows.slice(0, deviceLogBook_1.DEVICE_LOG_LINE_LIMIT))).to.deep.equal(Array.from({ length: deviceLogBook_1.DEVICE_LOG_LINE_LIMIT }, (_, i) => `line ${i}`));
+        (0, chai_1.expect)(rows[deviceLogBook_1.DEVICE_LOG_LINE_LIMIT].message).to.equal('Xenon: This session reached 10,000 device log lines. From here on, only errors are kept.');
+        const errors = rows.slice(deviceLogBook_1.DEVICE_LOG_LINE_LIMIT + 1, -1);
+        (0, chai_1.expect)(errors.every((r) => levelOf(r.message) === 'E')).to.equal(true);
+        (0, chai_1.expect)(rows[rows.length - 1].message).to.equal("Xenon: 1,000 device log lines were left out to keep this session's log within its limit.");
+    }));
+    it("only records this server's own Android phones", () => {
+        (0, chai_1.expect)(logs.appliesTo(PHONE)).to.equal(true);
+        (0, chai_1.expect)(logs.appliesTo(Object.assign(Object.assign({}, PHONE), { nodeId: 'a-node' }))).to.equal(false);
+        (0, chai_1.expect)(logs.appliesTo(Object.assign(Object.assign({}, PHONE), { platform: 'ios' }))).to.equal(false);
+        (0, chai_1.expect)(logs.appliesTo(Object.assign(Object.assign({}, PHONE), { cloud: true }))).to.equal(false);
+        (0, chai_1.expect)(logs.appliesTo(undefined)).to.equal(false);
+    });
+    it("reads lines as this server's time when the phone's clock can't be read", () => __awaiter(this, void 0, void 0, function* () {
+        logs.clock = null;
+        const since = Date.now() - 60000;
+        yield logs.start({ sessionId, device: PHONE, since });
+        logcat.proc.stdout.write(lineAt(since - 600000, 'before'));
+        logcat.proc.stdout.write(lineAt(since + 10, 'during'));
+        yield settle();
+        yield logs.stop(sessionId);
+        (0, chai_1.expect)(textOf(yield deviceLogs(sessionId))).to.deep.equal(['during']);
+    }));
+    it('stops however often it is told to', () => __awaiter(this, void 0, void 0, function* () {
+        yield logs.start({ sessionId, device: PHONE, since: Date.now() });
+        yield logs.stop(sessionId);
+        yield logs.stop(sessionId);
+        (0, chai_1.expect)(logs.isRecording(sessionId)).to.equal(false);
+    }));
+    it('starts nothing when stopped before the phone answered', () => __awaiter(this, void 0, void 0, function* () {
+        let answer;
+        logs.clock = null;
+        logs.readClock = () => new Promise((r) => (answer = r));
+        const started = logs.start({ sessionId, device: PHONE, since: Date.now() });
+        yield logs.stop(sessionId);
+        answer(deviceLogBook_1.UNKNOWN_CLOCK);
+        yield started;
+        (0, chai_1.expect)(logcat.procs).to.have.length(0);
+    }));
+});

package/lib/test/unit/session-end-failure-analysis.spec.js

@@ -0,0 +1,133 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const event_manager_1 = require("../../src/dashboard/event-manager");
+const SessionManager_1 = require("../../src/sessions/SessionManager");
+const SessionMetricsService_1 = require("../../src/services/metrics/SessionMetricsService");
+const SocketServer_1 = require("../../src/services/SocketServer");
+const MetricsService_1 = require("../../src/services/MetricsService");
+const device_store_1 = require("../../src/data-service/device-store");
+const AIService_1 = require("../../src/services/AIService");
+const failure_analysis_service_1 = require("../../src/dashboard/services/failure-analysis-service");
+const SessionStatus_1 = require("../../src/types/SessionStatus");
+const container_registration_1 = require("../helpers/container-registration");
+const scratch_database_1 = require("../helpers/scratch-database");
+/**
+ * Ending a failed session doesn't wait for the AI to explain the failure.
+ *
+ * `onSessionStopped` awaited the AI call, and every way a session ends awaits
+ * `onSessionStopped`: the client's `driver.quit()`, a hub's forwarded DELETE,
+ * an inactivity timeout, a shutdown. Only Ollama's calls had a time limit; the
+ * OpenAI and Anthropic SDKs wait minutes and retry. So a quit could outlast
+ * the client's own timeout. The category (rules, no AI) is still saved before
+ * the session's end returns.
+ */
+describe('ending a failed session', () => {
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    const ID = 'sess-end-analysis';
+    let restore;
+    let explain;
+    beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
+        restore = (0, container_registration_1.saveRegistrations)(SessionMetricsService_1.SessionMetricsService, SocketServer_1.SocketServer, MetricsService_1.MetricsService);
+        typedi_1.Container.set(SessionMetricsService_1.SessionMetricsService, { stop: sinon_1.default.stub().resolves() });
+        typedi_1.Container.set(SocketServer_1.SocketServer, {
+            emitToDashboard: sinon_1.default.stub(),
+            emitToDashboardForDevices: sinon_1.default.stub().resolves(),
+            hasScopedDashboard: sinon_1.default.stub().returns(false),
+        });
+        typedi_1.Container.set(MetricsService_1.MetricsService, {
+            incrementSessionSuccess: sinon_1.default.stub(),
+            incrementSessionFailure: sinon_1.default.stub(),
+        });
+        sinon_1.default.stub(SessionManager_1.SESSION_MANAGER, 'getSession').returns(undefined);
+        sinon_1.default
+            .stub(device_store_1.DeviceStoreFactory, 'getStore')
+            .returns({ getDevices: () => __awaiter(void 0, void 0, void 0, function* () { return []; }), findDevices: () => __awaiter(void 0, void 0, void 0, function* () { return []; }) });
+        yield scratch.db.log.deleteMany();
+        yield scratch.db.sessionLog.deleteMany();
+        yield scratch.db.session.deleteMany();
+        yield scratch.db.session.create({
+            data: {
+                id: ID,
+                status: 'running',
+                desired_capabilities: '{}',
+                session_capabilities: '{}',
+                node_id: 'node',
+                has_live_video: false,
+                device_udid: 'R58M123',
+                device_platform: 'android',
+                device_version: '10',
+            },
+        });
+        // An AI provider that answers when the test says.
+        let resolve;
+        const promise = new Promise((r) => (resolve = r));
+        explain = { resolve, promise };
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'isEnabled').returns(true);
+        sinon_1.default.stub(AIService_1.AI_SERVICE, 'analyzeFailure').returns(promise);
+    }));
+    afterEach(() => __awaiter(void 0, void 0, void 0, function* () {
+        // The analysis the session's end started finishes before the stubs go,
+        // so it never reaches a real AI provider or database. One still pending
+        // gets no answer, so it writes nothing; asking again for the same
+        // session waits for the one running.
+        explain.resolve(null);
+        yield (0, failure_analysis_service_1.explainSessionFailure)(ID);
+        sinon_1.default.restore();
+        restore();
+    }));
+    const row = () => scratch.db.session.findUniqueOrThrow({ where: { id: ID } });
+    const within = (ms, p) => Promise.race([
+        p.then(() => 'ended'),
+        new Promise((r) => setTimeout(() => r('still waiting for the AI'), ms)),
+    ]);
+    const end = () => event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(ID, SessionStatus_1.SessionStatus.FAILED, 'no such element', {
+        notify: false,
+    });
+    it('returns while the AI is still explaining the failure', () => __awaiter(void 0, void 0, void 0, function* () {
+        (0, chai_1.expect)(yield within(2000, end())).to.equal('ended');
+        // The AI is still asked, once the session has ended.
+        const analyzeFailure = AIService_1.AI_SERVICE.analyzeFailure;
+        for (let i = 0; i < 100 && !analyzeFailure.called; i++) {
+            yield new Promise((r) => setTimeout(r, 20));
+        }
+        (0, chai_1.expect)(analyzeFailure.calledOnce).to.equal(true);
+        (0, chai_1.expect)(analyzeFailure.firstCall.args[0]).to.include({
+            sessionId: ID,
+            failureReason: 'no such element',
+        });
+    }));
+    it('has filed the session under its category when it returns', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield within(2000, end());
+        const saved = yield row();
+        (0, chai_1.expect)(saved.status).to.equal(SessionStatus_1.SessionStatus.FAILED);
+        (0, chai_1.expect)(saved.failure_category).to.equal('ELEMENT_NOT_FOUND');
+        (0, chai_1.expect)(saved.ai_analysis).to.equal(null);
+    }));
+    it("saves the AI's explanation when it comes", () => __awaiter(void 0, void 0, void 0, function* () {
+        yield within(2000, end());
+        explain.resolve('Root Cause: the Login button was covered by a dialog.');
+        let saved = yield row();
+        for (let i = 0; i < 100 && !saved.ai_analysis; i++) {
+            yield new Promise((r) => setTimeout(r, 20));
+            saved = yield row();
+        }
+        (0, chai_1.expect)(saved.ai_analysis).to.equal('Root Cause: the Login button was covered by a dialog.');
+        (0, chai_1.expect)(saved.failure_category).to.equal('ELEMENT_NOT_FOUND');
+    }));
+});

package/lib/test/unit/session-failed-webhook.spec.js

@@ -0,0 +1,170 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const axios_1 = __importDefault(require("axios"));
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const event_manager_1 = require("../../src/dashboard/event-manager");
+const SessionManager_1 = require("../../src/sessions/SessionManager");
+const SessionMetricsService_1 = require("../../src/services/metrics/SessionMetricsService");
+const SocketServer_1 = require("../../src/services/SocketServer");
+const MetricsService_1 = require("../../src/services/MetricsService");
+const NotificationService_1 = require("../../src/services/NotificationService");
+const device_store_1 = require("../../src/data-service/device-store");
+const SessionStatus_1 = require("../../src/types/SessionStatus");
+const prisma_1 = require("../../src/prisma");
+const container_registration_1 = require("../helpers/container-registration");
+/**
+ * `session_failed` goes out when a session ends as failed, whichever way it
+ * ends, and once per session.
+ *
+ * It was sent only from the client's own delete, so a session that timed out
+ * for inactivity, whose driver crashed, or whose heartbeat stopped was marked
+ * failed on the dashboard and counted in the failure metric, and no webhook
+ * heard of it. Every one of those ends through `onSessionStopped`, which is
+ * where the final status is decided, so that is where it is sent. A session
+ * that ends twice (a crash, then the client's delete) is still sent once.
+ */
+describe('session_failed: when it is sent', () => {
+    const URL = 'https://hooks.example.com/T000/B000/xyz';
+    let restore;
+    let post;
+    let row;
+    let svc;
+    beforeEach(() => {
+        restore = (0, container_registration_1.saveRegistrations)(SessionMetricsService_1.SessionMetricsService, SocketServer_1.SocketServer, MetricsService_1.MetricsService);
+        typedi_1.Container.set(SessionMetricsService_1.SessionMetricsService, { stop: sinon_1.default.stub().resolves() });
+        typedi_1.Container.set(SocketServer_1.SocketServer, {
+            emitToDashboard: sinon_1.default.stub(),
+            emitToDashboardForDevices: sinon_1.default.stub().resolves(),
+            hasScopedDashboard: sinon_1.default.stub().returns(false),
+        });
+        typedi_1.Container.set(MetricsService_1.MetricsService, {
+            incrementSessionSuccess: sinon_1.default.stub(),
+            incrementSessionFailure: sinon_1.default.stub(),
+        });
+        sinon_1.default.stub(SessionManager_1.SESSION_MANAGER, 'getSession').returns(undefined);
+        sinon_1.default
+            .stub(device_store_1.DeviceStoreFactory, 'getStore')
+            .returns({ getDevices: () => __awaiter(void 0, void 0, void 0, function* () { return []; }), findDevices: () => __awaiter(void 0, void 0, void 0, function* () { return []; }) });
+        sinon_1.default.stub(prisma_1.prisma.sessionLog, 'findFirst').resolves(null);
+        row = {
+            id: 'sess-1',
+            name: 'Checkout flow',
+            status: 'running',
+            failure_reason: null,
+            device_udid: 'R58M123',
+            device_name: 'Galaxy S9+',
+            device_platform: 'android',
+            device_version: '10',
+            startTime: new Date('2026-10-04T09:00:00.000Z'),
+            endTime: null,
+            desired_capabilities: '{"appium:app":"/secret.apk"}',
+        };
+        sinon_1.default.stub(prisma_1.prisma.session, 'findFirst').callsFake((() => __awaiter(void 0, void 0, void 0, function* () { return (Object.assign({}, row)); })));
+        sinon_1.default.stub(prisma_1.prisma.session, 'update').callsFake(((_a) => __awaiter(void 0, [_a], void 0, function* ({ data }) {
+            row = Object.assign(Object.assign({}, row), data);
+            return row;
+        })));
+        // A real service with a slack webhook for the event, so what is checked is
+        // what would reach the channel.
+        svc = typedi_1.Container.get(NotificationService_1.NotificationService);
+        post = sinon_1.default.stub(axios_1.default, 'post').resolves({ status: 200 });
+        sinon_1.default
+            .stub(svc, 'getConfigs')
+            .resolves([
+            { id: 'w', url: URL, events: '["session_failed"]', type: 'slack', active: true },
+        ]);
+        // A session id nobody has notified for yet, however many specs ran before.
+        row.id = `sess-${Math.random().toString(36).slice(2)}`;
+    });
+    afterEach(() => {
+        sinon_1.default.restore();
+        restore();
+    });
+    /** onSessionStopped fires the webhook without being awaited: let it finish. */
+    const settle = () => new Promise((resolve) => setTimeout(resolve, 30));
+    it('is sent when a session times out for inactivity (device-utils)', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id, 'failed', 'Session timed out due to inactivity');
+        yield settle();
+        (0, chai_1.expect)(post.calledOnce).to.equal(true);
+        const text = post.firstCall.args[1].attachments[0].text;
+        (0, chai_1.expect)(text).to.contain(row.id);
+        (0, chai_1.expect)(text).to.contain('Session timed out due to inactivity');
+        (0, chai_1.expect)(text).to.contain('Galaxy S9+');
+        (0, chai_1.expect)(text).not.to.contain('undefined');
+    }));
+    it('is sent when the driver crashes (plugin onUnexpectedShutdown)', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id, SessionStatus_1.SessionStatus.FAILED, 'Driver shut down unexpectedly');
+        yield settle();
+        (0, chai_1.expect)(post.calledOnce).to.equal(true);
+        (0, chai_1.expect)(post.firstCall.args[1].attachments[0].text).to.contain('Driver shut down unexpectedly');
+    }));
+    it('is sent when the heartbeat stops (OrphanSweeper)', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id, SessionStatus_1.SessionStatus.FAILED, 'Session heartbeat timeout');
+        yield settle();
+        (0, chai_1.expect)(post.calledOnce).to.equal(true);
+    }));
+    it('is sent when the client deletes a session a command failed in', () => __awaiter(void 0, void 0, void 0, function* () {
+        prisma_1.prisma.sessionLog.findFirst.resolves({
+            command_name: 'findElement',
+            response: '{"value":{"error":"no such element"}}',
+        });
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id);
+        yield settle();
+        (0, chai_1.expect)(post.calledOnce).to.equal(true);
+        (0, chai_1.expect)(post.firstCall.args[1].attachments[0].text).to.contain('no such element');
+    }));
+    it('is sent when the test marked the session failed and the client then deletes it', () => __awaiter(void 0, void 0, void 0, function* () {
+        row.status = 'failed';
+        row.failure_reason = 'Assertion failed: total is 12';
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id);
+        yield settle();
+        (0, chai_1.expect)(post.calledOnce).to.equal(true);
+        (0, chai_1.expect)(post.firstCall.args[1].attachments[0].text).to.contain('Assertion failed: total is 12');
+    }));
+    it('is not sent for a session that passes', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id);
+        yield settle();
+        (0, chai_1.expect)(row.status).to.equal('success');
+        (0, chai_1.expect)(post.called).to.equal(false);
+    }));
+    it('is sent once when a session ends twice: a crash, then the client’s delete', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id, SessionStatus_1.SessionStatus.FAILED, 'Driver shut down unexpectedly');
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id);
+        yield settle();
+        (0, chai_1.expect)(post.callCount).to.equal(1);
+    }));
+    it('is not sent when the server itself is shutting down', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id, SessionStatus_1.SessionStatus.FAILED, 'Hub shutdown', {
+            notify: false,
+        });
+        yield settle();
+        (0, chai_1.expect)(row.status).to.equal('failed'); // still recorded as failed
+        (0, chai_1.expect)(post.called).to.equal(false);
+    }));
+    it('does not hold the session’s end up for a webhook, and survives one that fails', () => __awaiter(void 0, void 0, void 0, function* () {
+        post.rejects(new Error('slack is down'));
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id, SessionStatus_1.SessionStatus.FAILED, 'boom');
+        yield settle();
+        (0, chai_1.expect)(row.status).to.equal('failed');
+    }));
+    it('does not send the session’s capabilities to the webhook', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(row.id, SessionStatus_1.SessionStatus.FAILED, 'boom');
+        yield settle();
+        (0, chai_1.expect)(JSON.stringify(post.firstCall.args[1])).not.to.contain('secret.apk');
+    }));
+});