@xenon-device-management/xenon 2.14.0 → 2.15.0

package/lib/test/unit/enable-dashboard-option.spec.js

@@ -0,0 +1,57 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const express_1 = __importDefault(require("express"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const loopbackRequest_1 = __importDefault(require("../helpers/loopbackRequest"));
+const index_1 = require("../../src/app/index");
+const IPluginArgs_1 = require("../../src/interfaces/IPluginArgs");
+/**
+ * `enableDashboard` was described as "Serve the React dashboard at /xenon/
+ * and the Socket.io event stream". It never decided that: the server mounts
+ * /xenon on every start (ServerManager.registerRoutes) and nothing under it
+ * reads the option. What it decides is how much a hub records of a session
+ * (SessionLifecycleService, the command interceptor, the session gateway's
+ * dashboard hooks).
+ */
+const schema = JSON.parse(fs_1.default.readFileSync(path_1.default.join(__dirname, '..', '..', 'schema.json'), 'utf8'));
+describe('the enableDashboard option', () => {
+    it('is never read by the dashboard: not to build it, not to answer a request', () => __awaiter(void 0, void 0, void 0, function* () {
+        const reads = [];
+        const args = new Proxy(Object.assign(Object.assign({}, IPluginArgs_1.DefaultPluginArgs), { enableDashboard: false }), {
+            get(target, key, receiver) {
+                var _a;
+                if (key === 'enableDashboard')
+                    reads.push((_a = new Error().stack) !== null && _a !== void 0 ? _a : '');
+                return Reflect.get(target, key, receiver);
+            },
+        });
+        const app = (0, express_1.default)();
+        app.use('/xenon', (0, index_1.createRouter)(args));
+        for (const url of ['/xenon/', '/xenon/sessions', '/xenon/api/health', '/xenon/api/session']) {
+            yield (0, loopbackRequest_1.default)(app).get(url);
+        }
+        (0, chai_1.expect)(reads, reads.join('\n\n')).to.be.empty;
+    }));
+    it('is described by what it does: records sessions, while the dashboard is always served', () => {
+        var _a;
+        const description = (_a = schema.properties.enableDashboard.description) !== null && _a !== void 0 ? _a : '';
+        (0, chai_1.expect)(description).to.match(/^Keep a full record of each Appium session for the dashboard/);
+        (0, chai_1.expect)(description).to.match(/The dashboard itself is always served at \/xenon\//);
+        (0, chai_1.expect)(description).not.to.match(/Serve the React dashboard|Socket\.io/);
+    });
+});

package/lib/test/unit/event-log-session-data.spec.js

@@ -0,0 +1,244 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const SocketServer_1 = require("../../src/services/SocketServer");
+const EventLogService_1 = require("../../src/services/EventLogService");
+const prisma_service_1 = require("../../src/data-service/prisma-service");
+const InterceptorService_1 = require("../../src/services/InterceptorService");
+const SelectorStateService_1 = require("../../src/services/SelectorStateService");
+const event_manager_1 = require("../../src/dashboard/event-manager");
+const SessionManager_1 = require("../../src/sessions/SessionManager");
+const SocketEvents_1 = require("../../src/enums/SocketEvents");
+const scratch_database_1 = require("../helpers/scratch-database");
+const container_registration_1 = require("../helpers/container-registration");
+/** What a session's data may carry: the app's sign-in, its cookies, a password typed, personal data. */
+const SECRETS = [
+    'Bearer eyJ-app-access-token',
+    'sid=app-session-cookie',
+    'hunter2',
+    'jane@example.com',
+];
+function capturedRequest(id) {
+    return {
+        id,
+        sessionId: 's1',
+        ts: 1700000000000,
+        method: 'POST',
+        url: 'https://api.example.com/v1/login',
+        host: 'api.example.com',
+        path: '/v1/login',
+        reqHeaders: { authorization: 'Bearer eyJ-app-access-token', cookie: 'sid=app-session-cookie' },
+        reqBody: JSON.stringify({ email: 'jane@example.com', password: 'hunter2' }),
+        resStatus: 200,
+        resHeaders: { 'set-cookie': 'sid=app-session-cookie; HttpOnly' },
+        resBody: JSON.stringify({ user: { email: 'jane@example.com' } }),
+        durationMs: 12,
+        mocked: false,
+        modified: false,
+    };
+}
+/** A socket.io server with no dashboard socket: the room broadcast, recorded. */
+function fakeIo() {
+    const broadcast = [];
+    const io = {
+        sockets: { adapter: { rooms: new Map() }, sockets: new Map() },
+        to: () => ({ emit: (event, data) => broadcast.push([event, data]) }),
+    };
+    return { io, broadcast };
+}
+describe("The event log keeps no copy of a session's own data", () => {
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let restore;
+    let eventLogEnv;
+    let appendSafe;
+    let broadcast;
+    beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
+        restore = (0, container_registration_1.saveRegistrations)(SocketServer_1.SocketServer, EventLogService_1.EventLogService);
+        // The log must be on, or "no row" would prove nothing.
+        eventLogEnv = process.env.XENON_EVENT_LOG;
+        delete process.env.XENON_EVENT_LOG;
+        yield scratch.db.eventLog.deleteMany({});
+        const eventLog = new EventLogService_1.EventLogService(new prisma_service_1.PrismaService());
+        appendSafe = sinon_1.default.spy(eventLog, 'appendSafe');
+        typedi_1.Container.set(EventLogService_1.EventLogService, eventLog);
+        const server = new SocketServer_1.SocketServer();
+        const fake = fakeIo();
+        server.io = fake.io;
+        broadcast = fake.broadcast;
+        typedi_1.Container.set(SocketServer_1.SocketServer, server);
+    }));
+    afterEach(() => {
+        sinon_1.default.restore();
+        restore();
+        if (eventLogEnv === undefined)
+            delete process.env.XENON_EVENT_LOG;
+        else
+            process.env.XENON_EVENT_LOG = eventLogEnv;
+    });
+    /** Waits for the fire-and-forget writes of `count` rows, then lets any later one land. */
+    function eventLogRows(count) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const deadline = Date.now() + 5000;
+            while ((yield scratch.db.eventLog.count()) < count && Date.now() < deadline) {
+                yield new Promise((r) => setTimeout(r, 10));
+            }
+            yield new Promise((r) => setTimeout(r, 50));
+            return scratch.db.eventLog.findMany();
+        });
+    }
+    const live = (event) => broadcast.filter(([name]) => name === event).map(([, data]) => data);
+    describe('a captured request', () => {
+        it('reaches the live dashboard whole, and the EventLog table never', () => __awaiter(void 0, void 0, void 0, function* () {
+            const svc = new InterceptorService_1.InterceptorService();
+            const emit = (evt) => svc.emit(evt, 'phone-a');
+            emit({ type: 'session_started', sessionId: 's1', port: 9100, host: '10.0.2.2' });
+            emit({ type: 'request', sessionId: 's1', payload: capturedRequest('q1') });
+            emit({ type: 'request', sessionId: 's1', payload: capturedRequest('q2') });
+            emit({ type: 'session_stopped', sessionId: 's1' });
+            const rows = yield eventLogRows(2);
+            // Written in the same millisecond, so in no particular order.
+            (0, chai_1.expect)(rows.map((r) => r.type)).to.have.members([
+                SocketEvents_1.SocketEvents.INTERCEPTOR_SESSION_STARTED,
+                SocketEvents_1.SocketEvents.INTERCEPTOR_SESSION_STOPPED,
+            ]);
+            for (const row of rows) {
+                for (const secret of SECRETS)
+                    (0, chai_1.expect)(row.payload).to.not.include(secret);
+            }
+            (0, chai_1.expect)(appendSafe.getCalls().map((c) => c.args[0].type)).to.not.include(SocketEvents_1.SocketEvents.INTERCEPTOR_REQUEST);
+            // The session page's Network panel still gets the whole entry, live.
+            (0, chai_1.expect)(live(SocketEvents_1.SocketEvents.INTERCEPTOR_REQUEST)).to.deep.equal([
+                capturedRequest('q1'),
+                capturedRequest('q2'),
+            ]);
+        }));
+    });
+    describe('a command', () => {
+        const SESSION = 's2';
+        beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
+            yield scratch.db.sessionLog.deleteMany({});
+            yield scratch.db.session.deleteMany({});
+            yield scratch.db.session.create({
+                data: {
+                    id: SESSION,
+                    desired_capabilities: '{}',
+                    session_capabilities: '{}',
+                    node_id: 'node',
+                    has_live_video: false,
+                    device_udid: 'phone-a',
+                    device_platform: 'android',
+                    device_version: '14',
+                },
+            });
+            sinon_1.default.stub(SessionManager_1.SESSION_MANAGER, 'getSession').returns({
+                getId: () => SESSION,
+                getDevice: () => ({ udid: 'phone-a', name: 'P', platform: 'android' }),
+                getXenonOption: () => undefined,
+                getScreenShot: () => __awaiter(void 0, void 0, void 0, function* () { return ''; }),
+            });
+            sinon_1.default.stub(SelectorStateService_1.SelectorStateService.prototype, 'onHealRecorded').resolves(null);
+        }));
+        function command(name, args, answer, healed) {
+            return event_manager_1.DASHBORD_EVENT_MANAGER.afterSessionCommand(SESSION, name, null, { body: args, method: 'POST', path: `/${name}`, originalUrl: `/${name}` }, {}, JSON.stringify({ value: answer, sessionId: SESSION }), healed);
+        }
+        it('logs and sends which command ran and how it went, never what it typed or answered', () => __awaiter(void 0, void 0, void 0, function* () {
+            var _a, _b;
+            const pageSource = '<hierarchy><node text="jane@example.com"/></hierarchy>';
+            yield command('setValue', ['hunter2', 'element-1'], null);
+            yield command('getPageSource', [], pageSource);
+            yield command('findElement', ['xpath', '//*[@text="Pay"]'], { ELEMENT: 'e' }, {
+                originalSelector: '//*[@text="Pay"]',
+                originalStrategy: 'xpath',
+                healedSelector: 'pay_button',
+                healedStrategy: 'id',
+                confidence: 0.9,
+                tier: 2,
+            });
+            // Three commands and the heal's own event.
+            const rows = yield eventLogRows(4);
+            const commands = rows
+                .filter((r) => r.type === SocketEvents_1.SocketEvents.SESSION_COMMAND)
+                .map((r) => JSON.parse(r.payload));
+            (0, chai_1.expect)(commands.map((c) => c.command_name)).to.have.members([
+                'setValue',
+                'getPageSource',
+                'findElement',
+            ]);
+            for (const row of rows) {
+                for (const secret of SECRETS)
+                    (0, chai_1.expect)(row.payload).to.not.include(secret);
+            }
+            for (const logged of commands) {
+                (0, chai_1.expect)(logged).to.not.have.any.keys('body', 'response', 'screenshot', 'url');
+                (0, chai_1.expect)(logged).to.include({ session_id: SESSION, is_success: true, is_error: false });
+            }
+            // What a reader of the log needs from a heal is there.
+            (0, chai_1.expect)(commands.find((c) => c.command_name === 'findElement')).to.include({
+                is_healed: true,
+                original_strategy: 'xpath',
+                original_selector: '//*[@text="Pay"]',
+                healed_strategy: 'id',
+                healed_selector: 'pay_button',
+                healing_confidence: 0.9,
+                healing_tier: 'Fuzzy XML',
+            });
+            // The session's own record, which goes with the session, keeps them.
+            const sessionLog = yield scratch.db.sessionLog.findMany({ where: { session_id: SESSION } });
+            (0, chai_1.expect)((_a = sessionLog.find((l) => l.command_name === 'setValue')) === null || _a === void 0 ? void 0 : _a.body).to.include('hunter2');
+            (0, chai_1.expect)((_b = sessionLog.find((l) => l.command_name === 'getPageSource')) === null || _b === void 0 ? void 0 : _b.response).to.include('jane@example.com');
+            // The live event is the same summary. It reaches every dashboard that
+            // can see the phone, and no client reads more (the dashboard and Xenon
+            // Studio don't subscribe to it).
+            const sent = live(SocketEvents_1.SocketEvents.SESSION_COMMAND);
+            (0, chai_1.expect)(sent.map((c) => c.command_name)).to.deep.equal([
+                'setValue',
+                'getPageSource',
+                'findElement',
+            ]);
+            for (const summary of sent) {
+                for (const secret of SECRETS)
+                    (0, chai_1.expect)(JSON.stringify(summary)).to.not.include(secret);
+                (0, chai_1.expect)(summary).to.deep.equal(commands.find((c) => c.command_name === summary.command_name));
+            }
+        }));
+    });
+    it("every other event is logged once and whole, by either emit; a command's own fields are named", () => __awaiter(void 0, void 0, void 0, function* () {
+        const server = typedi_1.Container.get(SocketServer_1.SocketServer);
+        server.emitToDashboard(SocketEvents_1.SocketEvents.INTERCEPTOR_REQUEST, capturedRequest('q3'));
+        yield server.emitToDashboardForDevices(SocketEvents_1.SocketEvents.INTERCEPTOR_REQUEST, capturedRequest('q4'), { udid: 'phone-a' });
+        (0, chai_1.expect)(appendSafe.called).to.equal(false);
+        const payload = { session_id: 's', command_name: 'click', body: '["hunter2"]', extra: 1 };
+        const others = Object.values(SocketEvents_1.SocketEvents).filter((event) => event !== SocketEvents_1.SocketEvents.INTERCEPTOR_REQUEST);
+        for (const event of others) {
+            yield server.emitToDashboardForDevices(event, payload, { udid: 'phone-a' });
+        }
+        server.emitToDashboard(SocketEvents_1.SocketEvents.SESSION_COMMAND, payload);
+        (0, chai_1.expect)(appendSafe.getCalls().map((c) => c.args[0])).to.deep.equal([
+            ...others.map((event) => ({
+                type: event,
+                payload: event === SocketEvents_1.SocketEvents.SESSION_COMMAND
+                    ? { session_id: 's', command_name: 'click' }
+                    : payload,
+            })),
+            { type: SocketEvents_1.SocketEvents.SESSION_COMMAND, payload: { session_id: 's', command_name: 'click' } },
+        ]);
+        // The writes are fire-and-forget: let them land here, or they land in the
+        // next test's table after its beforeEach emptied it.
+        (0, chai_1.expect)(yield eventLogRows(others.length + 1)).to.have.length(others.length + 1);
+    }));
+});

package/lib/test/unit/failure-analysis.spec.js

@@ -0,0 +1,182 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const AIService_1 = require("../../src/services/AIService");
+const CircuitBreaker_1 = require("../../src/services/CircuitBreaker");
+const failure_analysis_service_1 = require("../../src/dashboard/services/failure-analysis-service");
+const scratch_database_1 = require("../helpers/scratch-database");
+const fake_ai_provider_1 = require("../helpers/fake-ai-provider");
+/**
+ * A failed session gets two things: a category from its failure reason and
+ * failed commands (rules, no AI), and, with an AI provider, an explanation.
+ * They are saved separately, so the category never waits for the AI, and only
+ * a real explanation is saved: a rate limit, a timeout or a failed call saves
+ * nothing, and leaves an explanation saved earlier in place.
+ */
+describe('failure analysis of a failed session', () => {
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    const ai = (0, fake_ai_provider_1.useFakeAiProviders)();
+    const ID = 'sess-failure-analysis';
+    beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
+        yield scratch.db.log.deleteMany();
+        yield scratch.db.sessionLog.deleteMany();
+        yield scratch.db.session.deleteMany();
+        yield scratch.db.session.create({
+            data: {
+                id: ID,
+                status: 'failed',
+                failure_reason: 'An element could not be located on the page',
+                desired_capabilities: '{}',
+                session_capabilities: '{}',
+                node_id: 'node',
+                has_live_video: false,
+                device_udid: 'R58M123',
+                device_platform: 'android',
+                device_version: '10',
+            },
+        });
+        yield scratch.db.sessionLog.create({
+            data: {
+                session_id: ID,
+                command_name: 'findElement',
+                url: '/session/x/element',
+                method: 'POST',
+                title: 'findElement',
+                response: '{"value":{"error":"no such element"}}',
+                is_error: true,
+                is_success: false,
+            },
+        });
+    }));
+    afterEach(() => sinon_1.default.restore());
+    const row = () => scratch.db.session.findUniqueOrThrow({ where: { id: ID } });
+    it('files the session under its category without asking the AI', () => __awaiter(void 0, void 0, void 0, function* () {
+        ai.use('anthropic');
+        ai.answer({ text: 'Root Cause: x' });
+        yield (0, failure_analysis_service_1.categorizeSessionFailure)(ID);
+        (0, chai_1.expect)((yield row()).failure_category).to.equal('ELEMENT_NOT_FOUND');
+        (0, chai_1.expect)(ai.calls).to.have.length(0);
+    }));
+    it("saves the provider's explanation", () => __awaiter(void 0, void 0, void 0, function* () {
+        ai.use('anthropic');
+        ai.answer({ text: 'Root Cause: the Login button was covered by a dialog.' });
+        yield (0, failure_analysis_service_1.explainSessionFailure)(ID);
+        (0, chai_1.expect)((yield row()).ai_analysis).to.equal('Root Cause: the Login button was covered by a dialog.');
+    }));
+    for (const provider of fake_ai_provider_1.PROVIDERS) {
+        it(`saves no explanation when ${provider} is rate-limited`, () => __awaiter(void 0, void 0, void 0, function* () {
+            ai.use(provider);
+            ai.answer({ status: 429 });
+            yield (0, failure_analysis_service_1.explainSessionFailure)(ID);
+            (0, chai_1.expect)(ai.calls).to.have.length(1);
+            (0, chai_1.expect)((yield row()).ai_analysis).to.equal(null);
+        }));
+    }
+    it('leaves an earlier explanation when a later attempt gets none', () => __awaiter(void 0, void 0, void 0, function* () {
+        // A session can end twice (a driver crash, then the client's delete).
+        yield scratch.db.session.update({
+            where: { id: ID },
+            data: { ai_analysis: 'Root Cause: from the first end.' },
+        });
+        ai.use('gemini');
+        ai.answer({ status: 429 });
+        yield (0, failure_analysis_service_1.explainSessionFailure)(ID);
+        (0, chai_1.expect)((yield row()).ai_analysis).to.equal('Root Cause: from the first end.');
+    }));
+    describe('how many run', () => {
+        // Analyses run after their sessions have ended, so nothing else limits
+        // them: a broken build on a big lab ends hundreds of failed sessions in
+        // minutes, and each analysis holds a screenshot while it runs.
+        let answers;
+        let analyzeFailure;
+        beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
+            answers = [];
+            sinon_1.default.stub(AIService_1.AI_SERVICE, 'isEnabled').returns(true);
+            analyzeFailure = sinon_1.default
+                .stub(AIService_1.AI_SERVICE, 'analyzeFailure')
+                .callsFake(() => new Promise((resolve) => answers.push(resolve)));
+        }));
+        const settle = () => new Promise((r) => setTimeout(r, 50));
+        it('asks once for a session whose analysis is already running', () => __awaiter(void 0, void 0, void 0, function* () {
+            const first = (0, failure_analysis_service_1.explainSessionFailure)(ID);
+            const second = (0, failure_analysis_service_1.explainSessionFailure)(ID);
+            yield settle();
+            (0, chai_1.expect)(analyzeFailure.callCount).to.equal(1);
+            answers[0]('Root Cause: once.');
+            yield Promise.all([first, second]);
+            (0, chai_1.expect)((yield row()).ai_analysis).to.equal('Root Cause: once.');
+        }));
+        it(`runs ${failure_analysis_service_1.MAX_CONCURRENT_FAILURE_ANALYSES} at once; the others wait their turn`, () => __awaiter(void 0, void 0, void 0, function* () {
+            const ids = Array.from({ length: failure_analysis_service_1.MAX_CONCURRENT_FAILURE_ANALYSES + 2 }, (_, i) => `${ID}-${i}`);
+            for (const id of ids) {
+                yield scratch.db.session.create({
+                    data: {
+                        id,
+                        status: 'failed',
+                        desired_capabilities: '{}',
+                        session_capabilities: '{}',
+                        node_id: 'node',
+                        has_live_video: false,
+                        device_udid: 'R58M123',
+                        device_platform: 'android',
+                        device_version: '10',
+                    },
+                });
+            }
+            const all = ids.map((id) => (0, failure_analysis_service_1.explainSessionFailure)(id));
+            yield settle();
+            (0, chai_1.expect)(analyzeFailure.callCount).to.equal(failure_analysis_service_1.MAX_CONCURRENT_FAILURE_ANALYSES);
+            answers[0](null);
+            yield settle();
+            (0, chai_1.expect)(analyzeFailure.callCount).to.equal(failure_analysis_service_1.MAX_CONCURRENT_FAILURE_ANALYSES + 1);
+            for (let i = 0; i < 10 && answers.length < ids.length; i++) {
+                answers.forEach((answer) => answer(null));
+                yield settle();
+            }
+            answers.forEach((answer) => answer(null));
+            yield Promise.all(all);
+            (0, chai_1.expect)(analyzeFailure.callCount).to.equal(ids.length);
+        }));
+    });
+    describe('a provider that does not answer', () => {
+        const FAILURE = {
+            sessionId: ID,
+            failureReason: 'An element could not be located on the page',
+            commandLogs: [],
+            deviceLogs: [],
+        };
+        for (const provider of fake_ai_provider_1.PROVIDERS) {
+            it(`${provider}: is given up on after the time limit, and its request cancelled`, () => __awaiter(void 0, void 0, void 0, function* () {
+                var _a, _b;
+                const { model } = ai.use(provider);
+                ai.answer('hang');
+                const clock = sinon_1.default.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+                const service = new AIService_1.AIService();
+                const analysis = service.analyzeFailure(FAILURE);
+                yield clock.tickAsync(AIService_1.FAILURE_ANALYSIS_TIMEOUT_MS - 1);
+                (0, chai_1.expect)((_a = ai.calls[0].signal) === null || _a === void 0 ? void 0 : _a.aborted).to.equal(false);
+                yield clock.tickAsync(1);
+                (0, chai_1.expect)(yield analysis).to.equal(null);
+                (0, chai_1.expect)(ai.calls).to.have.length(1);
+                (0, chai_1.expect)((_b = ai.calls[0].signal) === null || _b === void 0 ? void 0 : _b.aborted).to.equal(true);
+                // A provider that doesn't answer is a failure, like one that refuses.
+                const breaker = CircuitBreaker_1.CIRCUIT_BREAKERS.snapshot().find((b) => b.key === `ai:${provider}:${model}`);
+                (0, chai_1.expect)(breaker === null || breaker === void 0 ? void 0 : breaker.consecutiveFailures).to.equal(1);
+            }));
+        }
+    });
+});

package/lib/test/unit/failure-categories.spec.js

@@ -0,0 +1,128 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const typescript_1 = __importDefault(require("typescript"));
+const swagger_1 = require("../../src/app/swagger");
+const failure_analysis_service_1 = require("../../src/dashboard/services/failure-analysis-service");
+const SessionManager_1 = require("../../src/sessions/SessionManager");
+/**
+ * A Session's `failure_category` is written in two places: the failure
+ * analysis (one of ANALYSIS_CATEGORIES) and a restart's recovery
+ * (HUB_RESTART). Both write upper case.
+ *
+ * The API reference showed `element_not_found`, a value the server never
+ * writes, and the dashboard had a runbook for "Infrastructure", a category
+ * nothing ever assigns. These hold both to what the server writes.
+ */
+const WRITTEN = [...failure_analysis_service_1.ANALYSIS_CATEGORIES, SessionManager_1.HUB_RESTART_CATEGORY];
+/**
+ * The dashboard's runbooks. web/ is an ES module package, so its .ts can't be
+ * required from here; the file imports nothing, so it is compiled and run.
+ */
+function loadRunbooks() {
+    const file = path_1.default.resolve(__dirname, '../../web/src/components/runbooks/runbook-content.ts');
+    const { outputText } = typescript_1.default.transpileModule(fs_1.default.readFileSync(file, 'utf8'), {
+        compilerOptions: { module: typescript_1.default.ModuleKind.CommonJS, target: typescript_1.default.ScriptTarget.ES2019 },
+    });
+    const mod = { exports: {} };
+    new Function('module', 'exports', outputText)(mod, mod.exports);
+    return mod.exports;
+}
+const { RUNBOOKS, runbookKey } = loadRunbooks();
+/** Every value found under a key named `key`, at any depth. */
+function valuesUnder(node, key, out = []) {
+    if (Array.isArray(node)) {
+        node.forEach((n) => valuesUnder(n, key, out));
+    }
+    else if (node && typeof node === 'object') {
+        for (const [k, v] of Object.entries(node)) {
+            if (k === key)
+                out.push(v);
+            valuesUnder(v, key, out);
+        }
+    }
+    return out;
+}
+/** The `failure_category` column of every CSV example (a string with that header). */
+function csvExampleCategories() {
+    const out = [];
+    for (const example of valuesUnder(swagger_1.swaggerSpec, 'example')) {
+        if (typeof example !== 'string')
+            continue;
+        const [header, ...rows] = example.trim().split('\n');
+        const column = header.split(',').indexOf('failure_category');
+        if (column < 0)
+            continue;
+        for (const row of rows)
+            out.push(row.split(',')[column]);
+    }
+    return out;
+}
+describe('failure categories', () => {
+    it('are written upper case, as the API reference and the dashboard read them', () => {
+        (0, chai_1.expect)(failure_analysis_service_1.ANALYSIS_CATEGORIES).to.include('UNKNOWN');
+        for (const category of WRITTEN)
+            (0, chai_1.expect)(category).to.match(/^[A-Z_]+$/);
+    });
+    describe('in the API reference', () => {
+        it('every example shows one the server writes, as it writes it', () => {
+            const shown = [
+                ...valuesUnder(swagger_1.swaggerSpec, 'failure_category').filter((v) => typeof v === 'string'),
+                ...csvExampleCategories(),
+            ];
+            // The session list's example and the build export's CSV.
+            (0, chai_1.expect)(shown).to.have.length.of.at.least(2);
+            for (const value of shown)
+                (0, chai_1.expect)(WRITTEN, String(value)).to.include(value);
+        });
+        it("the session record's description names every category, and no other", () => {
+            const described = valuesUnder(swagger_1.swaggerSpec, 'failure_category')
+                .filter((v) => !!(v === null || v === void 0 ? void 0 : v.description))
+                .map((v) => v.description);
+            (0, chai_1.expect)(described).to.have.length(1);
+            const named = Array.from(described[0].matchAll(/`([A-Z_]+)`/g), (m) => m[1]);
+            (0, chai_1.expect)([...named].sort()).to.deep.equal([...WRITTEN].sort());
+        });
+    });
+    describe("in the dashboard's runbooks", () => {
+        it('every category the server writes has a runbook of its own', () => {
+            for (const category of WRITTEN) {
+                (0, chai_1.expect)(Object.keys(RUNBOOKS), category).to.include(runbookKey(category));
+            }
+        });
+        it("each quotes a text that files a failure under its category, not just the category's name", () => {
+            // The names mislead: "Xenon command failure" is a "command failed"
+            // message, rarely Xenon's, and few real crashes are worded the way App
+            // crash's patterns expect. A runbook has to say what really files a
+            // failure there, so it must quote one of its category's texts. Only
+            // quoted text in the body counts: the title ("# WDA failure") and the
+            // prose around a quote ("says a command failed") would pass anything.
+            for (const { category, patterns } of failure_analysis_service_1.ERROR_PATTERNS) {
+                const { markdown } = RUNBOOKS[runbookKey(category)];
+                const body = markdown.split('\n').slice(1).join(' ').replace(/\s+/g, ' ');
+                const quoted = Array.from(body.matchAll(/"([^"]+)"/g), (m) => m[1]);
+                (0, chai_1.expect)(patterns.some((p) => quoted.some((q) => new RegExp(p, 'i').test(q))), `the ${category} runbook quotes none of ${JSON.stringify(patterns)}`).to.equal(true);
+            }
+        });
+        it('every runbook is for a category the server writes', () => {
+            const written = WRITTEN.map(runbookKey);
+            for (const key of Object.keys(RUNBOOKS)) {
+                (0, chai_1.expect)(written, `runbook "${key}"`).to.include(key);
+            }
+        });
+        it('a stored category opens its own runbook from the "Why it failed" card', () => {
+            var _a, _b;
+            // failure-summary.tsx links /runbooks/<the category, lower case>.
+            const linked = (category) => RUNBOOKS[runbookKey(category.toLowerCase())];
+            (0, chai_1.expect)((_a = linked(SessionManager_1.HUB_RESTART_CATEGORY)) === null || _a === void 0 ? void 0 : _a.title).to.equal('Hub restart');
+            (0, chai_1.expect)((_b = linked('TIMEOUT')) === null || _b === void 0 ? void 0 : _b.title).to.equal('Timeout');
+            (0, chai_1.expect)(linked('UNKNOWN')).to.equal(RUNBOOKS.unknown);
+        });
+    });
+});