@xenon-device-management/xenon 2.10.0 → 2.11.0

package/lib/test/unit/node-metrics-collector.spec.js

@@ -0,0 +1,226 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const NodeMetricsCollector_1 = require("../../src/services/metrics/NodeMetricsCollector");
+const nodeMetrics_1 = require("../../src/services/metrics/nodeMetrics");
+const sample = (at) => ({
+    at,
+    deviceCpuPct: 5,
+    deviceMemMb: 1,
+    deviceMemTotalMb: 2,
+    appCpuPct: null,
+    appMemMb: null,
+    appId: null,
+});
+const answer = (state, ats) => ({
+    kind: 'answer',
+    answer: { platform: 'android', state, samples: ats.map(sample) },
+});
+describe('NodeMetricsCollector: the hub collects a node session’s figures', () => {
+    let clock;
+    let replies;
+    let asked;
+    let got;
+    let warned;
+    let support;
+    /** While set, an ask waits on it: a node slow to answer. */
+    let gate;
+    const source = {
+        nodeOrigin: () => 'http://node:4723',
+        nodeMetrics: (after) => __awaiter(void 0, void 0, void 0, function* () {
+            var _a;
+            asked.push(after);
+            if (gate)
+                yield gate;
+            return (_a = replies.shift()) !== null && _a !== void 0 ? _a : answer('sampling', []);
+        }),
+    };
+    const make = (after = null) => new NodeMetricsCollector_1.NodeMetricsCollector({
+        source,
+        hooks: { onSample: (s) => got.push(s.at), onGiveUp: () => undefined },
+        support,
+        logger: { info: () => undefined, warn: (m) => warned.push(m) },
+        after,
+    });
+    beforeEach(() => {
+        clock = sinon_1.default.useFakeTimers({ now: 0 });
+        replies = [];
+        asked = [];
+        got = [];
+        warned = [];
+        gate = null;
+        support = new nodeMetrics_1.NodeMetricsSupport();
+        support.now = () => clock.now;
+        support.logger = { warn: (m) => warned.push(m) };
+    });
+    afterEach(() => clock.restore());
+    it('asks every 10 s from the newest sample it has, and passes the samples on', () => __awaiter(void 0, void 0, void 0, function* () {
+        replies = [answer('sampling', [1, 2]), answer('sampling', [3])];
+        const c = make();
+        c.start();
+        yield clock.tickAsync(10000);
+        yield clock.tickAsync(10000);
+        (0, chai_1.expect)(asked).to.deep.equal([null, 2]);
+        (0, chai_1.expect)(got).to.deep.equal([1, 2, 3]);
+        (0, chai_1.expect)(c.state()).to.equal('sampling');
+    }));
+    it('starts from the newest sample stored, after a hub restart', () => __awaiter(void 0, void 0, void 0, function* () {
+        const c = make(500);
+        c.start();
+        yield clock.tickAsync(10000);
+        (0, chai_1.expect)(asked).to.deep.equal([500]);
+    }));
+    it('asks once more when the session ends, and takes the last samples', () => __awaiter(void 0, void 0, void 0, function* () {
+        replies = [answer('sampling', [1]), answer('ended', [2, 3])];
+        const c = make();
+        c.start();
+        yield clock.tickAsync(10000);
+        yield c.stop();
+        (0, chai_1.expect)(got).to.deep.equal([1, 2, 3]);
+        (0, chai_1.expect)(c.state()).to.equal('stopped');
+        yield clock.tickAsync(30000);
+        (0, chai_1.expect)(asked).to.have.length(2);
+    }));
+    it('stops asking when the node says off, or that its sampler stopped', () => __awaiter(void 0, void 0, void 0, function* () {
+        for (const [state, ats, expected] of [
+            ['off', [], 'off'],
+            ['stopped', [7], 'stopped'],
+        ]) {
+            asked = [];
+            replies = [answer(state, [...ats])];
+            const c = make();
+            c.start();
+            yield clock.tickAsync(30000);
+            (0, chai_1.expect)(asked, state).to.have.length(1);
+            (0, chai_1.expect)(c.state(), state).to.equal(expected);
+            yield c.stop();
+            (0, chai_1.expect)(asked, `${state}: no last ask`).to.have.length(1);
+        }
+    }));
+    it('stops asking a node without the route, says so once, and asks it again later', () => __awaiter(void 0, void 0, void 0, function* () {
+        replies = [{ kind: 'unsupported', status: 404 }];
+        const first = make();
+        first.start();
+        yield clock.tickAsync(10000);
+        (0, chai_1.expect)(first.state()).to.equal('off');
+        const second = make();
+        second.start();
+        yield clock.tickAsync(10000);
+        (0, chai_1.expect)(asked).to.have.length(1);
+        (0, chai_1.expect)(second.state()).to.equal('off');
+        (0, chai_1.expect)(warned.filter((m) => m.includes('http://node:4723'))).to.have.length(1);
+        yield clock.tickAsync(10 * 60000);
+        const third = make();
+        third.start();
+        yield clock.tickAsync(10000);
+        (0, chai_1.expect)(asked).to.have.length(2);
+    }));
+    it('keeps the figures it has when the node later says off or refuses: it says stopped', () => __awaiter(void 0, void 0, void 0, function* () {
+        for (const last of [answer('off', []), { kind: 'refused' }]) {
+            asked = [];
+            replies = [answer('sampling', [1]), last];
+            const c = make();
+            c.start();
+            yield clock.tickAsync(20000);
+            (0, chai_1.expect)(asked, last.kind).to.have.length(2);
+            (0, chai_1.expect)(c.state(), last.kind).to.equal('stopped');
+        }
+    }));
+    it("stops asking when the node refuses the hub's token", () => __awaiter(void 0, void 0, void 0, function* () {
+        replies = [{ kind: 'refused' }];
+        const c = make();
+        c.start();
+        yield clock.tickAsync(30000);
+        (0, chai_1.expect)(asked).to.have.length(1);
+        (0, chai_1.expect)(c.state()).to.equal('off');
+    }));
+    it('keeps asking an unreachable node, saying so once per outage', () => __awaiter(void 0, void 0, void 0, function* () {
+        replies = [
+            { kind: 'unavailable', reason: 'ECONNREFUSED' },
+            { kind: 'unavailable', reason: 'ECONNREFUSED' },
+            answer('sampling', [1]),
+            { kind: 'unavailable', reason: 'timeout' },
+        ];
+        const c = make();
+        c.start();
+        yield clock.tickAsync(40000);
+        (0, chai_1.expect)(asked).to.have.length(4);
+        (0, chai_1.expect)(got).to.deep.equal([1]);
+        (0, chai_1.expect)(warned).to.have.length(2);
+        (0, chai_1.expect)(c.state()).to.equal('sampling');
+    }));
+    it('finishes stopping when the node is unreachable', () => __awaiter(void 0, void 0, void 0, function* () {
+        replies = [{ kind: 'unavailable', reason: 'timeout' }];
+        const c = make();
+        c.start();
+        yield c.stop();
+        (0, chai_1.expect)(asked).to.have.length(1);
+    }));
+    it('never asks twice at once, or takes a sample twice', () => __awaiter(void 0, void 0, void 0, function* () {
+        let release;
+        gate = new Promise((r) => (release = r));
+        replies = [answer('sampling', [1, 2]), answer('sampling', [2, 3])];
+        const c = make();
+        c.start();
+        yield clock.tickAsync(10000); // the first ask, held by the node
+        yield clock.tickAsync(20000); // two more ticks while it is held
+        (0, chai_1.expect)(asked).to.have.length(1);
+        gate = null;
+        release();
+        yield clock.tickAsync(10000); // the first answer arrives; the next ask goes
+        (0, chai_1.expect)(asked).to.deep.equal([null, 2]);
+        (0, chai_1.expect)(got).to.deep.equal([1, 2, 3]);
+    }));
+});
+describe("readNodeMetricsReply: what a node's answer means", () => {
+    const h = { [nodeMetrics_1.NODE_METRICS_HEADER]: '1' };
+    it('reads each answer', () => {
+        (0, chai_1.expect)((0, nodeMetrics_1.readNodeMetricsReply)(404, {}, undefined)).to.deep.equal({
+            kind: 'unsupported',
+            status: 404,
+        });
+        (0, chai_1.expect)((0, nodeMetrics_1.readNodeMetricsReply)(404, h, {})).to.deep.equal({ kind: 'refused' });
+        (0, chai_1.expect)((0, nodeMetrics_1.readNodeMetricsReply)(200, h, {
+            value: { platform: 'ios', state: 'sampling', samples: [sample(1)] },
+        })).to.deep.equal({
+            kind: 'answer',
+            answer: { platform: 'ios', state: 'sampling', samples: [sample(1)] },
+        });
+        // Without the header: an older node only for what an older node answers
+        // (its login's 401, an unknown route's 404, a catch-all's 2xx). A proxy's
+        // 502 or 503 in front of a new node is an outage, not an older node.
+        (0, chai_1.expect)((0, nodeMetrics_1.readNodeMetricsReply)(401, {}, {})).to.deep.equal({ kind: 'unsupported', status: 401 });
+        (0, chai_1.expect)((0, nodeMetrics_1.readNodeMetricsReply)(200, {}, '<html>')).to.deep.equal({
+            kind: 'unsupported',
+            status: 200,
+        });
+        for (const status of [500, 502, 503, 504]) {
+            (0, chai_1.expect)((0, nodeMetrics_1.readNodeMetricsReply)(status, {}, 'Bad Gateway'), String(status)).to.deep.equal({
+                kind: 'unavailable',
+                reason: `answered ${status}`,
+            });
+        }
+        (0, chai_1.expect)((0, nodeMetrics_1.readNodeMetricsReply)(503, h, {})).to.deep.equal({
+            kind: 'unavailable',
+            reason: 'answered 503',
+        });
+        (0, chai_1.expect)((0, nodeMetrics_1.readNodeMetricsReply)(200, h, { value: { state: 'weird', samples: [] } })).to.deep.equal({
+            kind: 'unavailable',
+            reason: 'answered 200',
+        });
+    });
+});

package/lib/test/unit/node-metrics-store.spec.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const NodeMetricsStore_1 = require("../../src/services/metrics/NodeMetricsStore");
+const nodeMetrics_1 = require("../../src/services/metrics/nodeMetrics");
+const types_1 = require("../../src/services/metrics/types");
+const sample = (at) => ({
+    at,
+    deviceCpuPct: 10,
+    deviceMemMb: 100,
+    deviceMemTotalMb: 4000,
+    appCpuPct: null,
+    appMemMb: null,
+    appId: null,
+});
+describe('NodeMetricsStore: a node holds its figures for the hub', () => {
+    let now;
+    let store;
+    beforeEach(() => {
+        now = 1000000;
+        store = new NodeMetricsStore_1.NodeMetricsStore();
+        store.now = () => now;
+    });
+    it('answers every sample without `after`, and only newer ones with it', () => {
+        store.begin('s1', 'Android');
+        [1, 2, 3].forEach((t) => store.add('s1', sample(t)));
+        (0, chai_1.expect)(store.read('s1', null)).to.deep.equal({
+            platform: 'android',
+            state: 'sampling',
+            samples: [sample(1), sample(2), sample(3)],
+        });
+        (0, chai_1.expect)(store.read('s1', 2).samples.map((s) => s.at)).to.deep.equal([3]);
+    });
+    it('drops what the hub has collected', () => {
+        store.begin('s1', 'android');
+        [1, 2, 3].forEach((t) => store.add('s1', sample(t)));
+        store.read('s1', 2);
+        (0, chai_1.expect)(store.read('s1', null).samples.map((s) => s.at)).to.deep.equal([3]);
+    });
+    it('keeps the newest 900 samples', () => {
+        store.begin('s1', 'android');
+        for (let t = 1; t <= types_1.MAX_BUFFERED_SAMPLES + 5; t++)
+            store.add('s1', sample(t));
+        const ats = store.read('s1', null).samples.map((s) => s.at);
+        (0, chai_1.expect)(ats).to.have.length(types_1.MAX_BUFFERED_SAMPLES);
+        (0, chai_1.expect)(ats[0]).to.equal(6);
+    });
+    it('says a sampler that gave up stopped, and an ended session ended', () => {
+        store.begin('s1', 'android');
+        store.gaveUp('s1');
+        (0, chai_1.expect)(store.read('s1', null).state).to.equal('stopped');
+        store.end('s1');
+        (0, chai_1.expect)(store.read('s1', null).state).to.equal('ended');
+    });
+    it('keeps an ended session 10 minutes for the last collection, then forgets it', () => {
+        store.begin('s1', 'android');
+        store.add('s1', sample(1));
+        store.end('s1');
+        now += nodeMetrics_1.KEEP_AFTER_END_MS - 1;
+        (0, chai_1.expect)(store.read('s1', null).samples).to.have.length(1);
+        now += 1;
+        (0, chai_1.expect)(store.read('s1', null)).to.deep.equal({ platform: '', state: 'off', samples: [] });
+    });
+    it('says off for a session it has nothing for', () => {
+        (0, chai_1.expect)(store.read('nope', null)).to.deep.equal({ platform: '', state: 'off', samples: [] });
+        store.add('nope', sample(1));
+        (0, chai_1.expect)(store.read('nope', null).samples).to.deep.equal([]);
+    });
+});

package/lib/test/unit/node-session-metrics-route.spec.js

@@ -0,0 +1,97 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const express_1 = __importDefault(require("express"));
+const supertest_1 = __importDefault(require("supertest"));
+const sinon_1 = __importDefault(require("sinon"));
+const nodeSessionMetrics_1 = require("../../src/gateway/nodeSessionMetrics");
+const nodeMetrics_1 = require("../../src/services/metrics/nodeMetrics");
+const hubSessionToken_1 = require("../../src/gateway/hubSessionToken");
+const commandAuth_1 = require("../../src/middleware/commandAuth");
+const IPluginArgs_1 = require("../../src/interfaces/IPluginArgs");
+const quiet = { info: () => undefined, warn: () => undefined, error: () => undefined };
+const ANSWER = { platform: 'android', state: 'sampling', samples: [] };
+describe("a node's session metrics route", () => {
+    let verify;
+    let read;
+    let enforced;
+    const app = () => {
+        const a = (0, express_1.default)();
+        a.get('/xenon/api/node/sessions/:sessionId/metrics', (0, nodeSessionMetrics_1.nodeSessionMetricsHandler)({
+            hubTokens: { verify },
+            enforced: () => enforced,
+            read,
+            logger: quiet,
+        }));
+        return a;
+    };
+    const get = (q = '') => (0, supertest_1.default)(app()).get(`/xenon/api/node/sessions/s1/metrics${q}`);
+    beforeEach(() => {
+        verify = sinon_1.default.stub();
+        read = sinon_1.default.stub().returns(ANSWER);
+        enforced = false;
+    });
+    afterEach(() => sinon_1.default.restore());
+    it('answers the figures, with no credential when per-command auth is off', () => __awaiter(void 0, void 0, void 0, function* () {
+        const res = yield get('?after=1500');
+        (0, chai_1.expect)(res.status).to.equal(200);
+        (0, chai_1.expect)(res.body).to.deep.equal({ value: ANSWER });
+        (0, chai_1.expect)(res.headers[nodeMetrics_1.NODE_METRICS_HEADER]).to.equal('1');
+        (0, chai_1.expect)(read.calledOnceWithExactly('s1', 1500)).to.equal(true);
+    }));
+    it("with per-command auth on, answers only the hub's token for that session", () => __awaiter(void 0, void 0, void 0, function* () {
+        enforced = true;
+        verify.withArgs('good', 's1').resolves(true);
+        verify.withArgs('other', 's1').resolves(false);
+        const missing = yield get();
+        const wrong = yield get().set(hubSessionToken_1.HUB_TOKEN_HEADER, 'other');
+        const right = yield get().set(hubSessionToken_1.HUB_TOKEN_HEADER, 'good');
+        (0, chai_1.expect)([missing.status, missing.body]).to.deep.equal([404, commandAuth_1.UNKNOWN_SESSION_BODY]);
+        (0, chai_1.expect)([wrong.status, wrong.body]).to.deep.equal([404, commandAuth_1.UNKNOWN_SESSION_BODY]);
+        (0, chai_1.expect)(right.status).to.equal(200);
+        (0, chai_1.expect)(read.calledOnce).to.equal(true);
+        for (const r of [missing, wrong, right])
+            (0, chai_1.expect)(r.headers[nodeMetrics_1.NODE_METRICS_HEADER]).to.equal('1');
+    }));
+    it("answers 503 when it can't check the hub's token", () => __awaiter(void 0, void 0, void 0, function* () {
+        enforced = true;
+        verify.rejects(new Error('JWKS unreachable'));
+        const res = yield get().set(hubSessionToken_1.HUB_TOKEN_HEADER, 'good');
+        (0, chai_1.expect)([res.status, res.body]).to.deep.equal([503, commandAuth_1.COMMAND_AUTH_UNAVAILABLE_BODY]);
+        (0, chai_1.expect)(read.called).to.equal(false);
+    }));
+    it('reads anything but a time as no `after`', () => {
+        (0, chai_1.expect)((0, nodeSessionMetrics_1.parseAfter)('1500')).to.equal(1500);
+        (0, chai_1.expect)((0, nodeSessionMetrics_1.parseAfter)('1500.5')).to.equal(1500.5);
+        for (const bad of [undefined, '', 'abc', '-5', '1e3', ['1']]) {
+            (0, chai_1.expect)((0, nodeSessionMetrics_1.parseAfter)(bad), String(bad)).to.equal(null);
+        }
+    });
+    it('exists only on a node', () => __awaiter(void 0, void 0, void 0, function* () {
+        const mount = (hub) => {
+            const a = (0, express_1.default)();
+            const r = express_1.default.Router();
+            (0, nodeSessionMetrics_1.registerNodeSessionMetrics)(r, Object.assign(Object.assign({}, IPluginArgs_1.DefaultPluginArgs), { hub }));
+            a.use('/xenon/api', r);
+            return a;
+        };
+        const onHub = yield (0, supertest_1.default)(mount()).get('/xenon/api/node/sessions/s1/metrics');
+        const onNode = yield (0, supertest_1.default)(mount('http://127.0.0.1:1')).get('/xenon/api/node/sessions/s1/metrics');
+        (0, chai_1.expect)(onHub.status).to.equal(404);
+        (0, chai_1.expect)(onHub.headers[nodeMetrics_1.NODE_METRICS_HEADER]).to.equal(undefined);
+        (0, chai_1.expect)(onNode.headers[nodeMetrics_1.NODE_METRICS_HEADER]).to.equal('1');
+    }));
+});

package/lib/test/unit/selector-health-detail.spec.js

@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 require("reflect-metadata");
 const chai_1 = require("chai");
 const supertest_1 = __importDefault(require("supertest"));
+const prisma_1 = require("../../src/prisma");
 const scratch_database_1 = require("../helpers/scratch-database");
 const selector_health_fixture_1 = require("../helpers/selector-health-fixture");
 const selectorDetail_1 = require("../../src/services/selector-health/selectorDetail");
@@ -128,4 +129,60 @@ describe('GET /healing/selectors/detail (real queries)', function () {
         const b = (yield detail(selector_health_fixture_1.READ_ONLY_A, { strategy: 'xpath', selector: selector_health_fixture_1.SEL.hot })).body;
         (0, chai_1.expect)(b.canAct).to.equal(false);
     }));
+    it('reads each heal once without its session, and each session once', () => __awaiter(this, void 0, void 0, function* () {
+        // 45 heals of one selector: odd ones on the shared phone, even ones on
+        // team A's; two in three found the same fix.
+        const now = Date.now();
+        yield scratch.db.sessionLog.createMany({
+            data: Array.from({ length: 45 }, (_, i) => ({
+                session_id: i % 2 ? 'sh-s-shared-1' : 'sh-s-a-1',
+                command_name: 'findElement',
+                url: '/element',
+                method: 'POST',
+                title: 'Find element',
+                response: '{}',
+                is_healed: true,
+                original_strategy: 'id',
+                original_selector: 'com.acme:id/many',
+                healed_strategy: 'id',
+                healed_selector: i % 3 ? 'com.acme:id/many_v2' : 'com.acme:id/many_v3',
+                healing_tier: i % 2 ? 'LLM' : 'Fuzzy XML',
+                healing_confidence: i % 3 ? 0.6 : null,
+                duration: 100,
+                createdAt: new Date(now - (i + 1) * selector_health_fixture_1.HOUR),
+            })),
+        });
+        const res = yield detail(selector_health_fixture_1.ADMIN, { strategy: 'id', selector: 'com.acme:id/many', days: '30' });
+        (0, chai_1.expect)(res.status).to.equal(200);
+        // Attaching the session to each heal took most of a second for a selector
+        // healed 50,000 times; the heals' sessions are read once each instead.
+        const heals = prisma_1.prisma.sessionLog.findMany.args.map((a) => a[0]);
+        (0, chai_1.expect)(heals).to.have.length(1);
+        (0, chai_1.expect)(heals[0].select).to.not.have.property('session');
+        const sessions = yield Promise.all(prisma_1.prisma.session.findMany.returnValues);
+        (0, chai_1.expect)(sessions
+            .flat()
+            .map((x) => x.id)
+            .sort()).to.deep.equal(['sh-s-a-1', 'sh-s-shared-1']);
+        const b = res.body;
+        (0, chai_1.expect)(b).to.deep.include({ heals: 45, sessions: 2, timeSpentMs: 4500 });
+        (0, chai_1.expect)(b.firstHealedAt).to.equal(new Date(now - 45 * selector_health_fixture_1.HOUR).toISOString());
+        (0, chai_1.expect)(b.lastHealedAt).to.equal(new Date(now - selector_health_fixture_1.HOUR).toISOString());
+        (0, chai_1.expect)(b.daily.reduce((s, d) => s + d.heals, 0)).to.equal(45);
+        (0, chai_1.expect)(b.recent).to.have.length(20);
+        (0, chai_1.expect)(b.recent[0].at).to.equal(new Date(now - selector_health_fixture_1.HOUR).toISOString());
+        (0, chai_1.expect)(b.suggestions.map((x) => [x.selector, x.count])).to.deep.equal([
+            ['com.acme:id/many_v2', 30],
+            ['com.acme:id/many_v3', 15],
+        ]);
+        (0, chai_1.expect)(b.suggestions[0].methods).to.have.members(['LLM', 'Fuzzy XML']);
+        (0, chai_1.expect)(b.suggestions[0].averageConfidence).to.be.closeTo(0.6, 1e-9);
+        (0, chai_1.expect)(b.suggestions[1].averageConfidence).to.equal(null);
+        (0, chai_1.expect)(b.platforms).to.deep.equal([{ name: 'android', count: 45 }]);
+        (0, chai_1.expect)(b.builds).to.deep.equal([{ id: selector_health_fixture_1.BUILD.id, name: selector_health_fixture_1.BUILD.name, count: 45 }]);
+        (0, chai_1.expect)(b.devices).to.deep.equal([
+            { udid: selector_health_fixture_1.PHONE.a, name: 'Team A Galaxy', count: 23 },
+            { udid: selector_health_fixture_1.PHONE.shared, name: 'Shared Pixel', count: 22 },
+        ]);
+    }));
 });

package/lib/test/unit/selector-verification-job.spec.js

@@ -41,6 +41,9 @@ describe('SelectorVerificationJob.run', () => {
                 update: sinon_1.default.stub().resolves({}),
             },
             selectorEvent: { create: sinon_1.default.stub().resolves({}) },
+            // No heal since any selector was marked fixed: see
+            // selector-verification-missed-regression.spec.ts for one.
+            sessionLog: { findFirst: sinon_1.default.stub().resolves(null) },
             $queryRaw: sinon_1.default.stub(),
         };
         prismaStub.$transaction = sinon_1.default

package/lib/test/unit/selector-verification-missed-regression.spec.js

@@ -0,0 +1,132 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const scratch_database_1 = require("../helpers/scratch-database");
+const SelectorVerificationJob_1 = require("../../src/services/SelectorVerificationJob");
+const HOUR = 60 * 60 * 1000;
+/**
+ * A heal of a selector someone marked fixed turns it back to "To fix" at once
+ * (SelectorStateService.onHealRecorded, called after the heal is written). That
+ * call can fail, say on a busy database, and nothing tried again: the selector
+ * stayed "being verified", and was even promoted to fixed once three other
+ * builds ran clean. The verification job now catches such a heal.
+ */
+describe('SelectorVerificationJob catches a heal the live check missed', function () {
+    this.timeout(90000);
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let socket;
+    const now = Date.now();
+    const fixedAt = new Date(now - 48 * HOUR);
+    before(() => __awaiter(this, void 0, void 0, function* () {
+        for (const b of ['mr-b1', 'mr-b2', 'mr-b3', 'mr-b4']) {
+            yield scratch.db.build.create({ data: { id: b, name: b } });
+            yield scratch.db.session.create({
+                data: {
+                    id: `mr-s-${b}`,
+                    build_id: b,
+                    device_udid: 'mr-phone',
+                    device_platform: 'android',
+                    device_version: '14',
+                    desired_capabilities: '{}',
+                    session_capabilities: '{}',
+                    node_id: 'localhost',
+                    has_live_video: false,
+                },
+            });
+        }
+    }));
+    beforeEach(() => __awaiter(this, void 0, void 0, function* () {
+        yield scratch.db.selectorEvent.deleteMany();
+        yield scratch.db.selectorState.deleteMany();
+        yield scratch.db.sessionLog.deleteMany();
+        socket = { emitToDashboard: sinon_1.default.stub() };
+    }));
+    const find = (build, selector, healed, at) => scratch.db.sessionLog.create({
+        data: {
+            session_id: `mr-s-${build}`,
+            command_name: 'findElement',
+            url: '/element',
+            method: 'POST',
+            title: 'Find element',
+            response: '{}',
+            is_healed: healed,
+            original_strategy: 'xpath',
+            original_selector: selector,
+            createdAt: new Date(at),
+        },
+    });
+    const state = (selector, status) => scratch.db.selectorState.create({
+        data: {
+            original_strategy: 'xpath',
+            original_selector: selector,
+            status,
+            fixed_at: fixedAt,
+            resolved_at: status === 'resolved' ? new Date(now - 24 * HOUR) : null,
+            clean_builds_count: status === 'resolved' ? 3 : 0,
+        },
+    });
+    const run = () => new SelectorVerificationJob_1.SelectorVerificationJob(scratch.db, socket).run();
+    const rowOf = (selector) => scratch.db.selectorState.findUniqueOrThrow({
+        where: {
+            original_strategy_original_selector: {
+                original_strategy: 'xpath',
+                original_selector: selector,
+            },
+        },
+    });
+    const actionsOf = (selector) => __awaiter(this, void 0, void 0, function* () {
+        return (yield scratch.db.selectorEvent.findMany({
+            where: { original_selector: selector },
+            orderBy: { createdAt: 'asc' },
+        })).map((e) => e.action);
+    });
+    it('sends a selector being verified back to fix, rather than promoting it', () => __awaiter(this, void 0, void 0, function* () {
+        yield state('//pending/broke', 'pending');
+        // Three clean builds since it was marked fixed, and one that healed.
+        for (const b of ['mr-b1', 'mr-b2', 'mr-b3'])
+            yield find(b, '//pending/broke', false, now - HOUR);
+        yield find('mr-b4', '//pending/broke', true, now - 2 * HOUR);
+        yield run();
+        const row = yield rowOf('//pending/broke');
+        (0, chai_1.expect)(row).to.include({ status: 'active', regression_count: 1, clean_builds_count: 0 });
+        (0, chai_1.expect)(row.fixed_at).to.equal(null);
+        (0, chai_1.expect)(yield actionsOf('//pending/broke')).to.deep.equal(['broke_again']);
+        (0, chai_1.expect)(socket.emitToDashboard.firstCall.args[0]).to.equal('selector_regressed');
+    }));
+    it('sends a fixed selector back to fix', () => __awaiter(this, void 0, void 0, function* () {
+        yield state('//resolved/broke', 'resolved');
+        yield find('mr-b1', '//resolved/broke', true, now - HOUR);
+        yield run();
+        const row = yield rowOf('//resolved/broke');
+        (0, chai_1.expect)(row).to.include({ status: 'active', regression_count: 1 });
+        (0, chai_1.expect)(row.resolved_at).to.equal(null);
+        (0, chai_1.expect)(yield actionsOf('//resolved/broke')).to.deep.equal(['broke_again']);
+    }));
+    it('leaves alone a selector that healed only before it was marked fixed', () => __awaiter(this, void 0, void 0, function* () {
+        yield state('//pending/clean', 'pending');
+        yield state('//resolved/clean', 'resolved');
+        yield find('mr-b1', '//pending/clean', true, fixedAt.getTime() - HOUR);
+        yield find('mr-b1', '//resolved/clean', true, fixedAt.getTime() - HOUR);
+        for (const b of ['mr-b1', 'mr-b2', 'mr-b3'])
+            yield find(b, '//pending/clean', false, now - HOUR);
+        yield run();
+        (0, chai_1.expect)((yield rowOf('//pending/clean')).status).to.equal('resolved');
+        (0, chai_1.expect)((yield rowOf('//resolved/clean')).status).to.equal('resolved');
+        (0, chai_1.expect)(yield actionsOf('//pending/clean')).to.deep.equal(['verified']);
+        (0, chai_1.expect)(yield actionsOf('//resolved/clean')).to.deep.equal([]);
+    }));
+});