@xenon-device-management/xenon 2.13.2 → 2.15.0

package/lib/test/unit/selector-learning-every-session.spec.js

@@ -0,0 +1,179 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const CommandInterceptor_1 = require("../../src/interceptors/CommandInterceptor");
+const HealEtalonService_1 = require("../../src/services/healing/HealEtalonService");
+const SelfHealingSwitch_1 = require("../../src/services/settings/SelfHealingSwitch");
+const SessionManager_1 = require("../../src/sessions/SessionManager");
+const event_manager_1 = require("../../src/dashboard/event-manager");
+const IPluginArgs_1 = require("../../src/interfaces/IPluginArgs");
+const waitFor_1 = require("../../src/services/autowait/waitFor");
+const container_registration_1 = require("../helpers/container-registration");
+const scratch_database_1 = require("../helpers/scratch-database");
+/**
+ * After a findElement finds its element, Xenon learns the selector's
+ * fingerprint (the element's attributes, position and path), which the
+ * Resilio and Fuzzy XML tiers heal it with later, on the server, without
+ * calling the AI provider.
+ *
+ * Through 2.14 it learnt only behind the dashboard's per-command record
+ * (`isHub && enableDashboard && SESSION_MANAGER.isValidSession`): never with
+ * `enableDashboard` off (the default), and never on a node. Those servers
+ * healed without fingerprints, so a broken selector went on to OCR, Visual AI
+ * and the LLM. Learning now follows the self-healing switch alone, on every
+ * session this server drives. The record itself is unchanged.
+ */
+describe('selector learning, on every session this server drives', () => {
+    (0, scratch_database_1.useScratchDatabase)();
+    let restoreContainer;
+    let etalons;
+    let learn;
+    let record;
+    beforeEach(() => {
+        restoreContainer = (0, container_registration_1.saveRegistrations)(HealEtalonService_1.HealEtalonService, SelfHealingSwitch_1.SelfHealingSwitch);
+        // The switch as at boot with nothing saved: it follows the options.
+        typedi_1.Container.set(SelfHealingSwitch_1.SelfHealingSwitch, new SelfHealingSwitch_1.SelfHealingSwitch());
+        etalons = {
+            getSignature: sinon_1.default.stub().resolves(null),
+            saveSignature: sinon_1.default.stub().resolves(),
+        };
+        typedi_1.Container.set(HealEtalonService_1.HealEtalonService, etalons);
+        learn = sinon_1.default.spy(CommandInterceptor_1.CommandInterceptor.prototype, 'triggerLearning');
+        record = sinon_1.default.stub(event_manager_1.DASHBORD_EVENT_MANAGER, 'afterSessionCommand').resolves();
+    });
+    afterEach(() => {
+        sinon_1.default.restore();
+        restoreContainer();
+    });
+    const options = (over = {}) => (Object.assign(Object.assign(Object.assign({}, IPluginArgs_1.DefaultPluginArgs), { enableDashboard: false }), over));
+    let sessions = 0;
+    /** A session's driver, with what learning reads from a found element. */
+    function sessionDriver(caps = {}) {
+        sessions += 1;
+        return {
+            sessionId: `learn-${sessions}-${Date.now()}`,
+            caps: Object.assign({ platformName: 'Android' }, caps),
+            // As UiAutomator2 answers: String(value), so "null" for an unset one.
+            getAttribute: (name, _id) => __awaiter(this, void 0, void 0, function* () { var _a; return String((_a = { 'resource-id': 'com.shop:id/pay', text: 'Pay now' }[name]) !== null && _a !== void 0 ? _a : null); }),
+            getElementRect: () => __awaiter(this, void 0, void 0, function* () { return ({ x: 40, y: 900, width: 300, height: 96 }); }),
+            getName: () => __awaiter(this, void 0, void 0, function* () { return 'android.widget.Button'; }),
+            getPageSource: () => __awaiter(this, void 0, void 0, function* () { return '<hierarchy><android.widget.Button text="Pay now" resource-id="com.shop:id/pay"/></hierarchy>'; }),
+        };
+    }
+    const PAY = ['id', 'com.shop:id/pay'];
+    const FOUND = { ELEMENT: 'el-1', 'element-6066-11e4-a52e-4f735466cecf': 'el-1' };
+    /** A findElement the driver answers with its element. */
+    function find(driver_1, pluginArgs_1, _a) {
+        return __awaiter(this, arguments, void 0, function* (driver, pluginArgs, { isHub, command = 'findElement' }) {
+            const response = command === 'findElements' ? [FOUND] : FOUND;
+            return typedi_1.Container.get(CommandInterceptor_1.CommandInterceptor).handle(() => __awaiter(this, void 0, void 0, function* () { return response; }), driver, command, [...PAY, driver.sessionId], pluginArgs, isHub);
+        });
+    }
+    /** Learning runs in the background, after the find has answered. */
+    function learnt() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const result = yield (0, waitFor_1.waitFor)(() => __awaiter(this, void 0, void 0, function* () { return (etalons.saveSignature.called ? true : null); }), {
+                timeoutMs: 2000,
+                intervalMs: 5,
+            });
+            return 'value' in result;
+        });
+    }
+    for (const [where, isHub] of [
+        ['a standalone server', true],
+        ["a node running its hub's session", false],
+    ]) {
+        it(`learns a found selector on ${where} with enableDashboard off`, () => __awaiter(void 0, void 0, void 0, function* () {
+            const driver = sessionDriver();
+            // The case: nothing registers it, as nothing registered it before.
+            (0, chai_1.expect)(SessionManager_1.SESSION_MANAGER.isValidSession(driver.sessionId)).to.equal(false);
+            const answer = yield find(driver, options(), { isHub });
+            (0, chai_1.expect)(answer).to.deep.equal(FOUND);
+            (0, chai_1.expect)(learn.calledOnce, 'learning started').to.equal(true);
+            (0, chai_1.expect)(yield learnt(), 'a fingerprint was saved').to.equal(true);
+            (0, chai_1.expect)(etalons.saveSignature.firstCall.args.slice(0, 2)).to.deep.equal(PAY);
+            // A fingerprint that says which element it is, which Fuzzy XML can use.
+            (0, chai_1.expect)(etalons.saveSignature.firstCall.args[2].attributes).to.deep.include({
+                name: 'resource-id',
+                value: 'com.shop:id/pay',
+            });
+            // The dashboard's record is still the dashboard's: nothing was recorded.
+            (0, chai_1.expect)(record.called).to.equal(false);
+        }));
+    }
+    it('learns a selector autowait found', () => __awaiter(void 0, void 0, void 0, function* () {
+        const driver = sessionDriver();
+        const pluginArgs = options({
+            autowait: {
+                enabled: true,
+                timeoutMs: 500,
+                intervalBetweenAttemptsMs: 10,
+                excludeEnabledCheck: [],
+            },
+        });
+        yield find(driver, pluginArgs, { isHub: true });
+        (0, chai_1.expect)(learn.calledOnce).to.equal(true);
+        (0, chai_1.expect)(yield learnt()).to.equal(true);
+    }));
+    it('learns once, and still records, where the dashboard records the session', () => __awaiter(void 0, void 0, void 0, function* () {
+        const driver = sessionDriver();
+        sinon_1.default.stub(SessionManager_1.SESSION_MANAGER, 'isValidSession').returns(true);
+        yield find(driver, options({ enableDashboard: true }), { isHub: true });
+        (0, chai_1.expect)(record.calledOnce).to.equal(true);
+        (0, chai_1.expect)(learn.calledOnce).to.equal(true);
+        (0, chai_1.expect)(yield learnt()).to.equal(true);
+    }));
+    it('learns nothing with self-healing off', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield find(sessionDriver(), options({ enableSelfHealing: false }), { isHub: true });
+        (0, chai_1.expect)(learn.called).to.equal(false);
+    }));
+    it('learns nothing from a session that turned its own healing off', () => __awaiter(void 0, void 0, void 0, function* () {
+        const driver = sessionDriver({ 'xe:options': { healingTiers: [] } });
+        yield find(driver, options(), { isHub: true });
+        (0, chai_1.expect)(learn.called).to.equal(false);
+    }));
+    it('still learns from a session that limits its tiers', () => __awaiter(void 0, void 0, void 0, function* () {
+        const driver = sessionDriver({ 'xe:options': { healingTiers: [3] } });
+        yield find(driver, options(), { isHub: true });
+        (0, chai_1.expect)(learn.calledOnce).to.equal(true);
+    }));
+    it('learns nothing from findElements, as before', () => __awaiter(void 0, void 0, void 0, function* () {
+        yield find(sessionDriver(), options(), { isHub: true, command: 'findElements' });
+        (0, chai_1.expect)(learn.called).to.equal(false);
+    }));
+    it('answers the find even when deciding whether to learn throws', () => __awaiter(void 0, void 0, void 0, function* () {
+        sinon_1.default.stub(SelfHealingSwitch_1.SelfHealingSwitch.prototype, 'isEnabled').throws(new Error('switch unreadable'));
+        const answer = yield find(sessionDriver(), options(), { isHub: true });
+        (0, chai_1.expect)(answer).to.deep.equal(FOUND);
+        (0, chai_1.expect)(learn.called).to.equal(false);
+    }));
+    it('answers the find even when learning fails', () => __awaiter(void 0, void 0, void 0, function* () {
+        etalons.getSignature.rejects(new Error('database is locked'));
+        const driver = sessionDriver();
+        const answer = yield find(driver, options(), { isHub: true });
+        (0, chai_1.expect)(answer).to.deep.equal(FOUND);
+        // The background pass ends, and the session may learn again.
+        const settled = yield (0, waitFor_1.waitFor)(() => __awaiter(void 0, void 0, void 0, function* () {
+            return typedi_1.Container.get(CommandInterceptor_1.CommandInterceptor).learningSessions.has(driver.sessionId)
+                ? null
+                : true;
+        }), { timeoutMs: 2000, intervalMs: 5 });
+        (0, chai_1.expect)('value' in settled).to.equal(true);
+        (0, chai_1.expect)(etalons.saveSignature.called).to.equal(false);
+    }));
+});

package/lib/test/unit/selector-verification-clean-build.spec.js

@@ -0,0 +1,130 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const scratch_database_1 = require("../helpers/scratch-database");
+const SelectorVerificationJob_1 = require("../../src/services/SelectorVerificationJob");
+const HOUR = 60 * 60 * 1000;
+const ELEMENT = '{"value":{"element-6066-11e4-a52e-4f735466cecf":"e-1"},"sessionId":"s"}';
+const ELEMENTS = '{"value":[{"element-6066-11e4-a52e-4f735466cecf":"e-1"}],"sessionId":"s"}';
+const EMPTY_LIST = '{"value":[],"sessionId":"s"}';
+const NOT_FOUND = '{"value":{"error":"An element could not be located on the page"},"sessionId":"s"}';
+/**
+ * A clean build is one where the selector was found without healing. Through
+ * 2.14 it was any build with a find of the selector that didn't heal, so a
+ * selector whose find failed outright in three builds was verified as fixed.
+ * The rows are the ones the dashboard writes (event-manager.afterSessionCommand):
+ * a failed find has `is_error` and an error body, an empty findElements has
+ * `{"value":[]}`, and the real SQL runs against a scratch database.
+ */
+describe('SelectorVerificationJob counts only builds where the selector was found', function () {
+    this.timeout(90000);
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    let socket;
+    const now = Date.now();
+    const fixedAt = new Date(now - 48 * HOUR);
+    const builds = ['cb-b1', 'cb-b2', 'cb-b3'];
+    before(() => __awaiter(this, void 0, void 0, function* () {
+        for (const b of builds) {
+            yield scratch.db.build.create({ data: { id: b, name: b } });
+            yield scratch.db.session.create({
+                data: {
+                    id: `cb-s-${b}`,
+                    build_id: b,
+                    device_udid: 'cb-phone',
+                    device_platform: 'android',
+                    device_version: '14',
+                    desired_capabilities: '{}',
+                    session_capabilities: '{}',
+                    node_id: 'localhost',
+                    has_live_video: false,
+                },
+            });
+        }
+    }));
+    beforeEach(() => __awaiter(this, void 0, void 0, function* () {
+        yield scratch.db.selectorEvent.deleteMany();
+        yield scratch.db.selectorState.deleteMany();
+        yield scratch.db.sessionLog.deleteMany();
+        socket = { emitToDashboard: sinon_1.default.stub() };
+        yield scratch.db.selectorState.create({
+            data: {
+                original_strategy: 'xpath',
+                original_selector: '//login',
+                status: 'pending',
+                fixed_at: fixedAt,
+                clean_builds_count: 0,
+            },
+        });
+    }));
+    const find = (build, command, response, isError = false) => scratch.db.sessionLog.create({
+        data: {
+            session_id: `cb-s-${build}`,
+            command_name: command,
+            url: command === 'findElement' ? '/findElement' : '/findElements',
+            method: 'POST',
+            title: 'Find element',
+            response,
+            is_success: !isError,
+            is_error: isError,
+            is_healed: false,
+            original_strategy: 'xpath',
+            original_selector: '//login',
+            createdAt: new Date(now - HOUR),
+        },
+    });
+    const run = () => new SelectorVerificationJob_1.SelectorVerificationJob(scratch.db, socket).run();
+    const row = () => scratch.db.selectorState.findUniqueOrThrow({
+        where: {
+            original_strategy_original_selector: {
+                original_strategy: 'xpath',
+                original_selector: '//login',
+            },
+        },
+    });
+    it('does not count builds where the find failed outright', () => __awaiter(this, void 0, void 0, function* () {
+        for (const b of builds)
+            yield find(b, 'findElement', NOT_FOUND, true);
+        yield run();
+        const state = yield row();
+        (0, chai_1.expect)(state.status).to.equal('pending');
+        (0, chai_1.expect)(state.clean_builds_count).to.equal(0);
+    }));
+    it('does not count builds where findElements found nothing', () => __awaiter(this, void 0, void 0, function* () {
+        for (const b of builds)
+            yield find(b, 'findElements', EMPTY_LIST);
+        yield run();
+        const state = yield row();
+        (0, chai_1.expect)(state.status).to.equal('pending');
+        (0, chai_1.expect)(state.clean_builds_count).to.equal(0);
+    }));
+    it('verifies the selector after three builds that found it', () => __awaiter(this, void 0, void 0, function* () {
+        yield find('cb-b1', 'findElement', ELEMENT);
+        yield find('cb-b2', 'findElements', ELEMENTS);
+        yield find('cb-b3', 'findElement', ELEMENT);
+        yield run();
+        (0, chai_1.expect)((yield row()).status).to.equal('resolved');
+    }));
+    it('counts a build that found it once, even if another find of it failed', () => __awaiter(this, void 0, void 0, function* () {
+        yield find('cb-b1', 'findElement', NOT_FOUND, true);
+        yield find('cb-b1', 'findElement', ELEMENT);
+        yield find('cb-b2', 'findElement', NOT_FOUND, true);
+        yield run();
+        const state = yield row();
+        (0, chai_1.expect)(state.status).to.equal('pending');
+        (0, chai_1.expect)(state.clean_builds_count).to.equal(1);
+    }));
+});

package/lib/test/unit/selector-verification-job.spec.js

@@ -65,9 +65,9 @@ describe('SelectorVerificationJob.run', () => {
             },
         ]);
         prismaStub.$queryRaw.resolves([
-            { build_id: 'b-1', healed: 0 },
-            { build_id: 'b-2', healed: 0 },
-            { build_id: 'b-3', healed: 0 },
+            { build_id: 'b-1', healed: 0, found: 1 },
+            { build_id: 'b-2', healed: 0, found: 1 },
+            { build_id: 'b-3', healed: 0, found: 1 },
         ]);
         prismaStub.selectorState.update.resolves({
             id: 'r-1',
@@ -99,8 +99,8 @@ describe('SelectorVerificationJob.run', () => {
             },
         ]);
         prismaStub.$queryRaw.resolves([
-            { build_id: 'b-1', healed: 0 },
-            { build_id: 'b-2', healed: 1 }, // not clean
+            { build_id: 'b-1', healed: 0, found: 1 },
+            { build_id: 'b-2', healed: 1, found: 1 }, // not clean
         ]);
         prismaStub.selectorState.update.resolves({
             id: 'r-1',
@@ -125,8 +125,8 @@ describe('SelectorVerificationJob.run', () => {
             },
         ]);
         prismaStub.$queryRaw.resolves([
-            { build_id: 'b-1', healed: 0 },
-            { build_id: 'b-2', healed: 1 },
+            { build_id: 'b-1', healed: 0, found: 1 },
+            { build_id: 'b-2', healed: 1, found: 1 },
         ]);
         yield job.run();
         (0, chai_1.expect)(prismaStub.selectorState.update.called).to.be.false;
@@ -183,9 +183,9 @@ describe('SelectorVerificationJob.run', () => {
         ]);
         prismaStub.$queryRaw.onFirstCall().rejects(new Error('boom'));
         prismaStub.$queryRaw.onSecondCall().resolves([
-            { build_id: 'b-1', healed: 0 },
-            { build_id: 'b-2', healed: 0 },
-            { build_id: 'b-3', healed: 0 },
+            { build_id: 'b-1', healed: 0, found: 1 },
+            { build_id: 'b-2', healed: 0, found: 1 },
+            { build_id: 'b-3', healed: 0, found: 1 },
         ]);
         prismaStub.selectorState.update.resolves({
             id: 'r-2',
@@ -222,9 +222,9 @@ describe('SelectorVerificationJob.run', () => {
             },
         ]);
         prismaStub.$queryRaw.resolves([
-            { build_id: 'b-1', healed: 0 },
-            { build_id: 'b-2', healed: 0 },
-            { build_id: 'b-3', healed: 0 },
+            { build_id: 'b-1', healed: 0, found: 1 },
+            { build_id: 'b-2', healed: 0, found: 1 },
+            { build_id: 'b-3', healed: 0, found: 1 },
         ]);
         prismaStub.selectorState.update.resolves({
             id: 'r-1',

package/lib/test/unit/self-healing-boot.spec.js

@@ -0,0 +1,80 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const plugin_1 = require("../../src/plugin");
+const ServerManager_1 = require("../../src/services/ServerManager");
+const PluginContext_1 = require("../../src/PluginContext");
+const SelfHealingSwitch_1 = require("../../src/services/settings/SelfHealingSwitch");
+const aiEngineSettings_1 = require("../../src/services/settings/aiEngineSettings");
+const logger_1 = __importDefault(require("../../src/logger"));
+/**
+ * The self-healing value saved on the Settings page, and the AI engine page's
+ * provider, model and base URL, are read once at boot, so a restart doesn't put
+ * the server back on the options it was started with. They have to be loaded
+ * once the database is ready (migrations ran), after the startup options were
+ * applied (the AI engine's are what a cleared value goes back to), and before
+ * the first route can answer or the first command arrive.
+ */
+describe('loading the saved self-healing and AI engine settings, at boot', () => {
+    let statics;
+    beforeEach(() => {
+        statics = {
+            nodeId: plugin_1.XenonPlugin.NODE_ID,
+            port: plugin_1.XenonPlugin.port,
+            basePath: plugin_1.XenonPlugin.nodeBasePath,
+        };
+    });
+    afterEach(() => {
+        sinon_1.default.restore();
+        plugin_1.XenonPlugin.NODE_ID = statics.nodeId;
+        plugin_1.XenonPlugin.port = statics.port;
+        plugin_1.XenonPlugin.nodeBasePath = statics.basePath;
+    });
+    it('runs once the database is ready, before routes are registered', () => __awaiter(void 0, void 0, void 0, function* () {
+        const steps = [];
+        const step = (name) => () => {
+            steps.push(name);
+            return Promise.resolve();
+        };
+        const boot = ServerManager_1.ServerManager.prototype;
+        sinon_1.default.stub(boot, 'resolvePluginArgs').resolves({});
+        sinon_1.default.stub(logger_1.default, 'banner');
+        sinon_1.default.stub(PluginContext_1.PluginContext.prototype, 'setContext');
+        sinon_1.default.stub(boot, 'syncDatabaseAndAIConfig').callsFake(step('database config'));
+        sinon_1.default.stub(boot, 'initializeCoreSubsystems').callsFake(step('database and migrations'));
+        // The real one kills every go-ios process on the machine.
+        sinon_1.default.stub(boot, 'reapLeftoverGoIos').callsFake(step('reap'));
+        sinon_1.default.stub(SelfHealingSwitch_1.SelfHealingSwitch.prototype, 'load').callsFake(step('self-healing setting'));
+        sinon_1.default.stub(aiEngineSettings_1.AiEngineSettings.prototype, 'load').callsFake(step('AI engine settings'));
+        sinon_1.default.stub(boot, 'registerRoutes').callsFake(() => {
+            steps.push('routes');
+            throw new Error('boot stopped by the test');
+        });
+        yield typedi_1.Container.get(ServerManager_1.ServerManager)
+            .updateServer({}, null, { port: 4723 })
+            .then(() => chai_1.expect.fail('the stubbed routes step should have stopped the boot'), (err) => (0, chai_1.expect)(err.message).to.equal('boot stopped by the test'));
+        (0, chai_1.expect)(steps).to.deep.equal([
+            'database config',
+            'database and migrations',
+            'reap',
+            'self-healing setting',
+            'AI engine settings',
+            'routes',
+        ]);
+    }));
+});