@xenon-device-management/xenon 2.14.0 → 2.16.0

package/lib/test/unit/ai-engine-settings.spec.js

@@ -0,0 +1,338 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const express_1 = __importDefault(require("express"));
+const sinon_1 = __importDefault(require("sinon"));
+const typedi_1 = require("typedi");
+const loopbackRequest_1 = __importDefault(require("../helpers/loopbackRequest"));
+const dashboard_1 = __importDefault(require("../../src/app/routers/dashboard"));
+const config_1 = require("../../src/config");
+const web_config_service_1 = require("../../src/data-service/web-config-service");
+const aiEngineSettings_1 = require("../../src/services/settings/aiEngineSettings");
+const logger_1 = __importDefault(require("../../src/logger"));
+const container_registration_1 = require("../helpers/container-registration");
+const scratch_database_1 = require("../helpers/scratch-database");
+/**
+ * The AI engine page's choice of provider was kept in memory only: `POST
+ * /config` changed the running server's settings and saved nothing, so a
+ * restart went back to the provider the server was started with. The model and
+ * base URL the same route accepts went the same way.
+ *
+ * They follow the other lab settings now: a value saved in the dashboard wins
+ * over the plugin option or environment variable the server started with, which
+ * wins over the default, and it applies to the next AI call without a restart.
+ * API keys are never saved or sent: they stay in the server's environment.
+ */
+describe('the AI engine settings', () => {
+    const scratch = (0, scratch_database_1.useScratchDatabase)();
+    const AI_FIELDS = [
+        'aiProvider',
+        'aiModel',
+        'aiBaseUrl',
+        'geminiModel',
+        'openaiModel',
+        'anthropicModel',
+        'ollamaModel',
+        'geminiApiKey',
+        'openaiApiKey',
+        'anthropicApiKey',
+    ];
+    let startedWith;
+    let restoreContainer;
+    /** The AI settings the server was started with (its option, else its environment). */
+    const startup = (over = {}) => {
+        const blank = {};
+        for (const field of AI_FIELDS)
+            blank[field] = undefined;
+        Object.assign(config_1.config, blank, { aiProvider: 'gemini' }, over);
+    };
+    /** A server process starting: nothing in memory, the settings read from the database. */
+    const restart = () => __awaiter(void 0, void 0, void 0, function* () {
+        typedi_1.Container.set(aiEngineSettings_1.AiEngineSettings, new aiEngineSettings_1.AiEngineSettings());
+        yield typedi_1.Container.get(aiEngineSettings_1.AiEngineSettings).load();
+    });
+    beforeEach(() => __awaiter(void 0, void 0, void 0, function* () {
+        startedWith = {};
+        for (const field of AI_FIELDS)
+            startedWith[field] = config_1.config[field];
+        restoreContainer = (0, container_registration_1.saveRegistrations)(aiEngineSettings_1.AiEngineSettings);
+        // A service of its own, as at boot, so nothing saved leaks in from another spec.
+        typedi_1.Container.set(aiEngineSettings_1.AiEngineSettings, new aiEngineSettings_1.AiEngineSettings());
+        yield scratch.db.webConfig.deleteMany({});
+    }));
+    afterEach(() => {
+        Object.assign(config_1.config, startedWith);
+        restoreContainer();
+        sinon_1.default.restore();
+    });
+    function app() {
+        const a = (0, express_1.default)();
+        a.use(express_1.default.json());
+        a.use((req, _res, next) => {
+            req.auth = {
+                kind: 'user-session',
+                userId: 'usr_x',
+                role: 'SUPER_ADMIN',
+                scopes: 'admin,devices,sessions,read',
+                rateLimit: 1000,
+            };
+            next();
+        });
+        dashboard_1.default.register(a);
+        return a;
+    }
+    const save = (body) => (0, loopbackRequest_1.default)(app()).post('/config').send(body).timeout(5000);
+    const read = () => (0, loopbackRequest_1.default)(app()).get('/config').timeout(5000);
+    describe('the provider chosen on the page', () => {
+        it('is saved, and shown by GET /config', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ anthropicApiKey: 'sk-ant-test' });
+            const saved = yield save({ aiProvider: 'anthropic' });
+            (0, chai_1.expect)(saved.status).to.equal(200);
+            (0, chai_1.expect)(yield typedi_1.Container.get(web_config_service_1.WebConfigService).getConfig()).to.deep.equal({
+                aiProvider: 'anthropic',
+            });
+            (0, chai_1.expect)((yield read()).body.aiProvider).to.equal('anthropic');
+        }));
+        it('applies to the running server at once, with no restart', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup();
+            yield save({ aiProvider: 'openai' });
+            (0, chai_1.expect)(config_1.config.aiProvider).to.equal('openai');
+        }));
+        it('is still the provider after a restart', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup();
+            yield save({ aiProvider: 'ollama' });
+            startup(); // the process starts again with its option or environment: gemini
+            yield restart();
+            (0, chai_1.expect)(config_1.config.aiProvider).to.equal('ollama');
+            (0, chai_1.expect)((yield read()).body.aiProvider).to.equal('ollama');
+        }));
+        it('wins over the provider the server was started with', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ aiProvider: 'openai' });
+            yield save({ aiProvider: 'anthropic' });
+            startup({ aiProvider: 'openai' });
+            yield restart();
+            (0, chai_1.expect)(config_1.config.aiProvider).to.equal('anthropic');
+        }));
+        it('leaves the started-with provider in charge when nothing was saved', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ aiProvider: 'anthropic' });
+            yield restart();
+            (0, chai_1.expect)(config_1.config.aiProvider).to.equal('anthropic');
+            (0, chai_1.expect)((yield read()).body.aiProvider).to.equal('anthropic');
+        }));
+        it('goes back to the started-with provider when the saved one is cleared', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ aiProvider: 'openai' });
+            yield save({ aiProvider: 'anthropic' });
+            yield save({ aiProvider: '' });
+            (0, chai_1.expect)(config_1.config.aiProvider).to.equal('openai');
+            startup({ aiProvider: 'openai' });
+            yield restart();
+            (0, chai_1.expect)(config_1.config.aiProvider).to.equal('openai');
+        }));
+        it('is saved without touching the other settings', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup();
+            yield save({ buildCleanupDays: 7 });
+            yield save({ aiProvider: 'openai' });
+            (0, chai_1.expect)(yield typedi_1.Container.get(web_config_service_1.WebConfigService).getConfig()).to.deep.equal({
+                buildCleanupDays: 7,
+                aiProvider: 'openai',
+            });
+        }));
+    });
+    describe('the models and base URL', () => {
+        it('are saved, applied at once, and kept across a restart', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup();
+            const saved = yield save({
+                ollamaModel: 'llava',
+                aiBaseUrl: 'http://gpu-box.lab:11434',
+                geminiModel: 'gemini-2.5-pro',
+                openaiModel: 'gpt-4.1',
+                anthropicModel: 'claude-opus-4-1',
+                aiModel: 'llava:13b',
+            });
+            (0, chai_1.expect)(saved.status).to.equal(200);
+            (0, chai_1.expect)(config_1.config.ollamaModel).to.equal('llava');
+            (0, chai_1.expect)(config_1.config.aiBaseUrl).to.equal('http://gpu-box.lab:11434');
+            startup();
+            yield restart();
+            (0, chai_1.expect)(config_1.config).to.include({
+                ollamaModel: 'llava',
+                aiBaseUrl: 'http://gpu-box.lab:11434',
+                geminiModel: 'gemini-2.5-pro',
+                openaiModel: 'gpt-4.1',
+                anthropicModel: 'claude-opus-4-1',
+                aiModel: 'llava:13b',
+            });
+            const shown = (yield read()).body;
+            (0, chai_1.expect)(shown).to.include({ ollamaModel: 'llava', aiBaseUrl: 'http://gpu-box.lab:11434' });
+        }));
+        it('win over the values the server was started with', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ ollamaModel: 'llama3', aiBaseUrl: 'http://localhost:11434' });
+            yield save({ ollamaModel: 'llava', aiBaseUrl: 'http://gpu-box.lab:11434' });
+            startup({ ollamaModel: 'llama3', aiBaseUrl: 'http://localhost:11434' });
+            yield restart();
+            (0, chai_1.expect)(config_1.config.ollamaModel).to.equal('llava');
+            (0, chai_1.expect)(config_1.config.aiBaseUrl).to.equal('http://gpu-box.lab:11434');
+        }));
+        it('go back to the started-with values when cleared with an empty value', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ ollamaModel: 'llama3', aiBaseUrl: 'http://localhost:11434' });
+            yield save({ ollamaModel: 'llava', aiBaseUrl: 'http://gpu-box.lab:11434' });
+            yield save({ ollamaModel: '', aiBaseUrl: '' });
+            (0, chai_1.expect)(config_1.config.ollamaModel).to.equal('llama3');
+            (0, chai_1.expect)(config_1.config.aiBaseUrl).to.equal('http://localhost:11434');
+        }));
+        it('go back to no value at all when cleared and the server was started with none', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup();
+            yield save({ geminiModel: 'gemini-2.5-pro' });
+            yield save({ geminiModel: '' });
+            (0, chai_1.expect)(config_1.config.geminiModel).to.equal(undefined);
+            (0, chai_1.expect)((yield read()).body.geminiModel).to.equal(undefined);
+        }));
+        it('take surrounding spaces off a model name', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup();
+            yield save({ openaiModel: '  gpt-4.1  ' });
+            (0, chai_1.expect)(config_1.config.openaiModel).to.equal('gpt-4.1');
+        }));
+    });
+    describe('what is refused, with a 400 naming the setting and nothing saved', () => {
+        var _a;
+        const refused = [
+            ['aiProvider', 'gpt'],
+            ['aiProvider', 'Gemini'],
+            ['aiProvider', 1],
+            ['aiProvider', null],
+            ['ollamaModel', 7],
+            ['ollamaModel', 'llama 3'],
+            ['geminiModel', 'a'.repeat(201)],
+            ['aiModel', 'gpt-4o\nx'],
+            ['aiBaseUrl', 'localhost:11434'],
+            ['aiBaseUrl', 'ftp://models.lab/'],
+            ['aiBaseUrl', 'not a url'],
+            // A key in the address would be a key saved in the database.
+            ['aiBaseUrl', 'https://user:secret@llm.lab/v1'],
+            ['aiBaseUrl', 'https://llm.lab/v1?key=secret'],
+        ];
+        for (const [field, value] of refused) {
+            it(`${field}: ${(_a = JSON.stringify(value)) === null || _a === void 0 ? void 0 : _a.slice(0, 40)}`, () => __awaiter(void 0, void 0, void 0, function* () {
+                startup();
+                const answer = yield save({ [field]: value, buildCleanupDays: 9 });
+                (0, chai_1.expect)(answer.status).to.equal(400);
+                (0, chai_1.expect)(answer.body).to.include({ error: 'invalid_setting', field });
+                (0, chai_1.expect)(answer.body.message).to.be.a('string').and.not.equal('');
+                (0, chai_1.expect)(yield typedi_1.Container.get(web_config_service_1.WebConfigService).getConfig()).to.deep.equal({});
+                (0, chai_1.expect)(config_1.config.aiProvider).to.equal('gemini');
+            }));
+        }
+    });
+    describe('API keys', () => {
+        it('are never saved, and never change the running server', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ geminiApiKey: 'server-key' });
+            const answer = yield save({
+                aiProvider: 'gemini',
+                geminiApiKey: 'page-key',
+                openaiApiKey: 'page-key',
+                anthropicApiKey: 'page-key',
+            });
+            (0, chai_1.expect)(answer.status).to.equal(200);
+            (0, chai_1.expect)(config_1.config.geminiApiKey).to.equal('server-key');
+            (0, chai_1.expect)(config_1.config.openaiApiKey).to.equal(undefined);
+            const rows = yield scratch.db.webConfig.findMany();
+            (0, chai_1.expect)(rows.map((row) => row.name)).to.deep.equal(['aiProvider']);
+            (0, chai_1.expect)(JSON.stringify(rows)).not.to.include('page-key');
+        }));
+        it('are never sent from the started-with base URL either', () => __awaiter(void 0, void 0, void 0, function* () {
+            // A key in the address: POST /config refuses one, but the environment
+            // can still hold it. GET /config is open to every admin.
+            startup({
+                aiProvider: 'openai',
+                aiBaseUrl: 'https://lab:pw-secret@llm.lab/v1?key=key-secret&region=eu',
+            });
+            const shown = (yield read()).body.aiBaseUrl;
+            (0, chai_1.expect)(shown).not.to.include('pw-secret');
+            (0, chai_1.expect)(shown).not.to.include('key-secret');
+            (0, chai_1.expect)(shown).to.include('llm.lab/v1');
+            // The server still uses the address as it was given.
+            (0, chai_1.expect)(config_1.config.aiBaseUrl).to.equal('https://lab:pw-secret@llm.lab/v1?key=key-secret&region=eu');
+        }));
+        it('are never sent, only whether one is set', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ geminiApiKey: 'server-key', aiProvider: 'gemini' });
+            const shown = (yield read()).body;
+            (0, chai_1.expect)(JSON.stringify(shown)).not.to.include('server-key');
+            (0, chai_1.expect)(shown).to.include({ geminiSet: true, openaiSet: false, anthropicSet: false });
+        }));
+    });
+    describe('the model parameters the page used to show', () => {
+        it('are not sent: temperature, max tokens and top P were never used by any AI call', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup();
+            yield save({ aiTemperature: 0.2, aiMaxTokens: 1024, aiTopP: 0.5 });
+            const shown = (yield read()).body;
+            (0, chai_1.expect)(shown).not.to.have.any.keys('aiTemperature', 'aiMaxTokens', 'aiTopP');
+            (0, chai_1.expect)(yield typedi_1.Container.get(web_config_service_1.WebConfigService).getConfig()).to.deep.equal({});
+        }));
+    });
+    describe('at boot', () => {
+        it('a database that cannot be read leaves the started-with settings in charge, said once', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ aiProvider: 'anthropic' });
+            sinon_1.default.stub(web_config_service_1.WebConfigService.prototype, 'getConfig').rejects(new Error('disk I/O error'));
+            const warn = sinon_1.default.stub(logger_1.default, 'warn');
+            yield restart();
+            (0, chai_1.expect)(config_1.config.aiProvider).to.equal('anthropic');
+            (0, chai_1.expect)(warn.callCount).to.equal(1);
+            (0, chai_1.expect)(String(warn.firstCall.args[0])).to.include('disk I/O error');
+        }));
+        it('a saved value that could not work is skipped, not applied', () => __awaiter(void 0, void 0, void 0, function* () {
+            startup({ aiProvider: 'openai' });
+            // Written by hand: POST /config refuses it.
+            yield scratch.db.webConfig.create({
+                data: { id: 'aiProvider', name: 'aiProvider', value: 'gpt' },
+            });
+            yield restart();
+            (0, chai_1.expect)(config_1.config.aiProvider).to.equal('openai');
+        }));
+    });
+    describe('effectiveAiEngine', () => {
+        it('takes a saved value over the startup one, field by field', () => {
+            (0, chai_1.expect)((0, aiEngineSettings_1.effectiveAiEngine)({ aiProvider: 'gemini', ollamaModel: 'llama3', aiBaseUrl: 'http://a:1' }, { aiProvider: 'ollama', aiBaseUrl: 'http://b:2' })).to.deep.equal({
+                aiProvider: 'ollama',
+                aiModel: undefined,
+                aiBaseUrl: 'http://b:2',
+                geminiModel: undefined,
+                openaiModel: undefined,
+                anthropicModel: undefined,
+                ollamaModel: 'llama3',
+            });
+        });
+        it('falls back to gemini when neither names a provider', () => {
+            (0, chai_1.expect)((0, aiEngineSettings_1.effectiveAiEngine)({}, {}).aiProvider).to.equal('gemini');
+            (0, chai_1.expect)((0, aiEngineSettings_1.effectiveAiEngine)({}, { aiProvider: '' }).aiProvider).to.equal('gemini');
+        });
+        it("keeps a started-with provider it doesn't know, rather than pick another for it", () => {
+            // AIService then runs with no provider, and says so; a fallback to gemini
+            // would send screenshots somewhere the server's owner didn't choose.
+            (0, chai_1.expect)((0, aiEngineSettings_1.effectiveAiEngine)({ aiProvider: 'claude' }, {}).aiProvider).to.equal('claude');
+        });
+        it('names every setting it decides, and no key', () => {
+            (0, chai_1.expect)([...aiEngineSettings_1.AI_ENGINE_SETTINGS].sort()).to.deep.equal([
+                'aiBaseUrl',
+                'aiModel',
+                'aiProvider',
+                'anthropicModel',
+                'geminiModel',
+                'ollamaModel',
+                'openaiModel',
+            ].sort());
+        });
+    });
+});

package/lib/test/unit/ai-provider-switch.spec.js

@@ -0,0 +1,130 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const sinon_1 = __importDefault(require("sinon"));
+const config_1 = require("../../src/config");
+const AIService_1 = require("../../src/services/AIService");
+const LlmHealingProvider_1 = require("../../src/services/healing/LlmHealingProvider");
+const VisualAiHealingProvider_1 = require("../../src/services/healing/VisualAiHealingProvider");
+const fake_ai_provider_1 = require("../helpers/fake-ai-provider");
+/**
+ * The AI engine page changes the provider while the server runs (`POST
+ * /config` writes `config.aiProvider` and the models). The AI service set its
+ * provider up when it was created, and again only inside its own calls, while
+ * failure analysis and the LLM and visual healing tiers first asked
+ * `isEnabled()`, which read what was set up last. A server started with a
+ * provider that had no key never turned those on when a configured one was
+ * chosen; one started with a configured provider kept using it after a
+ * provider with no key was chosen.
+ */
+describe('AI provider chosen while the server runs', () => {
+    const ai = (0, fake_ai_provider_1.useFakeAiProviders)();
+    const SCREEN = 'c2NyZWVuc2hvdA==';
+    const FAILURE = {
+        sessionId: 'sess-switch',
+        failureReason: 'An element could not be located on the page',
+        commandLogs: [],
+        deviceLogs: [],
+    };
+    afterEach(() => sinon_1.default.restore());
+    it('turns on when a provider with a key is chosen after one without', () => {
+        ai.use('gemini', { key: false });
+        const service = new AIService_1.AIService();
+        (0, chai_1.expect)(service.isEnabled()).to.equal(false);
+        ai.use('anthropic');
+        (0, chai_1.expect)(service.isEnabled()).to.equal(true);
+    });
+    it('turns off when a provider without a key is chosen, and stops asking the old one', () => __awaiter(void 0, void 0, void 0, function* () {
+        ai.use('openai');
+        ai.answer({ text: 'Root Cause: from OpenAI' });
+        const service = new AIService_1.AIService();
+        (0, chai_1.expect)(service.isEnabled()).to.equal(true);
+        ai.use('anthropic', { key: false });
+        (0, chai_1.expect)(service.isEnabled()).to.equal(false);
+        (0, chai_1.expect)(yield service.analyzeFailure(FAILURE)).to.equal(null);
+        (0, chai_1.expect)(ai.calls).to.have.length(0);
+    }));
+    it('asks the model chosen last', () => __awaiter(void 0, void 0, void 0, function* () {
+        ai.use('openai');
+        ai.answer({ text: 'Root Cause: x' });
+        const service = new AIService_1.AIService();
+        yield service.analyzeFailure(FAILURE);
+        config_1.config.openaiModel = 'test-openai-chosen-later';
+        yield service.analyzeFailure(FAILURE);
+        (0, chai_1.expect)(JSON.parse(ai.calls[1].body).model).to.equal('test-openai-chosen-later');
+    }));
+    it('a provider set only by the plugin options is on at the first heal', () => __awaiter(void 0, void 0, void 0, function* () {
+        // AI_SERVICE is made when the plugin loads, before ServerManager applies
+        // the plugin's AI options (syncDatabaseAndAIConfig, through updateConfig).
+        (0, chai_1.expect)(AIService_1.AI_SERVICE.isEnabled()).to.equal(false);
+        (0, config_1.updateConfig)({
+            aiProvider: 'ollama',
+            aiBaseUrl: fake_ai_provider_1.OLLAMA_URL,
+            ollamaModel: `test-ollama-${Math.random().toString(36).slice(2)}`,
+        });
+        ai.answer({ text: '{"x": 120, "y": 340}' });
+        const healed = yield new VisualAiHealingProvider_1.VisualAiHealingProvider().heal({
+            sessionId: 'sess-switch',
+            driver: {},
+            strategy: 'id',
+            selector: 'login',
+            screenshotBase64: SCREEN,
+        });
+        (0, chai_1.expect)(ai.calls).to.have.length(1);
+        (0, chai_1.expect)(healed === null || healed === void 0 ? void 0 : healed.rect).to.deep.equal({ x: 100, y: 320, width: 40, height: 40 });
+    }));
+    describe('the healing tiers', () => {
+        beforeEach(() => {
+            // The server started with a provider that has no key.
+            ai.use('gemini', { key: false });
+            (0, chai_1.expect)(AIService_1.AI_SERVICE.isEnabled()).to.equal(false);
+            // Then a configured one is chosen on the AI engine page.
+            ai.use('anthropic');
+        });
+        it('the LLM tier asks the provider chosen on the AI engine page', () => __awaiter(void 0, void 0, void 0, function* () {
+            ai.answer({
+                text: '{"recommendedXpath": "//button[@text=\'Log in\']", "reason": "same text"}',
+            });
+            const driver = { findElement: sinon_1.default.stub().resolves({ ELEMENT: 'el-1' }) };
+            const healed = yield new LlmHealingProvider_1.LlmHealingProvider().heal({
+                sessionId: 'sess-switch',
+                driver,
+                strategy: 'id',
+                selector: 'login',
+                pageSource: '<hierarchy/>',
+                screenshotBase64: SCREEN,
+            });
+            (0, chai_1.expect)(ai.calls).to.have.length(1);
+            (0, chai_1.expect)(healed === null || healed === void 0 ? void 0 : healed.id).to.equal('el-1');
+            (0, chai_1.expect)(driver.findElement.firstCall.args).to.deep.equal([
+                'xpath',
+                "//button[@text='Log in']",
+            ]);
+        }));
+        it('the visual tier asks the provider chosen on the AI engine page', () => __awaiter(void 0, void 0, void 0, function* () {
+            ai.answer({ text: '{"x": 120, "y": 340}' });
+            const healed = yield new VisualAiHealingProvider_1.VisualAiHealingProvider().heal({
+                sessionId: 'sess-switch',
+                driver: {},
+                strategy: 'id',
+                selector: 'login',
+                screenshotBase64: SCREEN,
+            });
+            (0, chai_1.expect)(ai.calls).to.have.length(1);
+            (0, chai_1.expect)(healed === null || healed === void 0 ? void 0 : healed.rect).to.deep.equal({ x: 100, y: 320, width: 40, height: 40 });
+        }));
+    });
+});

package/lib/test/unit/ai-rate-limit.spec.js

@@ -0,0 +1,126 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+require("reflect-metadata");
+const chai_1 = require("chai");
+const AIService_1 = require("../../src/services/AIService");
+const CircuitBreaker_1 = require("../../src/services/CircuitBreaker");
+const fake_ai_provider_1 = require("../helpers/fake-ai-provider");
+/**
+ * A rate-limited provider (HTTP 429) is a failed call, whichever provider it
+ * is.
+ *
+ * Gemini's provider answered a 429 with the string
+ * 'CONNECTION_OK_RATE_LIMITED' instead of failing, so a failed session's AI
+ * analysis was saved as that string and shown on the session page, in the
+ * copied report and in bug reports. The circuit breaker counted it as a
+ * success, so it never stopped calling a provider that kept refusing. Only the
+ * visual assertion and the screen description checked for the string.
+ */
+describe('AI provider rate limits', () => {
+    const ai = (0, fake_ai_provider_1.useFakeAiProviders)();
+    const FAILURE = {
+        sessionId: 'sess-rate-limit',
+        failureReason: 'An element could not be located on the page',
+        commandLogs: [],
+        deviceLogs: [],
+    };
+    const SCREEN = 'c2NyZWVuc2hvdA==';
+    const failureOf = (p) => p.then((value) => {
+        throw new Error(`expected a failure, got ${JSON.stringify(value)}`);
+    }, (e) => e);
+    for (const provider of fake_ai_provider_1.PROVIDERS) {
+        describe(provider, () => {
+            let service;
+            let model;
+            beforeEach(() => {
+                ({ model } = ai.use(provider));
+                ai.answer({ status: 429 });
+                service = new AIService_1.AIService();
+            });
+            it('gives no failure analysis', () => __awaiter(void 0, void 0, void 0, function* () {
+                (0, chai_1.expect)(yield service.analyzeFailure(FAILURE)).to.equal(null);
+                (0, chai_1.expect)(ai.calls).to.have.length(1);
+            }));
+            it('counts against the circuit breaker, which stops calling after five', () => __awaiter(void 0, void 0, void 0, function* () {
+                for (let i = 0; i < 6; i++)
+                    yield service.analyzeFailure(FAILURE);
+                (0, chai_1.expect)(ai.calls).to.have.length(5);
+                const breaker = CircuitBreaker_1.CIRCUIT_BREAKERS.snapshot().find((b) => b.key === `ai:${provider}:${model}`);
+                (0, chai_1.expect)(breaker === null || breaker === void 0 ? void 0 : breaker.state).to.equal('open');
+            }));
+            it('fails a visual assertion, saying the provider is rate-limited', () => __awaiter(void 0, void 0, void 0, function* () {
+                const err = yield failureOf(service.assertVisual(SCREEN, 'The cart is empty'));
+                (0, chai_1.expect)(err.message).to.match(/rate-limited/);
+                (0, chai_1.expect)(err.message).to.match(/not checked/);
+            }));
+            it('gives no screen description, saying the provider is rate-limited', () => __awaiter(void 0, void 0, void 0, function* () {
+                const err = yield failureOf(service.describeScreen(SCREEN));
+                (0, chai_1.expect)(err.message).to.match(/rate-limited/);
+            }));
+            it('fails a locator test, saying the provider is rate-limited', () => __awaiter(void 0, void 0, void 0, function* () {
+                const err = yield failureOf(service.visualFind(SCREEN, 'the cart icon', { throwOnError: true }));
+                (0, chai_1.expect)(err.message).to.match(/rate-limited/);
+            }));
+            it('finds nothing for a healing or an ai-icon search', () => __awaiter(void 0, void 0, void 0, function* () {
+                (0, chai_1.expect)(yield service.visualFind(SCREEN, 'the cart icon')).to.equal(null);
+                (0, chai_1.expect)(yield service.healLocator({ selector: '//x', strategy: 'xpath', xml: '<hierarchy/>' })).to.equal(null);
+            }));
+            it('says in plain words, once the breaker has opened, that it is waiting', () => __awaiter(void 0, void 0, void 0, function* () {
+                for (let i = 0; i < 5; i++)
+                    yield failureOf(service.describeScreen(SCREEN));
+                // The sixth isn't sent. Testers see this (Omni-Scan, Test locator).
+                const described = yield failureOf(service.describeScreen(SCREEN));
+                (0, chai_1.expect)(described.message).to.match(/has been failing/);
+                (0, chai_1.expect)(described.message).to.match(/again in \d+ s/);
+                (0, chai_1.expect)(described.message).not.to.match(/circuit|ai:/i);
+                const asserted = yield failureOf(service.assertVisual(SCREEN, 'The cart is empty'));
+                (0, chai_1.expect)(asserted.message).to.match(/has been failing/);
+                (0, chai_1.expect)(asserted.message).to.match(/not checked/);
+                (0, chai_1.expect)(asserted.message).not.to.match(/circuit|ai:/i);
+                (0, chai_1.expect)(ai.calls).to.have.length(5);
+            }));
+            it('fails the connection test, saying the provider is rate-limited', () => __awaiter(void 0, void 0, void 0, function* () {
+                const result = yield service.testConnection({});
+                (0, chai_1.expect)(result.success).to.equal(false);
+                (0, chai_1.expect)(result.message).to.match(/rate-limited or out of quota/);
+            }));
+        });
+    }
+    it('gemini: an error that only mentions 429 is not a rate limit', () => __awaiter(void 0, void 0, void 0, function* () {
+        ai.use('gemini');
+        ai.answer({
+            status: 400,
+            message: 'The input token count (1429384) exceeds the maximum number of tokens allowed.',
+        });
+        const service = new AIService_1.AIService();
+        const err = yield failureOf(service.describeScreen(SCREEN));
+        (0, chai_1.expect)(err.message).not.to.match(/rate-limited/);
+        // A 400 is the request's fault: the breaker doesn't count it, so all six are sent.
+        for (let i = 0; i < 5; i++)
+            yield service.analyzeFailure(FAILURE);
+        (0, chai_1.expect)(ai.calls).to.have.length(6);
+    }));
+    it('ollama: is asked again as soon as it is back', () => __awaiter(void 0, void 0, void 0, function* () {
+        ai.use('ollama');
+        ai.ollamaUp(false);
+        const service = new AIService_1.AIService();
+        (0, chai_1.expect)(yield service.analyzeFailure(FAILURE)).to.equal(null);
+        ai.ollamaUp(true);
+        ai.answer({ text: 'Root Cause: the app crashed.' });
+        (0, chai_1.expect)(yield service.analyzeFailure(FAILURE)).to.equal('Root Cause: the app crashed.');
+    }));
+    it("still gives a provider's real answer", () => __awaiter(void 0, void 0, void 0, function* () {
+        ai.use('gemini');
+        ai.answer({ text: 'Root Cause: the Login button was covered by a dialog.' });
+        (0, chai_1.expect)(yield new AIService_1.AIService().analyzeFailure(FAILURE)).to.equal('Root Cause: the Login button was covered by a dialog.');
+    }));
+});