@xenon-device-management/xenon 2.14.0 → 2.15.0

package/lib/src/middleware/authMiddleware.js

@@ -24,7 +24,13 @@ const PluginContext_1 = require("../PluginContext");
 const hubSessionToken_1 = require("../gateway/hubSessionToken");
 const localDeviceHosts_1 = require("../device-managers/localDeviceHosts");
 const SESSION_COOKIE = 'xenon_dashboard_session';
-const SESSION_TTL_MS = 24 * 60 * 60 * 1000;
+/** An API key's expiresAt as epoch ms, or undefined for a key that never expires. */
+function keyExpiresAt(expiresAt) {
+    if (!expiresAt)
+        return undefined;
+    const ms = new Date(expiresAt).getTime();
+    return Number.isNaN(ms) ? undefined : ms;
+}
 // Role → scopes derivation. The same table is used in profile token creation
 // to enforce that members can never grant 'admin'.
 // Cookie-session scopes: a logged-in user gets the full set of scopes
@@ -185,6 +191,7 @@ function authMiddleware(req, res, next) {
                 apiKeyId: row.id,
                 rateLimit: row.rateLimit,
                 teamIds,
+                credentialExpiresAt: keyExpiresAt(row.expiresAt),
             };
             req.apiKey = { id: row.id, scopes: row.scopes, rateLimit: row.rateLimit, teamId: (_b = row.teamId) !== null && _b !== void 0 ? _b : null };
             return next();
@@ -224,6 +231,7 @@ function authMiddleware(req, res, next) {
                     teamId: (_e = payload.teamId) !== null && _e !== void 0 ? _e : null,
                     rateLimit: 300,
                     teamIds,
+                    credentialExpiresAt: typeof payload.exp === 'number' ? payload.exp * 1000 : undefined,
                 };
                 return next();
             }
@@ -242,11 +250,14 @@ function authMiddleware(req, res, next) {
                 if (!user || user.status !== 'ACTIVE')
                     return res.status(401).json({ error: 'invalid session' });
                 const isSecure = req.secure || req.headers['x-forwarded-proto'] === 'https';
+                // Renewed for as long as the session row now lasts
+                // (XENON_USER_SESSION_TTL_MS). A fixed 24 hours here capped any longer
+                // setting: the browser dropped the cookie after a day away.
                 res.cookie(SESSION_COOKIE, cookie, {
                     httpOnly: true,
                     secure: isSecure,
                     sameSite: 'strict',
-                    maxAge: SESSION_TTL_MS,
+                    maxAge: userSessionSvc.ttlMs(),
                 });
                 const teamIds = yield (0, callerTeamIds_1.computeTeamIds)({
                     role: user.role,
@@ -279,7 +290,7 @@ function authMiddleware(req, res, next) {
                     httpOnly: true,
                     secure: isSecure,
                     sameSite: 'strict',
-                    maxAge: SESSION_TTL_MS,
+                    maxAge: userSessionSvc.ttlMs(),
                 });
                 const teamIds = yield (0, callerTeamIds_1.computeTeamIds)({
                     role: user.role,
@@ -295,6 +306,7 @@ function authMiddleware(req, res, next) {
                     apiKeyId: row.id,
                     rateLimit: row.rateLimit,
                     teamIds,
+                    credentialExpiresAt: keyExpiresAt(row.expiresAt),
                 };
                 req.apiKey = { id: row.id, scopes: row.scopes, rateLimit: row.rateLimit, teamId: (_g = row.teamId) !== null && _g !== void 0 ? _g : null };
                 return next();

package/lib/src/middleware/commandCaller.js

@@ -103,9 +103,9 @@ class CommandCallerVerifier {
             if (!out)
                 return null;
             // A Bearer token carries the scopes of the credential that minted it
-            // (/auth/token), so it is judged like a key. Judging it by role alone,
-            // as a xenon-session token is for leases, would let an ADMIN's ordinary
-            // key mint its way to an override.
+            // (/auth/token), so it is judged like a key. Judging it by role alone
+            // would let an ADMIN's ordinary key mint its way to an override, as a
+            // xenon-session token's lease override did through 2.14.
             return {
                 caller: {
                     userId: out.user.id,

package/lib/src/middleware/verifyCredential.js

@@ -42,9 +42,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ACCEPTED_BEARER_AUDIENCES = void 0;
+exports.SessionTokenSubjectError = exports.ACCEPTED_BEARER_AUDIENCES = void 0;
 exports.verifyKeyPairCredential = verifyKeyPairCredential;
 exports.verifyBearerCredential = verifyBearerCredential;
+exports.isWrongSessionToken = isWrongSessionToken;
+exports.verifySessionTokenCredential = verifySessionTokenCredential;
 const jose = __importStar(require("jose"));
 const typedi_1 = require("typedi");
 const ApiKeyService_1 = require("../services/ApiKeyService");
@@ -96,6 +98,12 @@ function verifyKeyPairCredential(accessKey, token) {
  * JWT) means the token is wrong. Any other error (typically "JWT key service
  * not initialized") means it could not be checked, and is rethrown once no
  * audience has verified.
+ *
+ * The payload's `scopes` are the token's as of now: `admin` is dropped once
+ * its user is a MEMBER. A token's scopes are fixed when it is minted, for up
+ * to a day, and REST (resolveActor, scopeGuard) and per-command auth's
+ * override read them, so through 2.14 an admin demoted since kept admin
+ * powers until the token expired.
  */
 function verifyBearerCredential(token) {
     return __awaiter(this, void 0, void 0, function* () {
@@ -119,6 +127,55 @@ function verifyBearerCredential(token) {
         const user = yield typedi_1.Container.get(UserService_1.UserService).findById(String(payload.sub));
         if (!user || user.status !== 'ACTIVE')
             return null;
+        return { payload: Object.assign(Object.assign({}, payload), { scopes: scopesForLiveRole(payload.scopes, user.role) }), user };
+    });
+}
+/** A token's `scopes` claim without `admin` when its user is now a MEMBER. */
+function scopesForLiveRole(scopes, role) {
+    const list = String(scopes !== null && scopes !== void 0 ? scopes : '')
+        .split(',')
+        .map((s) => s.trim())
+        .filter(Boolean);
+    if (role === 'ADMIN' || role === 'SUPER_ADMIN')
+        return list.join(',');
+    return list.filter((s) => s !== 'admin').join(',');
+}
+/** The subject of a session token is no longer an ACTIVE user (deleted or Inactive). */
+class SessionTokenSubjectError extends Error {
+    constructor() {
+        super('the session token names a user who is deleted or not active');
+        this.name = 'SessionTokenSubjectError';
+    }
+}
+exports.SessionTokenSubjectError = SessionTokenSubjectError;
+/**
+ * Whether verifySessionTokenCredential's error means the token is wrong (a
+ * bad signature, audience or lifetime, not a JWT, or a departed user), as
+ * opposed to a check that could not run (the signing key or the database
+ * unavailable). A wrong token counts as none; a check that could not run
+ * refuses the create, as a key whose owner can't be looked up does.
+ */
+function isWrongSessionToken(err) {
+    return err instanceof jose.errors.JOSEError || err instanceof SessionTokenSubjectError;
+}
+/**
+ * A `xenon-session` token (the `xe:options.sessionToken` capability), checked
+ * as REST checks a Bearer token: a valid signature, audience and lifetime,
+ * and a subject who is an ACTIVE user, looked up now. Through 2.14 session
+ * create checked the signature alone, so a deleted or Inactive user's token
+ * created sessions as them until it expired.
+ *
+ * Throws for a token that does not verify, as JwtKeyService.verify does, so
+ * the session-token gate and attribution treat a wrong signature and a
+ * departed user alike. Whether the token may create sessions at all (its
+ * `scopes`) is the caller's to judge.
+ */
+function verifySessionTokenCredential(token) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const payload = yield typedi_1.Container.get(JwtKeyService_1.JwtKeyService).verify(token, { audience: 'xenon-session' });
+        const user = yield typedi_1.Container.get(UserService_1.UserService).findById(String(payload.sub));
+        if (!user || user.status !== 'ACTIVE')
+            throw new SessionTokenSubjectError();
         return { payload, user: user };
     });
 }

package/lib/src/plugin.js

@@ -66,6 +66,7 @@ const IPluginArgs_1 = require("./interfaces/IPluginArgs");
 const NodeDevices_1 = __importDefault(require("./device-managers/NodeDevices"));
 const appiumUmbrella_1 = require("./sessions/appiumUmbrella");
 const LiveSessionOwners_1 = require("./services/device-access/LiveSessionOwners");
+const sessionMemory_1 = require("./sessions/sessionMemory");
 const SessionMetricsService_1 = require("./services/metrics/SessionMetricsService");
 const PhoneNetworkRestore_1 = require("./services/network/PhoneNetworkRestore");
 const config_1 = require("./config");
@@ -73,6 +74,7 @@ const SessionManager_1 = require("./sessions/SessionManager");
 const event_manager_1 = require("./dashboard/event-manager");
 const asset_manager_1 = require("./dashboard/asset-manager");
 const session_service_1 = require("./dashboard/services/session-service");
+const shutdownReason_1 = require("./services/session/shutdownReason");
 const DEVICE_MANAGER_LOCK_NAME = 'DeviceManager';
 class XenonPlugin extends base_plugin_1.default {
     constructor(pluginName, cliArgs) {
@@ -104,11 +106,12 @@ class XenonPlugin extends base_plugin_1.default {
             return yield typedi_1.Container.get(CommandInterceptor_1.CommandInterceptor).handle(next, driver, commandName, args, this.pluginArgs, XenonPlugin.IS_HUB);
         });
     }
-    onUnexpectedShutdown(driver, _cause) {
+    onUnexpectedShutdown(driver, cause) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
             const sessionId = driver.sessionId;
             typedi_1.Container.get(LiveSessionOwners_1.LiveSessionOwners).forget(sessionId);
+            (0, sessionMemory_1.forgetSessionMemory)(sessionId);
             if (sessionId)
                 yield typedi_1.Container.get(SessionMetricsService_1.SessionMetricsService).stop(sessionId);
             // Appium's new-command timeout ends a session here, not in deleteSession:
@@ -136,7 +139,9 @@ class XenonPlugin extends base_plugin_1.default {
             logger_1.default.info(`Unblocking device mapped with filter ${JSON.stringify(deviceFilter)} onUnexpectedShutdown from server`);
             if (XenonPlugin.IS_HUB && this.pluginArgs.enableDashboard && sessionId) {
                 const sessionLog = this.xenonLog.withSession(sessionId, (_a = driver.caps) === null || _a === void 0 ? void 0 : _a.udid);
-                sessionLog.info('Unexpected shutdown for session, updating dashboard...');
+                // Appium's cause, so an idle session is filed as a timeout.
+                const reason = (0, shutdownReason_1.unexpectedShutdownReason)(cause);
+                sessionLog.info(`Unexpected shutdown for session (${reason}), updating dashboard...`);
                 const session = SessionManager_1.SESSION_MANAGER.getSession(sessionId);
                 if (session && session.isVideoRecordingInProgress()) {
                     try {
@@ -154,7 +159,7 @@ class XenonPlugin extends base_plugin_1.default {
                         logger_1.default.debug(`[${sessionId}] rescue-video: Failed: ${err.message}`);
                     }
                 }
-                yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(sessionId, SessionStatus_1.SessionStatus.FAILED, 'Driver shut down unexpectedly');
+                yield event_manager_1.DASHBORD_EVENT_MANAGER.onSessionStopped(sessionId, SessionStatus_1.SessionStatus.FAILED, reason);
                 typedi_1.Container.get(TracingService_1.TracingService).endSpan(sessionId, 'ERROR', {
                     'xenon.session.stop_reason': 'Unexpected shutdown',
                 });

package/lib/src/services/AICommandService.js

@@ -49,12 +49,19 @@ let AICommandService = class AICommandService {
         return __awaiter(this, void 0, void 0, function* () {
             this.logger.info(`Testing AI locator: ${locator.strategy}=${locator.selector}`);
             const omniService = typedi_1.Container.get(OmniVisionService_1.OmniVisionService);
+            // A session's own route: its test may act on what it found.
+            const keep = (found) => {
+                if (driver === null || driver === void 0 ? void 0 : driver.sessionId)
+                    for (const el of found)
+                        omniService.remember(driver.sessionId, el);
+                return found;
+            };
             if (locator.strategy === '-custom:ai-text') {
-                return yield omniService.findByText(driver, locator.selector);
+                return keep(yield omniService.findByText(driver, locator.selector));
             }
             else if (locator.strategy === '-custom:ai-icon') {
                 const result = yield omniService.findByIcon(driver, locator.selector);
-                return result ? [result] : [];
+                return keep(result ? [result] : []);
             }
             throw new Error(`Unsupported AI strategy: ${locator.strategy}`);
         });

package/lib/src/services/AIService.js

@@ -45,7 +45,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AI_SERVICE = exports.AIService = void 0;
+exports.AI_SERVICE = exports.AIService = exports.FAILURE_ANALYSIS_TIMEOUT_MS = exports.AI_CALL_TIMEOUT_MS = exports.AITimeoutError = exports.AIProviderPausedError = exports.AIRateLimitedError = void 0;
 exports.parseVisualVerdict = parseVisualVerdict;
 const generative_ai_1 = require("@google/generative-ai");
 const openai_1 = __importDefault(require("openai"));
@@ -56,8 +56,88 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const config_1 = require("../config");
 const CircuitBreaker_1 = require("./CircuitBreaker");
-/** What GeminiProvider answers, instead of throwing, when the provider is rate-limited (429). */
-const RATE_LIMITED = 'CONNECTION_OK_RATE_LIMITED';
+/**
+ * A provider refused a call for its rate limit or quota (HTTP 429), whichever
+ * provider it is. A failed call like any other: its caller gets no answer and
+ * the circuit breaker counts it (`status`). Through 2.14 Gemini's provider
+ * answered a 429 with the text 'CONNECTION_OK_RATE_LIMITED', which was saved
+ * as a failed session's AI analysis and counted by the breaker as a success.
+ */
+class AIRateLimitedError extends Error {
+    constructor(detail) {
+        super('The AI provider is rate-limited or out of quota');
+        this.name = 'AIRateLimitedError';
+        this.status = 429;
+        this.detail = detail;
+    }
+}
+exports.AIRateLimitedError = AIRateLimitedError;
+/** A provider's 429, as its SDK (`status`) or axios (`response.status`) reports it. */
+function isRateLimit(err) {
+    var _a;
+    return err instanceof AIRateLimitedError || (err === null || err === void 0 ? void 0 : err.status) === 429 || ((_a = err === null || err === void 0 ? void 0 : err.response) === null || _a === void 0 ? void 0 : _a.status) === 429;
+}
+/**
+ * The circuit breaker didn't send the call: the provider's recent calls
+ * failed. Still a CircuitOpenError, in words for testers: the Omni-Scan, Test
+ * locator and visual assertion answers show it, and "Circuit open for
+ * 'ai:gemini:<model>'; retry in 43120ms" isn't.
+ */
+class AIProviderPausedError extends CircuitBreaker_1.CircuitOpenError {
+    constructor(from) {
+        super(from.key, from.retryAfterMs);
+        this.seconds = Math.max(1, Math.ceil(from.retryAfterMs / 1000));
+        this.message = `The AI provider has been failing, so Xenon will call it again in ${this.seconds} s`;
+    }
+}
+exports.AIProviderPausedError = AIProviderPausedError;
+/** A call that outlasted its time limit; its request was cancelled. The circuit breaker counts it (`code`). */
+class AITimeoutError extends Error {
+    constructor(ms) {
+        super(`The AI provider didn't answer within ${Math.round(ms / 1000)} s`);
+        this.name = 'AITimeoutError';
+        this.code = 'ETIMEDOUT';
+    }
+}
+exports.AITimeoutError = AITimeoutError;
+/**
+ * How long an AI call may take, retries included, unless it says otherwise
+ * (failure analysis). Most are made while a test command runs: the LLM and
+ * visual healing tiers inside a failing findElement (both: 2 of these), the
+ * visual assertion and the screen description inside an execute script, an
+ * ai-icon find. The OpenAI and Anthropic SDKs otherwise wait up to 10 minutes
+ * a try, three tries, and Gemini has no limit, which held the command past
+ * the client's own timeout. Ollama's generate call already had 30 s.
+ */
+exports.AI_CALL_TIMEOUT_MS = 30000;
+/**
+ * How long a failed session's AI analysis may take, retries included. The
+ * OpenAI and Anthropic SDKs otherwise wait up to 10 minutes a try, three
+ * tries. The analysis runs after the session has ended (onSessionStopped
+ * doesn't wait for it), so this bounds the work left running, not a client's
+ * wait: an answer with a screenshot usually takes seconds.
+ */
+exports.FAILURE_ANALYSIS_TIMEOUT_MS = 120000;
+/** `call`, given a signal that cancels it after `ms`, when it fails with AITimeoutError. */
+function withTimeLimit(ms, call) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const controller = new AbortController();
+        let timer;
+        const expired = new Promise((_, reject) => {
+            timer = setTimeout(() => {
+                // The time-out first, so it, not the cancelled request's error, is the answer.
+                reject(new AITimeoutError(ms));
+                controller.abort();
+            }, ms);
+        });
+        try {
+            return yield Promise.race([call(controller.signal), expired]);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    });
+}
 /**
  * A provider's answer to the assertion prompt, `{ "result": true|false,
  * "reason": "..." }`, read leniently (code fences, text around the JSON,
@@ -97,7 +177,7 @@ class GeminiProvider {
         this.modelName =
             modelName && modelName.trim() !== '' ? modelName.trim() : 'gemini-3-flash-preview';
     }
-    analyze(prompt, screenshotBase64) {
+    analyze(prompt, screenshotBase64, signal) {
         return __awaiter(this, void 0, void 0, function* () {
             const parts = [prompt];
             if (screenshotBase64) {
@@ -116,17 +196,17 @@ class GeminiProvider {
             for (const version of versions) {
                 try {
                     const model = this.genAI.getGenerativeModel({ model: this.modelName }, { apiVersion: version });
-                    const result = yield model.generateContent(parts);
+                    const result = yield model.generateContent(parts, signal ? { signal } : undefined);
                     const response = yield result.response;
                     return response.text();
                 }
                 catch (err) {
                     lastError = err;
-                    // If we get a 429, the connection IS WORKING, just rate limited.
-                    if (err.message.includes('429') || err.message.includes('Quota exceeded')) {
-                        logger_1.default.info(`[Gemini] Connection verified (Rate Limited) via ${version} endpoint.`);
-                        return 'CONNECTION_OK_RATE_LIMITED';
-                    }
+                    // The other endpoint has the same quota. By status only: the message
+                    // holds the URL and the provider's text, where "429" can be a token
+                    // count or part of the model's name.
+                    if (err.status === 429)
+                        throw new AIRateLimitedError(err.message);
                     if (err.message.includes('404') ||
                         err.message.includes('not found') ||
                         err.message.includes('unsupported')) {
@@ -145,7 +225,7 @@ class OpenAIProvider {
         this.client = new openai_1.default({ apiKey, baseURL });
         this.model = model;
     }
-    analyze(prompt, screenshotBase64) {
+    analyze(prompt, screenshotBase64, signal) {
         return __awaiter(this, void 0, void 0, function* () {
             const messages = [
                 {
@@ -163,7 +243,7 @@ class OpenAIProvider {
                 model: this.model,
                 messages,
                 max_tokens: 500,
-            });
+            }, { signal });
             return response.choices[0].message.content || '';
         });
     }
@@ -173,7 +253,7 @@ class AnthropicProvider {
         this.client = new sdk_1.default({ apiKey });
         this.model = model;
     }
-    analyze(prompt, screenshotBase64) {
+    analyze(prompt, screenshotBase64, signal) {
         return __awaiter(this, void 0, void 0, function* () {
             const content = [{ type: 'text', text: prompt }];
             if (screenshotBase64) {
@@ -190,7 +270,7 @@ class AnthropicProvider {
                 model: this.model,
                 max_tokens: 500,
                 messages: [{ role: 'user', content }],
-            });
+            }, { signal });
             return response.content[0].text || '';
         });
     }
@@ -206,10 +286,11 @@ class OllamaProvider {
     checkAvailability() {
         return __awaiter(this, void 0, void 0, function* () {
             const now = Date.now();
-            // Use cached status if checked recently
-            if (this.isAvailable !== null &&
-                now - this.lastAvailabilityCheck < this.AVAILABILITY_CHECK_INTERVAL) {
-                return this.isAvailable;
+            // An answer is kept for a while; "down" is checked again at the next call
+            // (a refused local connection costs nothing), so Ollama is used as soon as
+            // it is back.
+            if (this.isAvailable && now - this.lastAvailabilityCheck < this.AVAILABILITY_CHECK_INTERVAL) {
+                return true;
             }
             try {
                 // Quick health check - Ollama has a /api/tags endpoint
@@ -225,7 +306,7 @@ class OllamaProvider {
             }
         });
     }
-    analyze(prompt, screenshotBase64) {
+    analyze(prompt, screenshotBase64, signal) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
             // Check if Ollama is available before attempting
@@ -239,7 +320,7 @@ class OllamaProvider {
                     prompt: prompt,
                     images: screenshotBase64 ? [screenshotBase64] : [],
                     stream: false,
-                }, { timeout: 30000 });
+                }, { timeout: 30000, signal });
                 return response.data.response || '';
             }
             catch (err) {
@@ -257,6 +338,8 @@ class AIService {
     constructor() {
         this.provider = null;
         this.isMock = false;
+        /** The settings `provider` was set up from (initializeProvider). */
+        this.setUpFrom = null;
         this.initializeProvider();
     }
     // Key the breaker by provider + resolved model name so a misbehaving model
@@ -276,16 +359,60 @@ class AIService {
         return `ai:${provider}:${model || 'default'}`;
     }
     // Single choke point for every LLM call so circuit-breaker state is shared
-    // across analyzeFailure / visualFind / healLocator.
-    callProvider(prompt, screenshotBase64) {
-        return __awaiter(this, void 0, void 0, function* () {
-            return CircuitBreaker_1.CIRCUIT_BREAKERS.execute(this.breakerKey(), () => this.provider.analyze(prompt, screenshotBase64));
+    // across analyzeFailure / visualFind / healLocator. A rate limit is
+    // AIRateLimitedError whichever provider sent it; a call is cancelled after
+    // `timeoutMs` (AI_CALL_TIMEOUT_MS unless given: AITimeoutError). The breaker
+    // counts both, and a call it holds back is AIProviderPausedError.
+    callProvider(prompt_1, screenshotBase64_1) {
+        return __awaiter(this, arguments, void 0, function* (prompt, screenshotBase64, opts = {}) {
+            var _a;
+            const provider = this.provider;
+            try {
+                return yield CircuitBreaker_1.CIRCUIT_BREAKERS.execute(this.breakerKey(), () => {
+                    var _a;
+                    return withTimeLimit((_a = opts.timeoutMs) !== null && _a !== void 0 ? _a : exports.AI_CALL_TIMEOUT_MS, (signal) => provider.analyze(prompt, screenshotBase64, signal));
+                });
+            }
+            catch (err) {
+                if (isRateLimit(err) && !(err instanceof AIRateLimitedError)) {
+                    throw new AIRateLimitedError((_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : String(err));
+                }
+                if (err instanceof CircuitBreaker_1.CircuitOpenError && !(err instanceof AIProviderPausedError)) {
+                    throw new AIProviderPausedError(err);
+                }
+                throw err;
+            }
         });
     }
+    /**
+     * Sets the provider up from the server's AI settings, which the AI engine
+     * page changes while the server runs (`POST /config`). Again only when they
+     * changed, so a provider's client, and Ollama's availability check, carry
+     * over from one call to the next.
+     */
     initializeProvider() {
         const providerType = config_1.config.aiProvider;
         const genericModel = config_1.config.aiModel;
         const baseUrl = config_1.config.aiBaseUrl;
+        const settings = JSON.stringify([
+            providerType,
+            genericModel,
+            baseUrl,
+            config_1.config.geminiApiKey,
+            config_1.config.openaiApiKey,
+            config_1.config.anthropicApiKey,
+            config_1.config.geminiModel,
+            config_1.config.openaiModel,
+            config_1.config.anthropicModel,
+            config_1.config.ollamaModel,
+            process.env.GEMINI_API_KEY === 'mock',
+        ]);
+        if (settings === this.setUpFrom)
+            return;
+        this.setUpFrom = settings;
+        // A provider chosen without a key is no provider, never the previous one.
+        this.provider = null;
+        this.isMock = false;
         logger_1.default.info(`[AIService] Initializing with provider: ${providerType}`);
         if (process.env.GEMINI_API_KEY === 'mock') {
             this.isMock = true;
@@ -321,7 +448,15 @@ class AIService {
             logger_1.default.error(`[AIService] Initialization error: ${err.message}`);
         }
     }
+    /**
+     * Whether a provider is set up for the settings in force now. Failure
+     * analysis and the LLM and visual healing tiers ask this first; it used to
+     * answer for the provider set up last, so choosing a configured provider on
+     * the AI engine page never turned them on, and choosing one without a key
+     * left the previous provider answering.
+     */
     isEnabled() {
+        this.initializeProvider();
         return this.provider !== null || this.isMock;
     }
     analyzeFailure(context) {
@@ -341,7 +476,9 @@ Fix: Add a pre-emptive check for the location permission dialog or use the \`aut
             try {
                 const prompt = this.constructPrompt(context);
                 const screenshotBase64 = this.getScreenshotBase64(context.screenshotPath);
-                const text = yield this.callProvider(prompt, screenshotBase64 || undefined);
+                const text = yield this.callProvider(prompt, screenshotBase64 || undefined, {
+                    timeoutMs: exports.FAILURE_ANALYSIS_TIMEOUT_MS,
+                });
                 logger_1.default.info(`[AIService] Analysis complete for ${context.sessionId}`);
                 return text;
             }
@@ -349,6 +486,12 @@ Fix: Add a pre-emptive check for the location permission dialog or use the \`aut
                 if (err instanceof CircuitBreaker_1.CircuitOpenError) {
                     logger_1.default.debug(`[AIService] Analysis skipped: ${err.message}`);
                 }
+                else if (err instanceof AIRateLimitedError) {
+                    logger_1.default.warn(`[AIService] No analysis for ${context.sessionId}: ${err.message}: ${err.detail}`);
+                }
+                else if (err instanceof AITimeoutError) {
+                    logger_1.default.warn(`[AIService] No analysis for ${context.sessionId}: ${err.message}`);
+                }
                 else {
                     logger_1.default.error(`[AIService] Analysis failed: ${err.message}`);
                 }
@@ -429,11 +572,14 @@ Answer with JSON only, exactly: {"result": true or false, "reason": "one sentenc
                 answer = yield this.callProvider(prompt, screenshotBase64);
             }
             catch (err) {
+                if (err instanceof AIRateLimitedError || err instanceof AITimeoutError) {
+                    throw new Error(`${err.message}, ${notChecked}. Try again later.`);
+                }
+                if (err instanceof AIProviderPausedError) {
+                    throw new Error(`The AI provider has been failing, ${notChecked}. Xenon will call it again in ${err.seconds} s.`);
+                }
                 throw new Error(`The AI provider failed, ${notChecked}: ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
             }
-            if (answer === RATE_LIMITED) {
-                throw new Error(`The AI provider is rate-limited, ${notChecked}. Try again later.`);
-            }
             const verdict = parseVisualVerdict(answer);
             if (!verdict) {
                 const said = answer.replace(/\s+/g, ' ').trim().slice(0, 200);
@@ -446,7 +592,9 @@ Answer with JSON only, exactly: {"result": true or false, "reason": "one sentenc
     /**
      * A short description of the screen in the screenshot, for
      * `xenon: analyzeScreen` and device control's Omni-Scan. Throws when there
-     * is no provider or the call fails, so the caller can say why there is none.
+     * is no provider or the call fails (AIRateLimitedError for a rate limit,
+     * AITimeoutError after AI_CALL_TIMEOUT_MS), so the caller can say why there
+     * is none.
      */
     describeScreen(screenshotBase64) {
         return __awaiter(this, void 0, void 0, function* () {
@@ -458,10 +606,7 @@ You are looking at a screenshot of a mobile app, for a software tester.
 Describe the screen in a few short sentences: which screen it is, its main elements and their state (for example filled in, empty, selected or disabled), and anything unexpected, such as an error message, a system dialog, a crash or a loading indicator.
 Describe only what is visible. Answer in plain text.
         `.trim();
-            const answer = yield this.callProvider(prompt, screenshotBase64);
-            if (answer === RATE_LIMITED)
-                throw new Error('The AI provider is rate-limited');
-            return answer;
+            return this.callProvider(prompt, screenshotBase64);
         });
     }
     /**
@@ -512,9 +657,10 @@ Describe only what is visible. Answer in plain text.
     }
     testConnection(testConfig) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const providerType = testConfig.aiProvider || config_1.config.aiProvider;
             try {
                 let testProvider = null;
-                const providerType = testConfig.aiProvider || config_1.config.aiProvider;
                 const model = testConfig.aiModel || config_1.config.aiModel;
                 const baseUrl = testConfig.aiBaseUrl || config_1.config.aiBaseUrl;
                 logger_1.default.info(`[AIService] Testing connection for provider: ${providerType}`);
@@ -547,17 +693,21 @@ Describe only what is visible. Answer in plain text.
                 }
                 if (!testProvider)
                     throw new Error('Failed to initialize provider for testing');
-                // Send a minimal ping command
-                const responseText = yield testProvider.analyze('Hello. Response: OK');
-                if (responseText === 'CONNECTION_OK_RATE_LIMITED') {
-                    return {
-                        success: true,
-                        message: `Successfully connected to ${providerType}! (Note: You are currently out of quota/rate-limited, but the setup is correct)`,
-                    };
-                }
+                // Send a minimal ping command. Not through the circuit breaker (a test
+                // shouldn't pause the provider for everyone), but with the time limit.
+                const pinged = testProvider;
+                yield withTimeLimit(exports.AI_CALL_TIMEOUT_MS, (signal) => pinged.analyze('Hello. Response: OK', undefined, signal));
                 return { success: true, message: `Successfully connected to ${providerType}!` };
             }
             catch (err) {
+                // Not a success: every AI call fails the same way until it has quota again.
+                if (isRateLimit(err)) {
+                    logger_1.default.warn(`[AIService] Connection test: ${providerType} is rate-limited: ${(_a = err.detail) !== null && _a !== void 0 ? _a : err.message}`);
+                    return {
+                        success: false,
+                        message: `${providerType} answered, but it is rate-limited or out of quota. AI features won't work with it until it has quota again.`,
+                    };
+                }
                 logger_1.default.error(`[AIService] Connection test failed: ${err.message}`);
                 return { success: false, message: `Connection failed: ${err.message}` };
             }