@xenon-device-management/xenon 2.14.0 → 2.16.0

package/lib/src/device-managers/AndroidDeviceManager.js

@@ -90,6 +90,11 @@ const typedi_2 = require("typedi");
  * they had worked. Within the 15 minutes a hub gives a node's install.
  */
 exports.ANDROID_INSTALL_TIMEOUT_MS = 10 * 60000;
+/** How long after adb, or its device tracking, failed to start it is tried again. */
+const ADB_RETRY_MS = 60000;
+function reasonOf(e) {
+    return e instanceof Error ? e.message : String(e);
+}
 /**
  * Whether a server set to `androidDeviceType` serves this phone: real phones,
  * emulators ('simulated'), or both, and with `bootedEmulators` only the
@@ -110,6 +115,11 @@ let AndroidDeviceManager = class AndroidDeviceManager {
         this.context = context;
         this.log = logger_1.default.scope('AndroidManager');
         this.adbAvailable = true;
+        // When adb last failed to start, discovery answers no phones until this
+        // time, then tries again.
+        this.adbRetryAt = 0;
+        // When adb's device tracking last failed to start, when to try it again.
+        this.trackerRetryAt = 0;
         this.abortControl = new Map();
         this.tracker = undefined;
         this.remoteTrackers = [];
@@ -161,8 +171,11 @@ let AndroidDeviceManager = class AndroidDeviceManager {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
             if (!this.adbAvailable) {
-                logger_1.default.info('adb is not available. So, returning empty list');
-                return [];
+                if (Date.now() < this.adbRetryAt) {
+                    logger_1.default.debug('adb is not available. So, returning empty list');
+                    return [];
+                }
+                this.adbAvailable = true;
             }
             let devices = [];
             try {
@@ -401,32 +414,57 @@ let AndroidDeviceManager = class AndroidDeviceManager {
     }
     getAdb() {
         return __awaiter(this, void 0, void 0, function* () {
-            try {
-                if (!this.adb) {
-                    try {
-                        this.adb = yield appium_adb_1.ADB.createADB({});
-                    }
-                    catch (e) {
-                        this.adbAvailable = false;
-                        this.log.error('Could not find ADB');
-                    }
-                    if (process.env.NODE_ENV !== 'test') {
-                        const client = adbkit_1.default.createClient();
-                        this.tracker = yield client.trackDevices();
-                        if (this.tracker && this.adb) {
-                            const originalADBTracking = this.createLocalAdbTracker(this.tracker, this.adb);
-                            yield originalADBTracking();
-                        }
-                    }
+            // One start at a time: calls that come while adb or its tracker is
+            // starting wait for that start. Two discoveries at once each started a
+            // tracker, and each tracker handled every plug event again.
+            if (!this.adbStarting) {
+                this.adbStarting = this.startAdb().finally(() => {
+                    this.adbStarting = undefined;
+                });
+            }
+            yield this.adbStarting;
+            return { adbInstance: this.adb, adbTracker: this.tracker };
+        });
+    }
+    startAdb() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.adb) {
+                try {
+                    this.adb = yield appium_adb_1.ADB.createADB({});
+                }
+                catch (e) {
+                    this.adbUnavailable('Could not start adb', e);
                 }
             }
-            catch (e) {
-                logger_1.default.error(`Failed to initialize ADB: ${e}`);
-                this.adbAvailable = false;
+            // Plug events. Without them discovery still lists phones each time it
+            // runs, so a failure here hides none; it is tried again a minute later.
+            if (this.adb &&
+                !this.tracker &&
+                process.env.NODE_ENV !== 'test' &&
+                Date.now() >= this.trackerRetryAt) {
+                try {
+                    const tracker = yield adbkit_1.default.createClient().trackDevices();
+                    yield this.createLocalAdbTracker(tracker, this.adb)();
+                    this.tracker = tracker;
+                }
+                catch (e) {
+                    this.trackerRetryAt = Date.now() + ADB_RETRY_MS;
+                    this.log.error(`Could not follow Android plug events through adb: ${reasonOf(e)}. ` +
+                        `Trying again in ${ADB_RETRY_MS / 1000} s.`);
+                }
             }
-            return { adbInstance: this.adb, adbTracker: this.tracker };
         });
     }
+    /**
+     * Discovery answers no phones for a minute, then tries again: adb may work
+     * by then (an SDK installed, a stuck adb server restarted). Through 2.15 it
+     * never tried again before a restart, and didn't say why.
+     */
+    adbUnavailable(what, e) {
+        this.adbAvailable = false;
+        this.adbRetryAt = Date.now() + ADB_RETRY_MS;
+        this.log.error(`${what}: ${reasonOf(e)}. Trying again in ${ADB_RETRY_MS / 1000} s.`);
+    }
     getAdbForDevice(udid) {
         return __awaiter(this, void 0, void 0, function* () {
             const { adbInstance } = yield this.getAdb();

package/lib/src/device-managers/IOSDeviceManager.js

@@ -266,6 +266,16 @@ let IOSDeviceManager = class IOSDeviceManager {
             var _a;
             if (device.cloud)
                 return true;
+            // A simulator's WebDriverAgent is the XCUITest driver's: it builds,
+            // installs and launches it on the session's wdaLocalPort (iOSCapabilities)
+            // once Appium creates the session, after this check, and boots a shut-down
+            // simulator itself. Xenon runs WDA only on an iPhone (go-ios runwda), so
+            // asking for it here refused every simulator whose WDA wasn't already up,
+            // and recoverHealth's reboot never starts one. Through 2.15 it did.
+            if (!device.realDevice) {
+                this.log.info(`[SessionStart] ${device.udid} is a simulator: the driver starts its WebDriverAgent`);
+                return true;
+            }
             this.log.info(`🚀 [SessionStart] Verifying readiness for ${device.udid}...`);
             const streamService = typedi_1.Container.get(IOSStreamService_1.default);
             const streamStatus = streamService.getStreamStatus(device.udid);

package/lib/src/device-managers/android/AndroidH264StreamService.js

@@ -93,27 +93,34 @@ let AndroidH264StreamService = class AndroidH264StreamService {
     startWatchdog() {
         // unref()ed below: the server's listener keeps the process alive; this only
         // needs to fire while it runs.
-        setInterval(() => {
-            const now = Date.now();
-            for (const [udid, s] of this.sessions.entries()) {
-                if (s.status !== 'running')
-                    continue;
-                if (s.mux.clientCount > 0) {
-                    s.emptyAt = undefined;
-                }
-                else if (s.emptyAt === undefined) {
-                    s.emptyAt = now;
-                }
-                else if (now - s.emptyAt > IDLE_TIMEOUT_MS) {
-                    logger_1.default.info(`[${udid}] Stopping idle H.264 stream (no viewers for ${IDLE_TIMEOUT_MS}ms)`);
-                    // The stream went, but the live-preview hold stayed: with H.264
-                    // preview, a tab closed without releasing kept the device busy for
-                    // good. stop() itself must not release (a recording starting calls
-                    // it); an idle stop does, if nothing else uses the device.
-                    void this.stop(udid).then(() => (0, previewHold_1.releaseIdlePreviewHold)(udid));
-                }
+        setInterval(() => this.sweep(Date.now()), 60000).unref();
+    }
+    /** One look at every capture: ends those nobody has watched for long enough. */
+    sweep(now) {
+        for (const [udid, s] of this.sessions.entries()) {
+            if (s.status !== 'running')
+                continue;
+            if (s.mux.clientCount > 0) {
+                s.emptyAt = undefined;
+            }
+            else if (s.endWhenUnwatched && !this.startPromises.has(udid)) {
+                // A page asked for MJPEG while this capture was starting for a viewer
+                // who then never came (endWhenUnwatched leaves a start alone).
+                logger_1.default.info(`[${udid}] Stopping the unwatched H.264 stream: the phone is shown as MJPEG`);
+                void this.stop(udid);
             }
-        }, 60000).unref();
+            else if (s.emptyAt === undefined) {
+                s.emptyAt = now;
+            }
+            else if (now - s.emptyAt > IDLE_TIMEOUT_MS) {
+                logger_1.default.info(`[${udid}] Stopping idle H.264 stream (no viewers for ${IDLE_TIMEOUT_MS}ms)`);
+                // The stream went, but the live-preview hold stayed: with H.264
+                // preview, a tab closed without releasing kept the device busy for
+                // good. stop() itself must not release (a recording starting calls
+                // it); an idle stop does, if nothing else uses the device.
+                void this.stop(udid).then(() => (0, previewHold_1.releaseIdlePreviewHold)(udid));
+            }
+        }
     }
     getMultiplexer(udid) {
         var _a;
@@ -131,6 +138,7 @@ let AndroidH264StreamService = class AndroidH264StreamService {
                 var _a, _b;
                 const mux = new H264Multiplexer_1.H264Multiplexer();
                 const session = { status: 'running', mux };
+                mux.onEmpty(() => this.endIfUnwatched(udid, session));
                 this.sessions.set(udid, session);
                 try {
                     let resolveConfig = () => undefined;
@@ -178,6 +186,44 @@ let AndroidH264StreamService = class AndroidH264StreamService {
             }
         });
     }
+    /**
+     * A page that showed this phone's H.264 preview now shows MJPEG (its player
+     * failed, or it cannot play H.264), and the MJPEG capture is starting for
+     * it. One capture per phone where possible, but never at the expense of a
+     * viewer still playing H.264: another tile, tab or admin may be. So the
+     * capture ends now if nobody watches it, else the moment its last viewer
+     * leaves, never after the idle wait. A viewer who joins meanwhile is served
+     * and ends it in turn. A capture still starting is left to the viewer it is
+     * starting for, and the watchdog ends it if that viewer never comes.
+     */
+    endWhenUnwatched(udid) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const session = this.sessions.get(udid);
+            if (!session || session.status !== 'running')
+                return;
+            session.endWhenUnwatched = true;
+            const viewers = session.mux.clientCount;
+            if (viewers > 0) {
+                logger_1.default.info(`[${udid}] H.264 stream kept beside MJPEG for its ${viewers} viewer(s); ` +
+                    'it ends when the last one leaves');
+                return;
+            }
+            if (this.startPromises.has(udid)) {
+                logger_1.default.info(`[${udid}] H.264 stream still starting: kept for the viewer it is starting for`);
+                return;
+            }
+            logger_1.default.info(`[${udid}] Stopping the unwatched H.264 stream: the phone is shown as MJPEG`);
+            yield this.stop(udid);
+        });
+    }
+    /** The last viewer left `session`: end it if a page asked (endWhenUnwatched). */
+    endIfUnwatched(udid, session) {
+        // A capture replaced since (a stop and a new start) is not this one.
+        if (this.sessions.get(udid) !== session || !session.endWhenUnwatched)
+            return;
+        logger_1.default.info(`[${udid}] Stopping the H.264 stream: its last viewer left, the phone is shown as MJPEG`);
+        void this.stop(udid);
+    }
     stop(udid) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
@@ -192,6 +238,9 @@ let AndroidH264StreamService = class AndroidH264StreamService {
                 /* best-effort */
             }
             this.sessions.delete(udid);
+            // Viewers still on it (a recording starting, a stream/stop) are told, so
+            // their players fall back to MJPEG instead of freezing on the last frame.
+            session.mux.close();
             logger_1.default.info(`[${udid}] H.264 stream terminated.`);
         });
     }

package/lib/src/device-managers/android/H264Multiplexer.js

@@ -9,6 +9,7 @@ class H264Multiplexer {
     constructor() {
         this.clients = new Set();
         this.gop = []; // current GOP: [keyframe, ...deltas-since]
+        this.closed = false;
     }
     setConfig(p) {
         this.config = p;
@@ -16,9 +17,21 @@ class H264Multiplexer {
     get clientCount() {
         return this.clients.size;
     }
-    /** Register a client sink. Returns a remover. */
-    addClient(send) {
-        const c = { send, started: false };
+    /** Called each time the last client leaves. */
+    onEmpty(listener) {
+        this.emptyListener = listener;
+    }
+    /**
+     * Register a client sink, and how to end it when the stream does (see
+     * `close`). Returns a remover (safe to call twice).
+     */
+    addClient(send, end) {
+        const c = { send, started: false, end };
+        if (this.closed) {
+            // Joined a stream already stopped: end it now rather than starve it.
+            end === null || end === void 0 ? void 0 : end();
+            return () => undefined;
+        }
         this.clients.add(c);
         if (this.config)
             send(this.config);
@@ -29,9 +42,30 @@ class H264Multiplexer {
             c.started = true;
         }
         return () => {
-            this.clients.delete(c);
+            var _a;
+            if (this.clients.delete(c) && this.clients.size === 0)
+                (_a = this.emptyListener) === null || _a === void 0 ? void 0 : _a.call(this);
         };
     }
+    /**
+     * The capture stopped: end every client (its socket closes, so its player
+     * falls back to MJPEG instead of freezing on its last frame). Not an
+     * `onEmpty`: nobody left, the stream did.
+     */
+    close() {
+        var _a;
+        this.closed = true;
+        const ending = [...this.clients];
+        this.clients.clear();
+        for (const c of ending) {
+            try {
+                (_a = c.end) === null || _a === void 0 ? void 0 : _a.call(c);
+            }
+            catch (_b) {
+                /* one client's end must not keep the others open */
+            }
+        }
+    }
     /** Feed one upstream packet; fans out per the join semantics above. */
     push(p) {
         if (p.type === 'config') {

package/lib/src/device-utils.js

@@ -87,6 +87,7 @@ const device_managers_1 = require("./device-managers");
 const device_service_1 = require("./data-service/device-service");
 const deviceClaims_1 = require("./data-service/deviceClaims");
 const PluginContext_1 = require("./PluginContext");
+const TracingService_1 = require("./services/TracingService");
 const logger_1 = __importDefault(require("./logger"));
 const Platform_1 = __importDefault(require("./enums/Platform"));
 const lodash_1 = __importDefault(require("lodash"));
@@ -250,7 +251,7 @@ function allocateDeviceForSession(capability_1, deviceTimeOutMs_1, deviceQueryIn
                         if (device !== null)
                             return true;
                     }
-                    logger_1.default.info(`Waiting for free device. Filter: ${JSON.stringify(filters)}}`);
+                    logger_1.default.info(`Waiting for free device. Filter: ${JSON.stringify(filters)}`);
                     return false;
                 }));
             }), { timeout, intervalBetweenAttempts });
@@ -269,7 +270,7 @@ function allocateDeviceForSession(capability_1, deviceTimeOutMs_1, deviceQueryIn
             else if (possibleDevice && (0, device_service_1.isDeviceReserved)(possibleDevice)) {
                 failureReason = `Device is reserved by ${possibleDevice.reservedBy}.`;
             }
-            throw new Error(`${failureReason}. Device request: ${JSON.stringify(filterCopy)}`);
+            throw new Error(`${failureReason} Device request: ${JSON.stringify(filterCopy)}`);
         }
         // We have a locked device here
         if (device !== null) {
@@ -696,8 +697,16 @@ function releaseBlockedDevices(newCommandTimeout) {
             if (timeSinceLastCmdExecuted > timeoutSeconds) {
                 // unblock regardless of whether the device has session or not
                 logger_1.default.info(`Unblocking device ${device.udid} at host ${device.host} because it has been idle for ${timeSinceLastCmdExecuted} seconds`);
+                const idleSession = (_a = device.claimSessionId) !== null && _a !== void 0 ? _a : device.session_id;
                 // Before anything below can release the phone (onSessionStopped too).
-                yield restoreIdleSessionNetwork((_a = device.claimSessionId) !== null && _a !== void 0 ? _a : device.session_id);
+                yield restoreIdleSessionNetwork(idleSession);
+                yield forgetIdleSessionMemory(idleSession);
+                if (idleSession) {
+                    typedi_1.Container.get(TracingService_1.TracingService).endSessionSpan(idleSession, {
+                        failed: true,
+                        reason: 'Session timed out due to inactivity',
+                    });
+                }
                 // Principal Protection: If this device has an active dashboard session, stop it properly
                 if (device.session_id) {
                     const sessionId = device.session_id;
@@ -739,6 +748,23 @@ function restoreIdleSessionNetwork(sessionId) {
         }
     });
 }
+/**
+ * The idle session's virtual elements and autowait settings
+ * (sessionMemory.ts). Imported lazily, as above. Never throws.
+ */
+function forgetIdleSessionMemory(sessionId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!sessionId)
+            return;
+        try {
+            const { forgetSessionMemory } = yield Promise.resolve().then(() => __importStar(require('./sessions/sessionMemory')));
+            forgetSessionMemory(sessionId);
+        }
+        catch (err) {
+            logger_1.default.warn(`Could not forget idle session ${sessionId}'s elements: ${err}`);
+        }
+    });
+}
 function setupCronReleaseBlockedDevices(intervalMs, newCommandTimeoutSec) {
     return __awaiter(this, void 0, void 0, function* () {
         if (cronTimerToReleaseBlockedDevices) {

package/lib/src/gateway/defaultGateway.js

@@ -34,7 +34,7 @@ const createWith = (hubGrants) => ({
 });
 const dashboardHooks = {
     before: (sessionId, command, req, res) => event_manager_1.DASHBORD_EVENT_MANAGER.beforeSessionCommand(sessionId, command, req, res),
-    after: (sessionId, command, req, res, body) => event_manager_1.DASHBORD_EVENT_MANAGER.afterSessionCommand(sessionId, command, null, req, res, body),
+    after: (sessionId, command, req, res, body, heal) => event_manager_1.DASHBORD_EVENT_MANAGER.afterSessionCommand(sessionId, command, null, req, res, body, heal),
 };
 /** A hub: sessions other servers run are forwarded to them. */
 function hubGatewayOptions(config) {

package/lib/src/gateway/forwardToNode.js

@@ -11,10 +11,10 @@ exports.readAnswer = readAnswer;
 exports.sendAnswer = sendAnswer;
 const http_1 = __importDefault(require("http"));
 const https_1 = __importDefault(require("https"));
-const https_proxy_agent_1 = require("https-proxy-agent");
-const http_proxy_agent_1 = require("http-proxy-agent");
 const internalCall_1 = require("./internalCall");
 const hubSessionToken_1 = require("./hubSessionToken");
+const healReport_1 = require("./healReport");
+const outboundProxy_1 = require("../helpers/outboundProxy");
 /**
  * Sending a session command on to the node that runs the session, and handing
  * the node's answer back to the client unchanged.
@@ -44,6 +44,7 @@ const NEVER_FORWARDED = new Set([
     'x-xenon-token',
     internalCall_1.INTERNAL_CALL_HEADER,
     hubSessionToken_1.HUB_TOKEN_HEADER,
+    healReport_1.HEAL_REPORT_HEADER,
 ]);
 function forwardedRequestHeaders(incoming) {
     const out = {};
@@ -58,17 +59,11 @@ function forwardedRequestHeaders(incoming) {
     out['accept-encoding'] = 'identity';
     return out;
 }
-/** HTTP(S)_PROXY from the environment, as the old command proxy honoured it. */
-function proxyAgentFor(url) {
-    const proxy = process.env.HTTP_PROXY || process.env.HTTPS_PROXY;
-    if (!proxy)
-        return undefined;
-    return url.protocol === 'https:' ? new https_proxy_agent_1.HttpsProxyAgent(proxy) : new http_proxy_agent_1.HttpProxyAgent(proxy);
-}
 /**
  * Send one request to a node and resolve with its response stream. No retry:
  * a WebDriver command is not safe to repeat. Credentials in the URL (a cloud
- * provider's) become basic auth.
+ * provider's) become basic auth. Through the environment's proxy for the URL,
+ * by axios's rule (helpers/outboundProxy.ts).
  */
 function sendToNode(request) {
     const url = new URL(request.url);
@@ -78,7 +73,7 @@ function sendToNode(request) {
     if (Buffer.isBuffer(body))
         headers['content-length'] = body.length;
     return new Promise((resolve, reject) => {
-        const outgoing = lib.request(url, { method: request.method, headers, agent: proxyAgentFor(url), signal: request.signal }, resolve);
+        const outgoing = lib.request(url, { method: request.method, headers, agent: (0, outboundProxy_1.proxyAgentFor)(url), signal: request.signal }, resolve);
         outgoing.on('error', reject);
         if (body && !Buffer.isBuffer(body)) {
             body.on('error', (err) => outgoing.destroy(err));

package/lib/src/gateway/healReport.js

@@ -0,0 +1,87 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HEAL_REPORT_HEADER = void 0;
+exports.runReportingHeals = runReportingHeals;
+exports.reportHeal = reportHeal;
+exports.takeHealReport = takeHealReport;
+const async_hooks_1 = require("async_hooks");
+/**
+ * A heal on a node's phone, on its way back to the hub.
+ *
+ * A session the hub created runs its commands on the node, so the node's
+ * CommandInterceptor heals its finds. The node keeps no record of such a
+ * session; the hub does. So the node puts the heal on its answer to the
+ * forwarded command, as the `x-xenon-heal` header, and the hub takes it off
+ * before relaying the answer and records it with the command, as it records a
+ * local session's heal.
+ *
+ * The node's session gateway runs a command that came from a hub inside
+ * `runReportingHeals`, so the interceptor, deep inside Appium's route, finds
+ * the answer to put it on. Anywhere else `reportHeal` does nothing, and the
+ * server records the heal itself.
+ */
+exports.HEAL_REPORT_HEADER = 'x-xenon-heal';
+/**
+ * A header longer than this isn't sent: the hub's HTTP parser refuses an
+ * answer whose headers pass 16 KB, and a refused answer would fail the
+ * command. A heal of selectors that long goes unrecorded instead.
+ */
+const MAX_HEADER_LENGTH = 8 * 1024;
+const answers = new async_hooks_1.AsyncLocalStorage();
+/** Run the rest of a command that came from a hub with its answer at hand. */
+function runReportingHeals(res, fn) {
+    const answer = { res };
+    const forget = () => {
+        answer.res = null;
+    };
+    res.once('finish', forget);
+    res.once('close', forget);
+    return answers.run(answer, fn);
+}
+/**
+ * Put `heal` on the answer to the hub's command this code runs for.
+ * `not-from-hub` when the command didn't come from a hub (or the answer has
+ * gone): the caller records the heal itself.
+ */
+function reportHeal(heal) {
+    var _a;
+    const res = (_a = answers.getStore()) === null || _a === void 0 ? void 0 : _a.res;
+    if (!res || res.headersSent)
+        return 'not-from-hub';
+    const encoded = Buffer.from(JSON.stringify(heal), 'utf8').toString('base64url');
+    if (encoded.length > MAX_HEADER_LENGTH)
+        return 'too-long';
+    res.setHeader(exports.HEAL_REPORT_HEADER, encoded);
+    return 'reported';
+}
+/**
+ * The heal a node put on its answer, taken off the headers so it isn't
+ * relayed to the client. Undefined when there is none, or it isn't one.
+ */
+function takeHealReport(headers) {
+    const raw = headers[exports.HEAL_REPORT_HEADER];
+    delete headers[exports.HEAL_REPORT_HEADER];
+    if (typeof raw !== 'string' || !raw)
+        return undefined;
+    try {
+        const heal = JSON.parse(Buffer.from(raw, 'base64url').toString('utf8'));
+        if (!heal ||
+            typeof heal.originalSelector !== 'string' ||
+            typeof heal.healedSelector !== 'string' ||
+            typeof heal.confidence !== 'number') {
+            return undefined;
+        }
+        const optional = (v) => (typeof v === 'string' ? v : undefined);
+        return {
+            originalSelector: heal.originalSelector,
+            originalStrategy: optional(heal.originalStrategy),
+            healedSelector: heal.healedSelector,
+            healedStrategy: optional(heal.healedStrategy),
+            confidence: heal.confidence,
+            tier: typeof heal.tier === 'number' ? heal.tier : undefined,
+        };
+    }
+    catch (_a) {
+        return undefined;
+    }
+}

package/lib/src/gateway/hubSessionToken.js

@@ -59,6 +59,7 @@ const jose = __importStar(require("jose"));
 const logger_1 = __importDefault(require("../logger"));
 const JwtKeyService_1 = require("../services/token/JwtKeyService");
 const singleUseLedger_1 = require("../services/token/singleUseLedger");
+const outboundProxy_1 = require("../helpers/outboundProxy");
 /**
  * The credential a hub's calls to a node carry.
  *
@@ -221,6 +222,22 @@ exports.HubSessionTokenIssuer = HubSessionTokenIssuer = __decorate([
 function hubJwksUrl(hub) {
     return new URL('/xenon/api/auth/jwks.json', hub);
 }
+/**
+ * A key lookup that fails after `ms`. jose's own timeout starts once its
+ * request has a socket, which a proxy agent hands it only after the proxy
+ * answers, so a proxy that never answers held every token check.
+ */
+function withinTime(getKey, ms) {
+    return (header, token) => {
+        let timer;
+        return Promise.race([
+            Promise.resolve(getKey(header, token)),
+            new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new Error(`the hub's keys did not arrive in ${ms} ms`)), ms);
+            }),
+        ]).finally(() => clearTimeout(timer));
+    };
+}
 /** The node could not check a hub token (the hub's keys could not be fetched). */
 class HubTokenUnavailableError extends Error {
 }
@@ -251,15 +268,21 @@ function isVerdict(err) {
  * a token names a key it does not have (a hub that rotated its key).
  */
 class HubSessionTokenVerifier {
-    constructor(hubUrl, keys) {
+    constructor(hubUrl, keys, opts = {}) {
+        var _a;
         /** The create tokens already taken, by id, until each one expires. */
         this.usedCreates = new singleUseLedger_1.SingleUseLedger(exports.HUB_CREATE_TTL_SECONDS);
+        const jwksUrl = hubJwksUrl(hubUrl);
+        const timeoutMs = (_a = opts.timeoutMs) !== null && _a !== void 0 ? _a : 5000;
         this.keys =
-            keys !== null && keys !== void 0 ? keys : jose.createRemoteJWKSet(hubJwksUrl(hubUrl), {
-                timeoutDuration: 5000,
+            keys !== null && keys !== void 0 ? keys : withinTime(jose.createRemoteJWKSet(jwksUrl, {
+                timeoutDuration: timeoutMs,
                 cooldownDuration: 30000,
                 cacheMaxAge: 10 * 60000,
-            });
+                // The proxy the node's axios calls to its hub take; jose's own
+                // fetch takes none.
+                agent: (0, outboundProxy_1.proxyAgentFor)(jwksUrl),
+            }), timeoutMs);
     }
     verify(token, sessionId) {
         return __awaiter(this, void 0, void 0, function* () {

package/lib/src/gateway/nodeWebDriverUrl.js

@@ -30,6 +30,7 @@ const logger_1 = __importDefault(require("../logger"));
 const helpers_1 = require("../helpers");
 const appiumBasePath_1 = require("../app/appiumBasePath");
 const PluginContext_1 = require("../PluginContext");
+const outboundProxy_1 = require("../helpers/outboundProxy");
 /**
  * Where a node's WebDriver API lives.
  *
@@ -95,11 +96,8 @@ let NodeBasePathResolver = class NodeBasePathResolver {
             let advertised;
             try {
                 const rejectUnauthorized = (_a = typedi_1.Container.get(PluginContext_1.PluginContext).pluginArgs) === null || _a === void 0 ? void 0 : _a.tlsRejectUnauthorized;
-                const response = yield axios_1.default.get(`${origin}${exports.WEBDRIVER_INFO_PATH}`, {
-                    timeout: exports.LOOKUP_TIMEOUT_MS,
-                    validateStatus: (status) => status === 200,
-                    httpsAgent: new https_1.default.Agent({ rejectUnauthorized: rejectUnauthorized !== false }),
-                });
+                const url = `${origin}${exports.WEBDRIVER_INFO_PATH}`;
+                const response = yield axios_1.default.get(url, Object.assign({ timeout: exports.LOOKUP_TIMEOUT_MS, validateStatus: (status) => status === 200, httpsAgent: new https_1.default.Agent({ rejectUnauthorized: rejectUnauthorized !== false }) }, (0, outboundProxy_1.axiosProxyConfig)(url, { rejectUnauthorized: rejectUnauthorized !== false })));
                 const basePath = (_b = response.data) === null || _b === void 0 ? void 0 : _b.basePath;
                 if (typeof basePath === 'string')
                     advertised = (0, appiumBasePath_1.normalizeBasePath)(basePath);

package/lib/src/gateway/sessionGateway.js

@@ -15,6 +15,7 @@ exports.createHubRouting = createHubRouting;
 const commandAuth_1 = require("../middleware/commandAuth");
 const internalCall_1 = require("./internalCall");
 const hubSessionToken_1 = require("./hubSessionToken");
+const healReport_1 = require("./healReport");
 const forwardToNode_1 = require("./forwardToNode");
 /** Where a session runs could not be determined (a database error). */
 exports.ROUTING_UNAVAILABLE_STATUS = 503;
@@ -62,7 +63,14 @@ function createSessionGatewayLayer(deps) {
             const latency = (_b = (_a = deps.latencyOf) === null || _a === void 0 ? void 0 : _a.call(deps, sessionId)) !== null && _b !== void 0 ? _b : 0;
             if (latency > 0)
                 yield new Promise((resolve) => setTimeout(resolve, latency));
-            next();
+            // On a node, a heal of the hub's command goes back to the hub on the answer
+            // (healReport.ts). The header proves nothing elsewhere: a hub or a
+            // standalone server checks no hub token, so a client could send one there
+            // to keep its heals out of the record.
+            if (fromHub && deps.hubTokens)
+                (0, healReport_1.runReportingHeals)(res, next);
+            else
+                next();
         });
     }
     return function xenonSessionGateway(req, res, next) {
@@ -176,8 +184,10 @@ function createHubRouting(deps) {
                         abort.abort();
                 });
                 let captured;
+                let heal;
                 try {
                     const upstream = yield (0, forwardToNode_1.sendToNode)(yield nodeRequest(req, sessionId, location, abort.signal));
+                    heal = (0, healReport_1.takeHealReport)(upstream.headers);
                     captured = yield (0, forwardToNode_1.relayAnswer)(upstream, res, !!hooks);
                 }
                 catch (error) {
@@ -192,7 +202,7 @@ function createHubRouting(deps) {
                 }
                 if (hooks && captured !== undefined) {
                     yield hooks
-                        .after(sessionId, command, req, res, captured)
+                        .after(sessionId, command, req, res, captured, heal)
                         .catch((error) => logger.warn(`Dashboard log failed for session ${sessionId}: ${(0, commandAuth_1.summarize)(error)}`));
                 }
             });