@xenon-device-management/xenon 2.15.0 → 2.17.0

package/lib/src/dashboard/services/failure-analysis-service.js

@@ -45,80 +45,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MAX_CONCURRENT_FAILURE_ANALYSES = exports.ANALYSIS_CATEGORIES = exports.ERROR_PATTERNS = void 0;
+exports.MAX_CONCURRENT_FAILURE_ANALYSES = void 0;
 exports.categorizeSessionFailure = categorizeSessionFailure;
 exports.explainSessionFailure = explainSessionFailure;
 const prisma_1 = require("../../prisma");
 const logger_1 = __importDefault(require("../../logger"));
-/**
- * Each category and the texts that file a failure under it, tried in this
- * order against the failure reason and the last five failed commands. The
- * dashboard's runbooks describe a category by these, not by its name
- * (failure-categories.spec.ts).
- */
-exports.ERROR_PATTERNS = [
-    {
-        category: 'ELEMENT_NOT_FOUND',
-        patterns: [
-            'NoSuchElementError',
-            'unable to find an element',
-            'An element could not be located',
-            'no such element',
-        ],
-    },
-    {
-        category: 'APP_CRASH',
-        patterns: [
-            'Appium crashed',
-            'process has died',
-            'activity has died',
-            'The application has crashed',
-            'Application not responding',
-            "org.openqa.selenium.WebDriverException: An unknown server-side error occurred while processing the command. Original error: The application under test with bundle id '.*' is not running or cannot be found",
-        ],
-    },
-    {
-        category: 'TIMEOUT',
-        patterns: ['timeout', 'timed out', 'TimeoutException', 'New Command Timeout', 'socket hang up'],
-    },
-    {
-        category: 'PERMISSION_BLOCKED',
-        patterns: ['Permission alert', 'Security alert', 'Always Allow', 'Allow while using app'],
-    },
-    {
-        category: 'WDA_FAILURE',
-        patterns: [
-            'WebDriverAgent',
-            'WDA',
-            'xcodebuild failed',
-            'crashed with code',
-            'Unable to connect to WDA',
-            'Session does not exist',
-            'the session is not in a running state',
-        ],
-    },
-    {
-        category: 'XENON_COMMAND_FAILURE',
-        patterns: ['Command failed', 'telemetry failed', 'interceptor error'],
-    },
-    {
-        category: 'SYSTEM_OVERLOAD',
-        patterns: ['OutOfMemory', 'MemoryLimit', 'thermal throttling', 'too many open files'],
-    },
-];
-/** What the analysis writes when no pattern matches. */
-const UNMATCHED = 'UNKNOWN';
-/**
- * Every category the analysis writes to a failed session's
- * `failure_category`, upper case as stored. The only other value the column
- * holds is `HUB_RESTART_CATEGORY` (SessionManager). The API reference's
- * examples and the dashboard's runbooks are held to these
- * (failure-categories.spec.ts).
- */
-exports.ANALYSIS_CATEGORIES = [
-    ...exports.ERROR_PATTERNS.map((p) => p.category),
-    UNMATCHED,
-];
+const failureCategories_1 = require("./failureCategories");
 /**
  * How many failed sessions' AI analyses run at once (explainSessionFailure);
  * the others wait their turn. They run after their sessions have ended, so
@@ -150,8 +82,9 @@ function endAnalysisTurn() {
 }
 /**
  * Files a failed session under a category (`failure_category`) by its failure
- * reason and its failed commands. Rules only, so it is quick: a session's end
- * waits for it. Never throws.
+ * reason and its last five failed commands' errors (categorizeFailure, in
+ * failureCategories.ts). Rules only, so it is quick: a session's end waits for
+ * it. Never throws.
  */
 function categorizeSessionFailure(sessionId) {
     return __awaiter(this, void 0, void 0, function* () {
@@ -164,22 +97,11 @@ function categorizeSessionFailure(sessionId) {
             });
             if (!session)
                 return;
-            const reason = session.failure_reason || '';
-            const logs_text = session.SessionLog.map((l) => `${l.title} ${l.response}`).join(' ');
-            const combined_text = (reason + ' ' + logs_text).toLowerCase();
-            let identifiedCategory = UNMATCHED;
-            for (const item of exports.ERROR_PATTERNS) {
-                if (item.patterns.some((p) => new RegExp(p, 'i').test(combined_text))) {
-                    identifiedCategory = item.category;
-                    break;
-                }
-            }
-            // Special case for App Crash - check Logcat/Syslog if available
-            // For now we rely on the error response text which usually mentions "process has died"
-            logger_1.default.info(`[FailureAnalysis] Session ${sessionId} identified as ${identifiedCategory}`);
+            const category = (0, failureCategories_1.categorizeFailure)(session.failure_reason || '', session.SessionLog.map((l) => (0, failureCategories_1.commandErrorOf)(l.response)));
+            logger_1.default.info(`[FailureAnalysis] Session ${sessionId} identified as ${category}`);
             yield prisma_1.prisma.session.update({
                 where: { id: sessionId },
-                data: { failure_category: identifiedCategory },
+                data: { failure_category: category },
             });
         }
         catch (err) {

package/lib/src/dashboard/services/failureCategories.js

@@ -0,0 +1,264 @@
+"use strict";
+/**
+ * What files a failed session under a `failure_category`, and how.
+ *
+ * The texts here are those the installed drivers, Appium and Xenon send
+ * (UiAutomator2 6.4.1 with its server 9.3.1, XCUITest 10.9.0 with
+ * WebDriverAgent 10.2.3, base-driver 10.x), plus a few a test may report in
+ * its own words (`xenon: setSessionStatus`), marked as such.
+ *
+ * Through 2.15 the rules were guesses:
+ * - none of App crash's texts existed anywhere;
+ * - a raw body's stack trace was matched too. Every UiAutomator2 error a hub
+ *   records carries netty's `IdleStateHandler`, so one that matched no
+ *   earlier rule was filed TIMEOUT;
+ * - the five failed commands were read as one text, so a failure the test
+ *   had recovered from could decide the category.
+ *
+ * A rule matches a W3C error code exactly (`codes`), or a phrase anywhere in
+ * the reason or a command's error or message (`phrases`), ignoring case.
+ * Codes are only on the rows a hub records for a node's sessions: this
+ * server's own rows hold the message alone, so every category needs phrases
+ * too. Xenon's own messages are read first, by how they start, since they
+ * quote the test's selector (xenonOwnCategory). Rules are tried in this order.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RETIRED_CATEGORIES = exports.ANALYSIS_CATEGORIES = exports.FAILURE_RULES = exports.UNMATCHED_CATEGORY = exports.HUB_RESTART_CATEGORY = void 0;
+exports.xenonOwnCategory = xenonOwnCategory;
+exports.commandErrorOf = commandErrorOf;
+exports.categorizeFailure = categorizeFailure;
+/** A session the server's restart or shutdown ended. SessionManager also writes it at boot. */
+exports.HUB_RESTART_CATEGORY = 'HUB_RESTART';
+/** What a failure that matches no rule is filed as. */
+exports.UNMATCHED_CATEGORY = 'UNKNOWN';
+exports.FAILURE_RULES = [
+    {
+        // ShutdownCoordinator's drain: the whole reason, never a phrase in one.
+        category: exports.HUB_RESTART_CATEGORY,
+        codes: ['Hub shutdown'],
+        phrases: [],
+    },
+    {
+        // Before SESSION_LOST: a proxied connect that fails with EMFILE ("Could
+        // not proxy command ... connect EMFILE") is the server out of files, not
+        // the helper gone. Not a bare "EMFILE": that is a substring of a locator
+        // such as "systemFileList".
+        category: 'SYSTEM_OVERLOAD',
+        codes: [],
+        phrases: ['OutOfMemory', 'too many open files', 'connect EMFILE'],
+    },
+    {
+        // The phone's automation helper (WebDriverAgent, the UiAutomator2
+        // server, Chromedriver) or the session itself went away during the run.
+        category: 'SESSION_LOST',
+        codes: ['invalid session id'],
+        phrases: [
+            // base-driver's proxy, when the helper doesn't answer at all (socket
+            // hang up, ECONNREFUSED, a read time-out).
+            'Could not proxy command to the remote server',
+            // UiAutomator2: its server's instrumentation exited ("probably crashed"
+            // is the helper, not the app), or the phone went away.
+            'instrumentation process is not running',
+            // WebDriverAgent restarted and lost the session.
+            'Session does not exist',
+            'A session is either terminated or not started',
+            // Appium's own, when a command races the session's end.
+            'The session with id',
+            // Appium ended the session itself (onUnexpectedShutdown).
+            'unexpectedly shut down',
+            'Driver shut down unexpectedly',
+            'Chromedriver quit unexpectedly',
+            // XCUITest's commandTimeouts, which then ends the session.
+            'Appium did not get any response from',
+            // Xenon lost the session: OrphanSweeper, SessionHeartbeatService, and
+            // the hub's answer for a node it can't reach (forwardToNode).
+            'Session heartbeat timeout',
+            'Session terminal failure',
+            'The node no longer has this session',
+            'Node session status failed',
+            'could not reach the node running this session',
+        ],
+    },
+    {
+        category: 'APP_CRASH',
+        codes: [],
+        phrases: [
+            // WebDriverAgent, when the app under test is gone.
+            'is not running, possibly crashed',
+            // A test's own words.
+            'The application has crashed',
+            'Application not responding',
+            'process has died',
+            'activity has died',
+        ],
+    },
+    {
+        category: 'STALE_ELEMENT',
+        codes: ['stale element reference'],
+        phrases: [
+            // base-driver's and WebDriverAgent's default, and the UiAutomator2
+            // server's.
+            'no longer attached to the DOM',
+            // The UiAutomator2 server's cache. "not present in the cache or has
+            // expired" is answered as "no such element", hence this rule first.
+            'does not exist in DOM anymore',
+            'do not exist in DOM anymore',
+            'is not linked to the same object in DOM anymore',
+            'is not present in the cache or has expired',
+            'StaleObjectException',
+            // WebDriverAgent's cache and snapshots.
+            'expired from the internal cache',
+            'is not present in the current view anymore',
+            // Safari and web views on an iPhone.
+            'Element does not exist in cache',
+        ],
+    },
+    {
+        // Specific phrases only: a bare "timeout" also matched Xenon's own
+        // autowait misses, a hung helper, heartbeat reasons and command titles.
+        category: 'TIMEOUT',
+        codes: ['timeout', 'script timeout'],
+        phrases: [
+            'New Command Timeout of',
+            'timed out due to inactivity',
+            // base-driver's defaults for `timeout` and `script timeout`.
+            'did not complete before its timeout expired',
+            'Timed out waiting for asynchronous script result',
+            // XCUITest and WebDriverAgent.
+            'waiting for XCTest to complete',
+            'cannot be launched within',
+            'Timed out while waiting until the screen gets locked',
+            'Did not receive any expected',
+            // UiAutomator2, when the app never lets the screen go idle (an endless
+            // animation, a video, a busy main thread) and it can't be read.
+            'hogging the main UI thread',
+            // A test's own words (Selenium's exception).
+            'TimeoutException',
+        ],
+    },
+    {
+        category: 'ELEMENT_NOT_FOUND',
+        codes: ['no such element'],
+        phrases: [
+            'An element could not be located',
+            // WebDriverAgent's identifier and predicate lookups.
+            "didn't match any elements",
+            // A test's own words (NoSuchElementException).
+            'NoSuchElement',
+        ],
+    },
+    {
+        // No native driver reports a blocking prompt: it shows up as Element not
+        // found. Only web pages on an iPhone, and a test's own words.
+        category: 'PERMISSION_BLOCKED',
+        codes: ['unexpected alert open'],
+        phrases: [
+            'A modal dialog was open',
+            'Permission alert',
+            'Security alert',
+            'Always Allow',
+            'Allow while using app',
+        ],
+    },
+];
+/** Every category the failure analysis writes, upper case as stored. */
+exports.ANALYSIS_CATEGORIES = [
+    ...exports.FAILURE_RULES.map((r) => r.category),
+    exports.UNMATCHED_CATEGORY,
+];
+/**
+ * Categories no longer written, which sessions filed through 2.15 still hold.
+ * Their runbooks stay. WDA_FAILURE's live cases are SESSION_LOST now, and
+ * XENON_COMMAND_FAILURE matched nothing real but base-driver's default
+ * stale-element text, which is STALE_ELEMENT.
+ */
+exports.RETIRED_CATEGORIES = ['WDA_FAILURE', 'XENON_COMMAND_FAILURE'];
+/**
+ * The category of a message Xenon itself throws, by how it starts. These
+ * quote the test's selector, which may hold any rule's phrase ("Allow
+ * notifications to be enabled"), so they are read before the rules.
+ */
+function xenonOwnCategory(text) {
+    const t = text.trim();
+    // CommandInterceptor's autowait: an element that never became enabled, or
+    // one that was never found.
+    if (t.startsWith('Autowait timed out after ')) {
+        return t.endsWith(' to be enabled') ? 'TIMEOUT' : 'ELEMENT_NOT_FOUND';
+    }
+    // OmniVision's miss, and an element id Xenon doesn't hold.
+    if (t.startsWith('Xenon found nothing on the screen matching '))
+        return 'ELEMENT_NOT_FOUND';
+    if (t.startsWith('Xenon has no element '))
+        return 'ELEMENT_NOT_FOUND';
+    return undefined;
+}
+/**
+ * The error and message of a failed command's recorded answer, and nothing
+ * else: never its stack trace, which names netty's IdleStateHandler on every
+ * UiAutomator2 error and the driver on every error.
+ */
+function commandErrorOf(response) {
+    var _a;
+    if (!response)
+        return {};
+    try {
+        const value = (_a = JSON.parse(response)) === null || _a === void 0 ? void 0 : _a.value;
+        if (!value || typeof value !== 'object')
+            return {};
+        return {
+            error: typeof value.error === 'string' ? value.error : undefined,
+            message: typeof value.message === 'string' ? value.message : undefined,
+        };
+    }
+    catch (_b) {
+        return {};
+    }
+}
+const norm = (s) => s.trim().toLowerCase();
+function ruleFor(texts, codes) {
+    for (const t of texts) {
+        const own = xenonOwnCategory(t);
+        if (own)
+            return own;
+    }
+    const haystack = texts.map(norm);
+    const codeSet = new Set(codes.map(norm));
+    for (const rule of exports.FAILURE_RULES) {
+        if (rule.codes.some((c) => codeSet.has(norm(c))))
+            return rule.category;
+        if (rule.phrases.some((p) => haystack.some((t) => t.includes(norm(p)))))
+            return rule.category;
+    }
+    return undefined;
+}
+/**
+ * A failed session's category, from its failure reason and its failed
+ * commands (newest first).
+ *
+ * The failure that ended the session decides alone: the reason, read with the
+ * newest failed command when the reason was taken from it (for a session on
+ * a node's phone the reason is that command's bare code, "no such element",
+ * and its message says more). A reason no rule matches is UNKNOWN: an earlier
+ * failure the test recovered from, or a find it expected to fail, doesn't
+ * decide. Only a session with no reason at all is read from its failed
+ * commands, newest first.
+ */
+function categorizeFailure(reason, failedCommands) {
+    const said = (c) => [c.error, c.message].filter((s) => !!s);
+    const codesOf = (c) => (c.error ? [c.error] : []);
+    const r = reason.trim();
+    if (r) {
+        const newest = failedCommands[0];
+        const fromNewest = !!newest && said(newest).some((s) => norm(s) === norm(r));
+        const category = fromNewest
+            ? ruleFor([r, ...said(newest)], codesOf(newest))
+            : ruleFor([r], [r]);
+        return category !== null && category !== void 0 ? category : exports.UNMATCHED_CATEGORY;
+    }
+    for (const c of failedCommands) {
+        const category = ruleFor(said(c), codesOf(c));
+        if (category)
+            return category;
+    }
+    return exports.UNMATCHED_CATEGORY;
+}

package/lib/src/device-managers/AndroidDeviceManager.js

@@ -90,6 +90,11 @@ const typedi_2 = require("typedi");
  * they had worked. Within the 15 minutes a hub gives a node's install.
  */
 exports.ANDROID_INSTALL_TIMEOUT_MS = 10 * 60000;
+/** How long after adb, or its device tracking, failed to start it is tried again. */
+const ADB_RETRY_MS = 60000;
+function reasonOf(e) {
+    return e instanceof Error ? e.message : String(e);
+}
 /**
  * Whether a server set to `androidDeviceType` serves this phone: real phones,
  * emulators ('simulated'), or both, and with `bootedEmulators` only the
@@ -110,6 +115,11 @@ let AndroidDeviceManager = class AndroidDeviceManager {
         this.context = context;
         this.log = logger_1.default.scope('AndroidManager');
         this.adbAvailable = true;
+        // When adb last failed to start, discovery answers no phones until this
+        // time, then tries again.
+        this.adbRetryAt = 0;
+        // When adb's device tracking last failed to start, when to try it again.
+        this.trackerRetryAt = 0;
         this.abortControl = new Map();
         this.tracker = undefined;
         this.remoteTrackers = [];
@@ -161,8 +171,11 @@ let AndroidDeviceManager = class AndroidDeviceManager {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
             if (!this.adbAvailable) {
-                logger_1.default.info('adb is not available. So, returning empty list');
-                return [];
+                if (Date.now() < this.adbRetryAt) {
+                    logger_1.default.debug('adb is not available. So, returning empty list');
+                    return [];
+                }
+                this.adbAvailable = true;
             }
             let devices = [];
             try {
@@ -401,32 +414,57 @@ let AndroidDeviceManager = class AndroidDeviceManager {
     }
     getAdb() {
         return __awaiter(this, void 0, void 0, function* () {
-            try {
-                if (!this.adb) {
-                    try {
-                        this.adb = yield appium_adb_1.ADB.createADB({});
-                    }
-                    catch (e) {
-                        this.adbAvailable = false;
-                        this.log.error('Could not find ADB');
-                    }
-                    if (process.env.NODE_ENV !== 'test') {
-                        const client = adbkit_1.default.createClient();
-                        this.tracker = yield client.trackDevices();
-                        if (this.tracker && this.adb) {
-                            const originalADBTracking = this.createLocalAdbTracker(this.tracker, this.adb);
-                            yield originalADBTracking();
-                        }
-                    }
+            // One start at a time: calls that come while adb or its tracker is
+            // starting wait for that start. Two discoveries at once each started a
+            // tracker, and each tracker handled every plug event again.
+            if (!this.adbStarting) {
+                this.adbStarting = this.startAdb().finally(() => {
+                    this.adbStarting = undefined;
+                });
+            }
+            yield this.adbStarting;
+            return { adbInstance: this.adb, adbTracker: this.tracker };
+        });
+    }
+    startAdb() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.adb) {
+                try {
+                    this.adb = yield appium_adb_1.ADB.createADB({});
+                }
+                catch (e) {
+                    this.adbUnavailable('Could not start adb', e);
                 }
             }
-            catch (e) {
-                logger_1.default.error(`Failed to initialize ADB: ${e}`);
-                this.adbAvailable = false;
+            // Plug events. Without them discovery still lists phones each time it
+            // runs, so a failure here hides none; it is tried again a minute later.
+            if (this.adb &&
+                !this.tracker &&
+                process.env.NODE_ENV !== 'test' &&
+                Date.now() >= this.trackerRetryAt) {
+                try {
+                    const tracker = yield adbkit_1.default.createClient().trackDevices();
+                    yield this.createLocalAdbTracker(tracker, this.adb)();
+                    this.tracker = tracker;
+                }
+                catch (e) {
+                    this.trackerRetryAt = Date.now() + ADB_RETRY_MS;
+                    this.log.error(`Could not follow Android plug events through adb: ${reasonOf(e)}. ` +
+                        `Trying again in ${ADB_RETRY_MS / 1000} s.`);
+                }
             }
-            return { adbInstance: this.adb, adbTracker: this.tracker };
         });
     }
+    /**
+     * Discovery answers no phones for a minute, then tries again: adb may work
+     * by then (an SDK installed, a stuck adb server restarted). Through 2.15 it
+     * never tried again before a restart, and didn't say why.
+     */
+    adbUnavailable(what, e) {
+        this.adbAvailable = false;
+        this.adbRetryAt = Date.now() + ADB_RETRY_MS;
+        this.log.error(`${what}: ${reasonOf(e)}. Trying again in ${ADB_RETRY_MS / 1000} s.`);
+    }
     getAdbForDevice(udid) {
         return __awaiter(this, void 0, void 0, function* () {
             const { adbInstance } = yield this.getAdb();

package/lib/src/device-managers/IOSDeviceManager.js

@@ -266,6 +266,16 @@ let IOSDeviceManager = class IOSDeviceManager {
             var _a;
             if (device.cloud)
                 return true;
+            // A simulator's WebDriverAgent is the XCUITest driver's: it builds,
+            // installs and launches it on the session's wdaLocalPort (iOSCapabilities)
+            // once Appium creates the session, after this check, and boots a shut-down
+            // simulator itself. Xenon runs WDA only on an iPhone (go-ios runwda), so
+            // asking for it here refused every simulator whose WDA wasn't already up,
+            // and recoverHealth's reboot never starts one. Through 2.15 it did.
+            if (!device.realDevice) {
+                this.log.info(`[SessionStart] ${device.udid} is a simulator: the driver starts its WebDriverAgent`);
+                return true;
+            }
             this.log.info(`🚀 [SessionStart] Verifying readiness for ${device.udid}...`);
             const streamService = typedi_1.Container.get(IOSStreamService_1.default);
             const streamStatus = streamService.getStreamStatus(device.udid);

package/lib/src/device-utils.js

@@ -87,6 +87,7 @@ const device_managers_1 = require("./device-managers");
 const device_service_1 = require("./data-service/device-service");
 const deviceClaims_1 = require("./data-service/deviceClaims");
 const PluginContext_1 = require("./PluginContext");
+const TracingService_1 = require("./services/TracingService");
 const logger_1 = __importDefault(require("./logger"));
 const Platform_1 = __importDefault(require("./enums/Platform"));
 const lodash_1 = __importDefault(require("lodash"));
@@ -250,7 +251,7 @@ function allocateDeviceForSession(capability_1, deviceTimeOutMs_1, deviceQueryIn
                         if (device !== null)
                             return true;
                     }
-                    logger_1.default.info(`Waiting for free device. Filter: ${JSON.stringify(filters)}}`);
+                    logger_1.default.info(`Waiting for free device. Filter: ${JSON.stringify(filters)}`);
                     return false;
                 }));
             }), { timeout, intervalBetweenAttempts });
@@ -269,13 +270,22 @@ function allocateDeviceForSession(capability_1, deviceTimeOutMs_1, deviceQueryIn
             else if (possibleDevice && (0, device_service_1.isDeviceReserved)(possibleDevice)) {
                 failureReason = `Device is reserved by ${possibleDevice.reservedBy}.`;
             }
-            throw new Error(`${failureReason}. Device request: ${JSON.stringify(filterCopy)}`);
+            throw new Error(`${failureReason} Device request: ${JSON.stringify(filterCopy)}`);
         }
         // We have a locked device here
         if (device !== null) {
             const lockedDevice = device;
-            // Principal Health Check Integration: Ensure device is READY before allocation
-            if (!lockedDevice.cloud) {
+            // Ready for a session: asked only of a phone this server drives. The
+            // check is this machine's (an iPhone's WebDriverAgent on 127.0.0.1, and a
+            // go-ios stream started to recover it). Another server's phone, a node's
+            // on a hub, is checked by that server when it allocates the phone the hub
+            // pins. Through 2.15 the hub checked it too: with the node on another
+            // machine nothing answered and the session was refused as unhealthy, and
+            // with both on one Mac the hub started a WebDriverAgent of its own on the
+            // node's iPhone.
+            const context = typedi_1.Container.get(PluginContext_1.PluginContext);
+            const ownPhone = (0, localDeviceHosts_1.isOwnDevice)((0, localDeviceHosts_1.localDeviceHosts)(context.pluginArgs, context.port), context.nodeId, lockedDevice);
+            if (!lockedDevice.cloud && ownPhone) {
                 const platform = lockedDevice.platform.toLowerCase();
                 const managers = yield getDeviceManager().deviceInstances();
                 const manager = managers.find((m) => {
@@ -648,7 +658,7 @@ function unblockCandidateDevices() {
 }
 function releaseBlockedDevices(newCommandTimeout) {
     return __awaiter(this, void 0, void 0, function* () {
-        var _a, _b, _c;
+        var _a, _b;
         const busyDevices = yield unblockCandidateDevices();
         // A phone an SDK lease holds is the lease's to time out (its heartbeats and
         // expiry, LeaseOrphanSweeper), not idle until a session runs on it. A lease
@@ -696,9 +706,16 @@ function releaseBlockedDevices(newCommandTimeout) {
             if (timeSinceLastCmdExecuted > timeoutSeconds) {
                 // unblock regardless of whether the device has session or not
                 logger_1.default.info(`Unblocking device ${device.udid} at host ${device.host} because it has been idle for ${timeSinceLastCmdExecuted} seconds`);
+                const idleSession = (_a = device.claimSessionId) !== null && _a !== void 0 ? _a : device.session_id;
                 // Before anything below can release the phone (onSessionStopped too).
-                yield restoreIdleSessionNetwork((_a = device.claimSessionId) !== null && _a !== void 0 ? _a : device.session_id);
-                yield forgetIdleSessionMemory((_b = device.claimSessionId) !== null && _b !== void 0 ? _b : device.session_id);
+                yield restoreIdleSessionNetwork(idleSession);
+                yield forgetIdleSessionMemory(idleSession);
+                if (idleSession) {
+                    typedi_1.Container.get(TracingService_1.TracingService).endSessionSpan(idleSession, {
+                        failed: true,
+                        reason: 'Session timed out due to inactivity',
+                    });
+                }
                 // Principal Protection: If this device has an active dashboard session, stop it properly
                 if (device.session_id) {
                     const sessionId = device.session_id;
@@ -714,7 +731,7 @@ function releaseBlockedDevices(newCommandTimeout) {
                 }
                 // Keyed on the session read here, so a release that loses the race to
                 // the session's own end (and the phone's next claim) frees nothing.
-                const claimant = (_c = device.claimSessionId) !== null && _c !== void 0 ? _c : device.session_id;
+                const claimant = (_b = device.claimSessionId) !== null && _b !== void 0 ? _b : device.session_id;
                 yield (claimant
                     ? (0, device_service_1.releaseSessionDevice)(device.udid, device.host, claimant)
                     : (0, device_service_1.unblockDevice)(device.udid, device.host)).catch((err) => logger_1.default.error(`Unable to release ${device.udid}: ${err}`));

package/lib/src/gateway/forwardToNode.js

@@ -11,11 +11,10 @@ exports.readAnswer = readAnswer;
 exports.sendAnswer = sendAnswer;
 const http_1 = __importDefault(require("http"));
 const https_1 = __importDefault(require("https"));
-const https_proxy_agent_1 = require("https-proxy-agent");
-const http_proxy_agent_1 = require("http-proxy-agent");
 const internalCall_1 = require("./internalCall");
 const hubSessionToken_1 = require("./hubSessionToken");
 const healReport_1 = require("./healReport");
+const outboundProxy_1 = require("../helpers/outboundProxy");
 /**
  * Sending a session command on to the node that runs the session, and handing
  * the node's answer back to the client unchanged.
@@ -60,17 +59,11 @@ function forwardedRequestHeaders(incoming) {
     out['accept-encoding'] = 'identity';
     return out;
 }
-/** HTTP(S)_PROXY from the environment, as the old command proxy honoured it. */
-function proxyAgentFor(url) {
-    const proxy = process.env.HTTP_PROXY || process.env.HTTPS_PROXY;
-    if (!proxy)
-        return undefined;
-    return url.protocol === 'https:' ? new https_proxy_agent_1.HttpsProxyAgent(proxy) : new http_proxy_agent_1.HttpProxyAgent(proxy);
-}
 /**
  * Send one request to a node and resolve with its response stream. No retry:
  * a WebDriver command is not safe to repeat. Credentials in the URL (a cloud
- * provider's) become basic auth.
+ * provider's) become basic auth. Through the environment's proxy for the URL,
+ * by axios's rule (helpers/outboundProxy.ts).
  */
 function sendToNode(request) {
     const url = new URL(request.url);
@@ -80,7 +73,7 @@ function sendToNode(request) {
     if (Buffer.isBuffer(body))
         headers['content-length'] = body.length;
     return new Promise((resolve, reject) => {
-        const outgoing = lib.request(url, { method: request.method, headers, agent: proxyAgentFor(url), signal: request.signal }, resolve);
+        const outgoing = lib.request(url, { method: request.method, headers, agent: (0, outboundProxy_1.proxyAgentFor)(url), signal: request.signal }, resolve);
         outgoing.on('error', reject);
         if (body && !Buffer.isBuffer(body)) {
             body.on('error', (err) => outgoing.destroy(err));