@xenon-device-management/xenon 2.13.2 → 2.15.0

package/lib/src/device-utils.js

@@ -57,7 +57,7 @@ exports.initializeStorage = initializeStorage;
 exports.getUtilizationTime = getUtilizationTime;
 exports.setUtilizationTime = setUtilizationTime;
 exports.getDeviceFiltersFromCapability = getDeviceFiltersFromCapability;
-exports.getBusyDevicesCount = getBusyDevicesCount;
+exports.getRunningSessionsCount = getRunningSessionsCount;
 exports.updateDeviceList = updateDeviceList;
 exports.refreshSimulatorState = refreshSimulatorState;
 exports.setupCronCheckStaleDevices = setupCronCheckStaleDevices;
@@ -72,6 +72,7 @@ exports.cleanPendingSessions = cleanPendingSessions;
 exports.setupCronCleanPendingSessions = setupCronCleanPendingSessions;
 exports.setupCronCleanExpiredReservations = setupCronCleanExpiredReservations;
 exports.setupCronCleanupBuilds = setupCronCleanupBuilds;
+exports.rescheduleCleanupBuilds = rescheduleCleanupBuilds;
 exports.setupCronReconcileDevices = setupCronReconcileDevices;
 exports.setupCronSweepOrphanSessions = setupCronSweepOrphanSessions;
 exports.setupCronSelectorVerification = setupCronSelectorVerification;
@@ -80,6 +81,7 @@ exports.stopAllTimers = stopAllTimers;
 const helpers_1 = require("./helpers");
 const XenonCapabilityManager_1 = require("./XenonCapabilityManager");
 const async_wait_until_1 = __importDefault(require("async-wait-until"));
+const async_lock_1 = __importDefault(require("async-lock"));
 const typedi_1 = require("typedi");
 const device_managers_1 = require("./device-managers");
 const device_service_1 = require("./data-service/device-service");
@@ -152,6 +154,9 @@ function isDeviceConfigPathAbsolute(path) {
         throw new Error(`Device Config Path ${path} should be absolute`);
     }
 }
+// One at a time through "is there a free slot, and which phone": see allocateDeviceForSession.
+const ALLOCATION_LOCK = new async_lock_1.default();
+const ALLOCATION_LOCK_KEY = 'allocate';
 // What a caller that passes no proof has: nothing. A lease then needs its token.
 const NO_LEASE_PROOF = {
     canOverride: false,
@@ -212,33 +217,42 @@ function allocateDeviceForSession(capability_1, deviceTimeOutMs_1, deviceQueryIn
             yield (0, async_wait_until_1.default)(() => __awaiter(this, void 0, void 0, function* () {
                 if (abandoned)
                     return false;
-                const maxSessions = getDeviceManager().getMaxSessionCount();
-                const busyDevicesCount = yield getBusyDevicesCount();
-                if (maxSessions !== undefined && busyDevicesCount === maxSessions) {
-                    logger_1.default.info(`Waiting for session available, already at max session count of: ${maxSessions}`);
-                    return false;
-                }
-                // Get matching devices that are not reserved
-                const candidates = (yield (0, device_service_1.getDevices)(filters)).filter((d) => !(0, device_service_1.isDeviceReserved)(d));
-                if (candidates.length > 0) {
-                    // Principal Intelligence: Multi-node consistent locking
-                    // Atomically claim one of them: busy, with a pending claim for the
-                    // session being created (deviceClaims.ts). Every candidate, not just
-                    // the first: the lock skips a phone an SDK lease holds, and offering
-                    // it alone stalled the create while other phones were free. Exact
-                    // (udid, host) pairs: a udid repeats across hosts, and a reserved
-                    // twin of a candidate must not be locked in its place.
-                    const locked = yield store.findAndLockDevice(Object.assign(Object.assign({}, filters), { udid: [...new Set(candidates.map((d) => d.udid))] }), { claim: true, only: new Set(candidates.map((d) => `${d.udid}@${d.host}`)) });
-                    if (locked && abandoned) {
-                        yield (0, device_service_1.releasePendingClaim)(locked);
+                // The limit check and the claim are one step. Requests that each read
+                // "one slot left" before any of them claimed would all take a phone and
+                // run past the limit, and then no later check would see it fixed.
+                return ALLOCATION_LOCK.acquire(ALLOCATION_LOCK_KEY, () => __awaiter(this, void 0, void 0, function* () {
+                    if (abandoned)
                         return false;
+                    const maxSessions = (0, deviceClaims_1.sessionCap)(getDeviceManager().getMaxSessionCount());
+                    if (maxSessions !== undefined) {
+                        const running = yield getRunningSessionsCount();
+                        if (running >= maxSessions) {
+                            logger_1.default.info(`Waiting for session available, already at max session count of: ${maxSessions}`);
+                            return false;
+                        }
                     }
-                    device = locked;
-                    if (device !== null)
-                        return true;
-                }
-                logger_1.default.info(`Waiting for free device. Filter: ${JSON.stringify(filters)}}`);
-                return false;
+                    // Get matching devices that are not reserved
+                    const candidates = (yield (0, device_service_1.getDevices)(filters)).filter((d) => !(0, device_service_1.isDeviceReserved)(d));
+                    if (candidates.length > 0) {
+                        // Principal Intelligence: Multi-node consistent locking
+                        // Atomically claim one of them: busy, with a pending claim for the
+                        // session being created (deviceClaims.ts). Every candidate, not just
+                        // the first: the lock skips a phone an SDK lease holds, and offering
+                        // it alone stalled the create while other phones were free. Exact
+                        // (udid, host) pairs: a udid repeats across hosts, and a reserved
+                        // twin of a candidate must not be locked in its place.
+                        const locked = yield store.findAndLockDevice(Object.assign(Object.assign({}, filters), { udid: [...new Set(candidates.map((d) => d.udid))] }), { claim: true, only: new Set(candidates.map((d) => `${d.udid}@${d.host}`)) });
+                        if (locked && abandoned) {
+                            yield (0, device_service_1.releasePendingClaim)(locked);
+                            return false;
+                        }
+                        device = locked;
+                        if (device !== null)
+                            return true;
+                    }
+                    logger_1.default.info(`Waiting for free device. Filter: ${JSON.stringify(filters)}}`);
+                    return false;
+                }));
             }), { timeout, intervalBetweenAttempts });
         }
         catch (err) {
@@ -446,12 +460,14 @@ function getDeviceFiltersFromCapability(capability, pluginArgs) {
     if ((deviceType === null || deviceType === void 0 ? void 0 : deviceType.startsWith('real')) && pluginArgs.iosDeviceType == 'simulated') {
         throw new Error('iosDeviceType value is set to "simulated" but app provided is not suitable for simulator device.');
     }
-    let name = undefined;
+    // iPad wins when both are asked for. The device store applies it; it is not
+    // a name match (a real phone is named by its owner), see appleFamilyOf.
+    let appleFamily = undefined;
     if (capability[customCapability.ipadOnly]) {
-        name = 'iPad';
+        appleFamily = 'ipad';
     }
     else if (capability[customCapability.iphoneOnly]) {
-        name = 'iPhone';
+        appleFamily = 'iphone';
     }
     // Ensure udid is always an array of strings for the filter
     let udidFilter = [];
@@ -469,7 +485,6 @@ function getDeviceFiltersFromCapability(capability, pluginArgs) {
         platformVersion: capability['appium:platformVersion']
             ? capability['appium:platformVersion']
             : undefined,
-        name,
         deviceType,
         udid: udidFilter,
         busy: false,
@@ -481,8 +496,8 @@ function getDeviceFiltersFromCapability(capability, pluginArgs) {
             ? capability[customCapability.tags].split(',').map(lodash_1.default.trim)
             : undefined,
     };
-    if (name !== undefined) {
-        caps = Object.assign(Object.assign({}, caps), { name });
+    if (appleFamily !== undefined) {
+        caps = Object.assign(Object.assign({}, caps), { appleFamily });
     }
     return caps;
 }
@@ -492,12 +507,16 @@ function getDeviceFiltersFromCapability(capability, pluginArgs) {
 function getDeviceManager() {
     return typedi_1.Container.get(device_managers_1.XenonManager);
 }
-function getBusyDevicesCount() {
+/**
+ * How many phones run an Appium session, or are being given one: what
+ * `maxSessions` limits. A preview, a recording or an idle SDK lease keeps a
+ * phone busy without being a session, so counting busy phones would be wrong
+ * (see `countsTowardMaxSessions`). On a hub it covers its nodes' phones too.
+ */
+function getRunningSessionsCount() {
     return __awaiter(this, void 0, void 0, function* () {
         const allDevices = yield (0, device_service_1.getAllDevices)();
-        return allDevices.filter((device) => {
-            return device.busy;
-        }).length;
+        return allDevices.filter(deviceClaims_1.countsTowardMaxSessions).length;
     });
 }
 function updateDeviceList(local, hubArgument, tlsRejectUnauthorized) {
@@ -629,7 +648,7 @@ function unblockCandidateDevices() {
 }
 function releaseBlockedDevices(newCommandTimeout) {
     return __awaiter(this, void 0, void 0, function* () {
-        var _a;
+        var _a, _b, _c;
         const busyDevices = yield unblockCandidateDevices();
         // A phone an SDK lease holds is the lease's to time out (its heartbeats and
         // expiry, LeaseOrphanSweeper), not idle until a session runs on it. A lease
@@ -677,6 +696,9 @@ function releaseBlockedDevices(newCommandTimeout) {
             if (timeSinceLastCmdExecuted > timeoutSeconds) {
                 // unblock regardless of whether the device has session or not
                 logger_1.default.info(`Unblocking device ${device.udid} at host ${device.host} because it has been idle for ${timeSinceLastCmdExecuted} seconds`);
+                // Before anything below can release the phone (onSessionStopped too).
+                yield restoreIdleSessionNetwork((_a = device.claimSessionId) !== null && _a !== void 0 ? _a : device.session_id);
+                yield forgetIdleSessionMemory((_b = device.claimSessionId) !== null && _b !== void 0 ? _b : device.session_id);
                 // Principal Protection: If this device has an active dashboard session, stop it properly
                 if (device.session_id) {
                     const sessionId = device.session_id;
@@ -692,7 +714,7 @@ function releaseBlockedDevices(newCommandTimeout) {
                 }
                 // Keyed on the session read here, so a release that loses the race to
                 // the session's own end (and the phone's next claim) frees nothing.
-                const claimant = (_a = device.claimSessionId) !== null && _a !== void 0 ? _a : device.session_id;
+                const claimant = (_c = device.claimSessionId) !== null && _c !== void 0 ? _c : device.session_id;
                 yield (claimant
                     ? (0, device_service_1.releaseSessionDevice)(device.udid, device.host, claimant)
                     : (0, device_service_1.unblockDevice)(device.udid, device.host)).catch((err) => logger_1.default.error(`Unable to release ${device.udid}: ${err}`));
@@ -700,6 +722,41 @@ function releaseBlockedDevices(newCommandTimeout) {
         }
     });
 }
+/**
+ * The network an idle session changed on its phone (profile, interceptor
+ * proxy), put back before the idle release frees the phone. Imported lazily,
+ * as this module's other session services are. Never throws.
+ */
+function restoreIdleSessionNetwork(sessionId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!sessionId)
+            return;
+        try {
+            const { PhoneNetworkRestore } = yield Promise.resolve().then(() => __importStar(require('./services/network/PhoneNetworkRestore')));
+            yield typedi_1.Container.get(PhoneNetworkRestore).restoreSession(sessionId, 'idle timeout');
+        }
+        catch (err) {
+            logger_1.default.warn(`Could not put back the network of idle session ${sessionId}: ${err}`);
+        }
+    });
+}
+/**
+ * The idle session's virtual elements and autowait settings
+ * (sessionMemory.ts). Imported lazily, as above. Never throws.
+ */
+function forgetIdleSessionMemory(sessionId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!sessionId)
+            return;
+        try {
+            const { forgetSessionMemory } = yield Promise.resolve().then(() => __importStar(require('./sessions/sessionMemory')));
+            forgetSessionMemory(sessionId);
+        }
+        catch (err) {
+            logger_1.default.warn(`Could not forget idle session ${sessionId}'s elements: ${err}`);
+        }
+    });
+}
 function setupCronReleaseBlockedDevices(intervalMs, newCommandTimeoutSec) {
     return __awaiter(this, void 0, void 0, function* () {
         if (cronTimerToReleaseBlockedDevices) {
@@ -805,11 +862,15 @@ function setupCronCleanExpiredReservations(intervalMs) {
     });
 }
 /**
- * Sets up a cron job to purge older builds and sessions based on configuration
+ * Sets up a cron job to purge older builds and sessions based on configuration.
+ * The schedule is the one saved on the dashboard's Maintenance page, else the
+ * `buildCleanupSchedule` option. Calling it again replaces the job, which is
+ * how a schedule saved at runtime takes effect (`rescheduleCleanupBuilds`).
  */
 function setupCronCleanupBuilds(pluginArgs) {
     return __awaiter(this, void 0, void 0, function* () {
-        const { buildCleanupSchedule = '0 0 * * *' } = pluginArgs;
+        const { loadEffectiveSettings } = yield Promise.resolve().then(() => __importStar(require('./services/settings/labSettings')));
+        const { buildCleanupSchedule } = yield loadEffectiveSettings(pluginArgs);
         const { CleanupService } = yield Promise.resolve().then(() => __importStar(require('./services/CleanupService')));
         const schedule = yield Promise.resolve().then(() => __importStar(require('node-schedule')));
         const cleanupService = typedi_1.Container.get(CleanupService);
@@ -823,6 +884,19 @@ function setupCronCleanupBuilds(pluginArgs) {
         }));
     });
 }
+/**
+ * Put a cleanup schedule just saved on the Maintenance page to work: the old
+ * timer is cancelled and a new one installed. A server that runs no cleanup
+ * (a cloud-provider hub, see `setupMaintenanceCrons`) stays without one.
+ */
+function rescheduleCleanupBuilds(pluginArgs) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        if ((_a = pluginArgs.cloud) === null || _a === void 0 ? void 0 : _a.cloudName)
+            return;
+        yield setupCronCleanupBuilds(pluginArgs);
+    });
+}
 let cronTimerSweepOrphanSessions;
 let cronTimerReconcileDevices;
 let cronTimerSelectorVerification;

package/lib/src/gateway/defaultGateway.js

@@ -34,7 +34,7 @@ const createWith = (hubGrants) => ({
 });
 const dashboardHooks = {
     before: (sessionId, command, req, res) => event_manager_1.DASHBORD_EVENT_MANAGER.beforeSessionCommand(sessionId, command, req, res),
-    after: (sessionId, command, req, res, body) => event_manager_1.DASHBORD_EVENT_MANAGER.afterSessionCommand(sessionId, command, null, req, res, body),
+    after: (sessionId, command, req, res, body, heal) => event_manager_1.DASHBORD_EVENT_MANAGER.afterSessionCommand(sessionId, command, null, req, res, body, heal),
 };
 /** A hub: sessions other servers run are forwarded to them. */
 function hubGatewayOptions(config) {

package/lib/src/gateway/forwardToNode.js

@@ -15,6 +15,7 @@ const https_proxy_agent_1 = require("https-proxy-agent");
 const http_proxy_agent_1 = require("http-proxy-agent");
 const internalCall_1 = require("./internalCall");
 const hubSessionToken_1 = require("./hubSessionToken");
+const healReport_1 = require("./healReport");
 /**
  * Sending a session command on to the node that runs the session, and handing
  * the node's answer back to the client unchanged.
@@ -44,6 +45,7 @@ const NEVER_FORWARDED = new Set([
     'x-xenon-token',
     internalCall_1.INTERNAL_CALL_HEADER,
     hubSessionToken_1.HUB_TOKEN_HEADER,
+    healReport_1.HEAL_REPORT_HEADER,
 ]);
 function forwardedRequestHeaders(incoming) {
     const out = {};

package/lib/src/gateway/healReport.js

@@ -0,0 +1,87 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HEAL_REPORT_HEADER = void 0;
+exports.runReportingHeals = runReportingHeals;
+exports.reportHeal = reportHeal;
+exports.takeHealReport = takeHealReport;
+const async_hooks_1 = require("async_hooks");
+/**
+ * A heal on a node's phone, on its way back to the hub.
+ *
+ * A session the hub created runs its commands on the node, so the node's
+ * CommandInterceptor heals its finds. The node keeps no record of such a
+ * session; the hub does. So the node puts the heal on its answer to the
+ * forwarded command, as the `x-xenon-heal` header, and the hub takes it off
+ * before relaying the answer and records it with the command, as it records a
+ * local session's heal.
+ *
+ * The node's session gateway runs a command that came from a hub inside
+ * `runReportingHeals`, so the interceptor, deep inside Appium's route, finds
+ * the answer to put it on. Anywhere else `reportHeal` does nothing, and the
+ * server records the heal itself.
+ */
+exports.HEAL_REPORT_HEADER = 'x-xenon-heal';
+/**
+ * A header longer than this isn't sent: the hub's HTTP parser refuses an
+ * answer whose headers pass 16 KB, and a refused answer would fail the
+ * command. A heal of selectors that long goes unrecorded instead.
+ */
+const MAX_HEADER_LENGTH = 8 * 1024;
+const answers = new async_hooks_1.AsyncLocalStorage();
+/** Run the rest of a command that came from a hub with its answer at hand. */
+function runReportingHeals(res, fn) {
+    const answer = { res };
+    const forget = () => {
+        answer.res = null;
+    };
+    res.once('finish', forget);
+    res.once('close', forget);
+    return answers.run(answer, fn);
+}
+/**
+ * Put `heal` on the answer to the hub's command this code runs for.
+ * `not-from-hub` when the command didn't come from a hub (or the answer has
+ * gone): the caller records the heal itself.
+ */
+function reportHeal(heal) {
+    var _a;
+    const res = (_a = answers.getStore()) === null || _a === void 0 ? void 0 : _a.res;
+    if (!res || res.headersSent)
+        return 'not-from-hub';
+    const encoded = Buffer.from(JSON.stringify(heal), 'utf8').toString('base64url');
+    if (encoded.length > MAX_HEADER_LENGTH)
+        return 'too-long';
+    res.setHeader(exports.HEAL_REPORT_HEADER, encoded);
+    return 'reported';
+}
+/**
+ * The heal a node put on its answer, taken off the headers so it isn't
+ * relayed to the client. Undefined when there is none, or it isn't one.
+ */
+function takeHealReport(headers) {
+    const raw = headers[exports.HEAL_REPORT_HEADER];
+    delete headers[exports.HEAL_REPORT_HEADER];
+    if (typeof raw !== 'string' || !raw)
+        return undefined;
+    try {
+        const heal = JSON.parse(Buffer.from(raw, 'base64url').toString('utf8'));
+        if (!heal ||
+            typeof heal.originalSelector !== 'string' ||
+            typeof heal.healedSelector !== 'string' ||
+            typeof heal.confidence !== 'number') {
+            return undefined;
+        }
+        const optional = (v) => (typeof v === 'string' ? v : undefined);
+        return {
+            originalSelector: heal.originalSelector,
+            originalStrategy: optional(heal.originalStrategy),
+            healedSelector: heal.healedSelector,
+            healedStrategy: optional(heal.healedStrategy),
+            confidence: heal.confidence,
+            tier: typeof heal.tier === 'number' ? heal.tier : undefined,
+        };
+    }
+    catch (_a) {
+        return undefined;
+    }
+}

package/lib/src/gateway/sessionGateway.js

@@ -15,6 +15,7 @@ exports.createHubRouting = createHubRouting;
 const commandAuth_1 = require("../middleware/commandAuth");
 const internalCall_1 = require("./internalCall");
 const hubSessionToken_1 = require("./hubSessionToken");
+const healReport_1 = require("./healReport");
 const forwardToNode_1 = require("./forwardToNode");
 /** Where a session runs could not be determined (a database error). */
 exports.ROUTING_UNAVAILABLE_STATUS = 503;
@@ -62,7 +63,14 @@ function createSessionGatewayLayer(deps) {
             const latency = (_b = (_a = deps.latencyOf) === null || _a === void 0 ? void 0 : _a.call(deps, sessionId)) !== null && _b !== void 0 ? _b : 0;
             if (latency > 0)
                 yield new Promise((resolve) => setTimeout(resolve, latency));
-            next();
+            // On a node, a heal of the hub's command goes back to the hub on the answer
+            // (healReport.ts). The header proves nothing elsewhere: a hub or a
+            // standalone server checks no hub token, so a client could send one there
+            // to keep its heals out of the record.
+            if (fromHub && deps.hubTokens)
+                (0, healReport_1.runReportingHeals)(res, next);
+            else
+                next();
         });
     }
     return function xenonSessionGateway(req, res, next) {
@@ -176,8 +184,10 @@ function createHubRouting(deps) {
                         abort.abort();
                 });
                 let captured;
+                let heal;
                 try {
                     const upstream = yield (0, forwardToNode_1.sendToNode)(yield nodeRequest(req, sessionId, location, abort.signal));
+                    heal = (0, healReport_1.takeHealReport)(upstream.headers);
                     captured = yield (0, forwardToNode_1.relayAnswer)(upstream, res, !!hooks);
                 }
                 catch (error) {
@@ -192,7 +202,7 @@ function createHubRouting(deps) {
                 }
                 if (hooks && captured !== undefined) {
                     yield hooks
-                        .after(sessionId, command, req, res, captured)
+                        .after(sessionId, command, req, res, captured, heal)
                         .catch((error) => logger.warn(`Dashboard log failed for session ${sessionId}: ${(0, commandAuth_1.summarize)(error)}`));
                 }
             });