@xenon-device-management/xenon 2.13.1 → 2.14.0

package/lib/src/device-utils.js

@@ -57,7 +57,7 @@ exports.initializeStorage = initializeStorage;
 exports.getUtilizationTime = getUtilizationTime;
 exports.setUtilizationTime = setUtilizationTime;
 exports.getDeviceFiltersFromCapability = getDeviceFiltersFromCapability;
-exports.getBusyDevicesCount = getBusyDevicesCount;
+exports.getRunningSessionsCount = getRunningSessionsCount;
 exports.updateDeviceList = updateDeviceList;
 exports.refreshSimulatorState = refreshSimulatorState;
 exports.setupCronCheckStaleDevices = setupCronCheckStaleDevices;
@@ -72,6 +72,7 @@ exports.cleanPendingSessions = cleanPendingSessions;
 exports.setupCronCleanPendingSessions = setupCronCleanPendingSessions;
 exports.setupCronCleanExpiredReservations = setupCronCleanExpiredReservations;
 exports.setupCronCleanupBuilds = setupCronCleanupBuilds;
+exports.rescheduleCleanupBuilds = rescheduleCleanupBuilds;
 exports.setupCronReconcileDevices = setupCronReconcileDevices;
 exports.setupCronSweepOrphanSessions = setupCronSweepOrphanSessions;
 exports.setupCronSelectorVerification = setupCronSelectorVerification;
@@ -80,6 +81,7 @@ exports.stopAllTimers = stopAllTimers;
 const helpers_1 = require("./helpers");
 const XenonCapabilityManager_1 = require("./XenonCapabilityManager");
 const async_wait_until_1 = __importDefault(require("async-wait-until"));
+const async_lock_1 = __importDefault(require("async-lock"));
 const typedi_1 = require("typedi");
 const device_managers_1 = require("./device-managers");
 const device_service_1 = require("./data-service/device-service");
@@ -99,6 +101,7 @@ const config_1 = require("./config");
 const device_store_1 = require("./data-service/device-store");
 const leaseSessionCaps_1 = require("./services/lease/leaseSessionCaps");
 const deviceVisibility_1 = require("./services/device-access/deviceVisibility");
+const activeLeases_1 = require("./services/lease/activeLeases");
 // Use a Proxy to ensure we're always using the latest store from the factory,
 // which is critical for test isolation when the factory cache is cleared.
 const pendingStore = new Proxy({}, {
@@ -151,6 +154,9 @@ function isDeviceConfigPathAbsolute(path) {
         throw new Error(`Device Config Path ${path} should be absolute`);
     }
 }
+// One at a time through "is there a free slot, and which phone": see allocateDeviceForSession.
+const ALLOCATION_LOCK = new async_lock_1.default();
+const ALLOCATION_LOCK_KEY = 'allocate';
 // What a caller that passes no proof has: nothing. A lease then needs its token.
 const NO_LEASE_PROOF = {
     canOverride: false,
@@ -182,6 +188,14 @@ function allocateDeviceForSession(capability_1, deviceTimeOutMs_1, deviceQueryIn
             // text or its stack tells them apart.
             if (!leasedDevice)
                 throw new Error((0, leaseSessionCaps_1.leaseRefusalMessage)(leaseIdCap));
+            // As for an allocated phone below: the idle sweeper reads this session's
+            // own timeout from the row, and its idle time from now, not from when the
+            // lease was taken. Without it a leased session was ended after the
+            // server-wide newCommandTimeoutSec, whatever the session asked for.
+            yield (0, device_service_1.updatedAllocatedDevice)(leasedDevice, {
+                newCommandTimeout: sessionNewCommandTimeout(firstMatch, pluginArgs),
+                lastCmdExecutedAt: Date.now(),
+            });
             return leasedDevice;
         }
         const filters = getDeviceFiltersFromCapability(firstMatch, pluginArgs);
@@ -195,29 +209,54 @@ function allocateDeviceForSession(capability_1, deviceTimeOutMs_1, deviceQueryIn
         const intervalBetweenAttempts = firstMatch[customCapability.deviceQueryInterval] || deviceQueryIntervalMs;
         let device = null;
         const store = device_store_1.DeviceStoreFactory.getStore();
+        // Set once the wait has given up. waitUntil rejects on its timeout but
+        // doesn't stop an attempt already running, which could then claim a phone
+        // for a create that has failed: a pending claim nobody would ever use.
+        let abandoned = false;
         try {
             yield (0, async_wait_until_1.default)(() => __awaiter(this, void 0, void 0, function* () {
-                const maxSessions = getDeviceManager().getMaxSessionCount();
-                const busyDevicesCount = yield getBusyDevicesCount();
-                if (maxSessions !== undefined && busyDevicesCount === maxSessions) {
-                    logger_1.default.info(`Waiting for session available, already at max session count of: ${maxSessions}`);
+                if (abandoned)
                     return false;
-                }
-                // Get matching devices and find one that is not reserved
-                const matchingDevices = yield (0, device_service_1.getDevices)(filters);
-                const availableCandidate = matchingDevices.find((d) => !(0, device_service_1.isDeviceReserved)(d));
-                if (availableCandidate) {
-                    // Principal Intelligence: Multi-node consistent locking
-                    // Attempt to atomically claim this specific device: busy, with a
-                    // pending claim for the session being created (deviceClaims.ts).
-                    device = yield store.findAndLockDevice(Object.assign(Object.assign({}, filters), { udid: [availableCandidate.udid] }), { claim: true });
-                    return device !== null;
-                }
-                logger_1.default.info(`Waiting for free device. Filter: ${JSON.stringify(filters)}}`);
-                return false;
+                // The limit check and the claim are one step. Requests that each read
+                // "one slot left" before any of them claimed would all take a phone and
+                // run past the limit, and then no later check would see it fixed.
+                return ALLOCATION_LOCK.acquire(ALLOCATION_LOCK_KEY, () => __awaiter(this, void 0, void 0, function* () {
+                    if (abandoned)
+                        return false;
+                    const maxSessions = (0, deviceClaims_1.sessionCap)(getDeviceManager().getMaxSessionCount());
+                    if (maxSessions !== undefined) {
+                        const running = yield getRunningSessionsCount();
+                        if (running >= maxSessions) {
+                            logger_1.default.info(`Waiting for session available, already at max session count of: ${maxSessions}`);
+                            return false;
+                        }
+                    }
+                    // Get matching devices that are not reserved
+                    const candidates = (yield (0, device_service_1.getDevices)(filters)).filter((d) => !(0, device_service_1.isDeviceReserved)(d));
+                    if (candidates.length > 0) {
+                        // Principal Intelligence: Multi-node consistent locking
+                        // Atomically claim one of them: busy, with a pending claim for the
+                        // session being created (deviceClaims.ts). Every candidate, not just
+                        // the first: the lock skips a phone an SDK lease holds, and offering
+                        // it alone stalled the create while other phones were free. Exact
+                        // (udid, host) pairs: a udid repeats across hosts, and a reserved
+                        // twin of a candidate must not be locked in its place.
+                        const locked = yield store.findAndLockDevice(Object.assign(Object.assign({}, filters), { udid: [...new Set(candidates.map((d) => d.udid))] }), { claim: true, only: new Set(candidates.map((d) => `${d.udid}@${d.host}`)) });
+                        if (locked && abandoned) {
+                            yield (0, device_service_1.releasePendingClaim)(locked);
+                            return false;
+                        }
+                        device = locked;
+                        if (device !== null)
+                            return true;
+                    }
+                    logger_1.default.info(`Waiting for free device. Filter: ${JSON.stringify(filters)}}`);
+                    return false;
+                }));
             }), { timeout, intervalBetweenAttempts });
         }
         catch (err) {
+            abandoned = true;
             // figure out whether the device is simply busy or non-existent
             const filterCopy = Object.assign({}, filters);
             delete filterCopy.busy;
@@ -258,11 +297,9 @@ function allocateDeviceForSession(capability_1, deviceTimeOutMs_1, deviceQueryIn
             // Since findAndLockDevice already sets busy: true, we don't need blockDevice(device.udid, device.host);
             logger_1.default.info(`📱 Locked device ${lockedDevice.udid} at host ${lockedDevice.host} for new session`);
             yield updateCapabilityForDevice(capability, lockedDevice);
-            let newCommandTimeout = firstMatch['appium:newCommandTimeout'];
-            if (!newCommandTimeout) {
-                newCommandTimeout = pluginArgs.newCommandTimeoutSec;
-            }
-            yield (0, device_service_1.updatedAllocatedDevice)(lockedDevice, { newCommandTimeout });
+            yield (0, device_service_1.updatedAllocatedDevice)(lockedDevice, {
+                newCommandTimeout: sessionNewCommandTimeout(firstMatch, pluginArgs),
+            });
             return lockedDevice;
         }
         else {
@@ -271,6 +308,10 @@ function allocateDeviceForSession(capability_1, deviceTimeOutMs_1, deviceQueryIn
         }
     });
 }
+/** The session's own `appium:newCommandTimeout` (seconds), else the server's. */
+function sessionNewCommandTimeout(firstMatch, pluginArgs) {
+    return firstMatch['appium:newCommandTimeout'] || pluginArgs.newCommandTimeoutSec;
+}
 /**
  * The device a lease-bound session may use, or null when it may not: the lease
  * is missing, inactive or expired, the session did not prove it holds it, or
@@ -419,12 +460,14 @@ function getDeviceFiltersFromCapability(capability, pluginArgs) {
     if ((deviceType === null || deviceType === void 0 ? void 0 : deviceType.startsWith('real')) && pluginArgs.iosDeviceType == 'simulated') {
         throw new Error('iosDeviceType value is set to "simulated" but app provided is not suitable for simulator device.');
     }
-    let name = undefined;
+    // iPad wins when both are asked for. The device store applies it; it is not
+    // a name match (a real phone is named by its owner), see appleFamilyOf.
+    let appleFamily = undefined;
     if (capability[customCapability.ipadOnly]) {
-        name = 'iPad';
+        appleFamily = 'ipad';
     }
     else if (capability[customCapability.iphoneOnly]) {
-        name = 'iPhone';
+        appleFamily = 'iphone';
     }
     // Ensure udid is always an array of strings for the filter
     let udidFilter = [];
@@ -442,7 +485,6 @@ function getDeviceFiltersFromCapability(capability, pluginArgs) {
         platformVersion: capability['appium:platformVersion']
             ? capability['appium:platformVersion']
             : undefined,
-        name,
         deviceType,
         udid: udidFilter,
         busy: false,
@@ -454,8 +496,8 @@ function getDeviceFiltersFromCapability(capability, pluginArgs) {
             ? capability[customCapability.tags].split(',').map(lodash_1.default.trim)
             : undefined,
     };
-    if (name !== undefined) {
-        caps = Object.assign(Object.assign({}, caps), { name });
+    if (appleFamily !== undefined) {
+        caps = Object.assign(Object.assign({}, caps), { appleFamily });
     }
     return caps;
 }
@@ -465,12 +507,16 @@ function getDeviceFiltersFromCapability(capability, pluginArgs) {
 function getDeviceManager() {
     return typedi_1.Container.get(device_managers_1.XenonManager);
 }
-function getBusyDevicesCount() {
+/**
+ * How many phones run an Appium session, or are being given one: what
+ * `maxSessions` limits. A preview, a recording or an idle SDK lease keeps a
+ * phone busy without being a session, so counting busy phones would be wrong
+ * (see `countsTowardMaxSessions`). On a hub it covers its nodes' phones too.
+ */
+function getRunningSessionsCount() {
     return __awaiter(this, void 0, void 0, function* () {
         const allDevices = yield (0, device_service_1.getAllDevices)();
-        return allDevices.filter((device) => {
-            return device.busy;
-        }).length;
+        return allDevices.filter(deviceClaims_1.countsTowardMaxSessions).length;
     });
 }
 function updateDeviceList(local, hubArgument, tlsRejectUnauthorized) {
@@ -602,8 +648,25 @@ function unblockCandidateDevices() {
 }
 function releaseBlockedDevices(newCommandTimeout) {
     return __awaiter(this, void 0, void 0, function* () {
-        var _a;
+        var _a, _b;
         const busyDevices = yield unblockCandidateDevices();
+        // A phone an SDK lease holds is the lease's to time out (its heartbeats and
+        // expiry, LeaseOrphanSweeper), not idle until a session runs on it. A lease
+        // sets lastCmdExecutedAt when taken, so without this the sweeper freed a
+        // leased phone the new-command timeout later, before a slow first session
+        // had even started on it.
+        let leases = new Map();
+        if (busyDevices.length) {
+            try {
+                leases = yield (0, activeLeases_1.activeLeasesByDevice)();
+            }
+            catch (err) {
+                // Not knowing which phones are leased, sweeping would free all of them:
+                // their lastCmdExecutedAt is when the lease was taken. Next tick.
+                logger_1.default.warn(`Idle sweep skipped: could not read active leases: ${err}`);
+                return;
+            }
+        }
         logger_1.default.debug(`Found ${busyDevices.length} device candidates to be released`);
         for (const device of busyDevices) {
             // need to keep this to make typescript happy. good thing tho.
@@ -622,11 +685,19 @@ function releaseBlockedDevices(newCommandTimeout) {
                 }
                 continue;
             }
+            // Leased, with no session or hold on it: not idle.
+            if (leases.has((0, activeLeases_1.leaseKey)(device.udid, device.host)) &&
+                !device.claimSessionId &&
+                !device.session_id) {
+                continue;
+            }
             const timeoutSeconds = device.newCommandTimeout != undefined ? device.newCommandTimeout : newCommandTimeout;
             const timeSinceLastCmdExecuted = (currentEpoch - device.lastCmdExecutedAt) / 1000;
             if (timeSinceLastCmdExecuted > timeoutSeconds) {
                 // unblock regardless of whether the device has session or not
                 logger_1.default.info(`Unblocking device ${device.udid} at host ${device.host} because it has been idle for ${timeSinceLastCmdExecuted} seconds`);
+                // Before anything below can release the phone (onSessionStopped too).
+                yield restoreIdleSessionNetwork((_a = device.claimSessionId) !== null && _a !== void 0 ? _a : device.session_id);
                 // Principal Protection: If this device has an active dashboard session, stop it properly
                 if (device.session_id) {
                     const sessionId = device.session_id;
@@ -642,7 +713,7 @@ function releaseBlockedDevices(newCommandTimeout) {
                 }
                 // Keyed on the session read here, so a release that loses the race to
                 // the session's own end (and the phone's next claim) frees nothing.
-                const claimant = (_a = device.claimSessionId) !== null && _a !== void 0 ? _a : device.session_id;
+                const claimant = (_b = device.claimSessionId) !== null && _b !== void 0 ? _b : device.session_id;
                 yield (claimant
                     ? (0, device_service_1.releaseSessionDevice)(device.udid, device.host, claimant)
                     : (0, device_service_1.unblockDevice)(device.udid, device.host)).catch((err) => logger_1.default.error(`Unable to release ${device.udid}: ${err}`));
@@ -650,6 +721,24 @@ function releaseBlockedDevices(newCommandTimeout) {
         }
     });
 }
+/**
+ * The network an idle session changed on its phone (profile, interceptor
+ * proxy), put back before the idle release frees the phone. Imported lazily,
+ * as this module's other session services are. Never throws.
+ */
+function restoreIdleSessionNetwork(sessionId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (!sessionId)
+            return;
+        try {
+            const { PhoneNetworkRestore } = yield Promise.resolve().then(() => __importStar(require('./services/network/PhoneNetworkRestore')));
+            yield typedi_1.Container.get(PhoneNetworkRestore).restoreSession(sessionId, 'idle timeout');
+        }
+        catch (err) {
+            logger_1.default.warn(`Could not put back the network of idle session ${sessionId}: ${err}`);
+        }
+    });
+}
 function setupCronReleaseBlockedDevices(intervalMs, newCommandTimeoutSec) {
     return __awaiter(this, void 0, void 0, function* () {
         if (cronTimerToReleaseBlockedDevices) {
@@ -755,11 +844,15 @@ function setupCronCleanExpiredReservations(intervalMs) {
     });
 }
 /**
- * Sets up a cron job to purge older builds and sessions based on configuration
+ * Sets up a cron job to purge older builds and sessions based on configuration.
+ * The schedule is the one saved on the dashboard's Maintenance page, else the
+ * `buildCleanupSchedule` option. Calling it again replaces the job, which is
+ * how a schedule saved at runtime takes effect (`rescheduleCleanupBuilds`).
  */
 function setupCronCleanupBuilds(pluginArgs) {
     return __awaiter(this, void 0, void 0, function* () {
-        const { buildCleanupSchedule = '0 0 * * *' } = pluginArgs;
+        const { loadEffectiveSettings } = yield Promise.resolve().then(() => __importStar(require('./services/settings/labSettings')));
+        const { buildCleanupSchedule } = yield loadEffectiveSettings(pluginArgs);
         const { CleanupService } = yield Promise.resolve().then(() => __importStar(require('./services/CleanupService')));
         const schedule = yield Promise.resolve().then(() => __importStar(require('node-schedule')));
         const cleanupService = typedi_1.Container.get(CleanupService);
@@ -773,6 +866,19 @@ function setupCronCleanupBuilds(pluginArgs) {
         }));
     });
 }
+/**
+ * Put a cleanup schedule just saved on the Maintenance page to work: the old
+ * timer is cancelled and a new one installed. A server that runs no cleanup
+ * (a cloud-provider hub, see `setupMaintenanceCrons`) stays without one.
+ */
+function rescheduleCleanupBuilds(pluginArgs) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        if ((_a = pluginArgs.cloud) === null || _a === void 0 ? void 0 : _a.cloudName)
+            return;
+        yield setupCronCleanupBuilds(pluginArgs);
+    });
+}
 let cronTimerSweepOrphanSessions;
 let cronTimerReconcileDevices;
 let cronTimerSelectorVerification;