@testim/testim-cli 3.289.0 → 3.290.1-beta

package/testRunHandler.js

@@ -1,603 +0,0 @@
-'use strict';
-
-const _ = require('lodash');
-const pRetry = require('p-retry');
-const utils = require('./utils');
-const config = require('./commons/config');
-const perf = require('./commons/performance-logger');
-const analytics = require('./commons/testimAnalytics');
-const testimCustomToken = require('./commons/testimCustomToken');
-const testimServicesApi = require('./commons/testimServicesApi');
-const remoteStepPlayback = require('./stepPlayers/remoteStepPlayback');
-const remoteStepService = require('./commons/socket/remoteStepService');
-const testResultService = require('./commons/socket/testResultService');
-const { URL } = require('url');
-const { getLogger } = require('./commons/logger');
-const { preloadTests } = require('./commons/preloadTests');
-const { timeoutMessages, CLI_MODE } = require('./commons/constants');
-const { SeleniumPerfStats } = require('./commons/SeleniumPerfStats');
-
-const logger = getLogger('test-run-handler');
-const RETRIES_ON_TIMEOUT = 3;
-const MAX_LIGHTWEIGHT_MODE_RUN_DATA_SIZE = 20 * 1000; // max size, in characters, of stringified run data sent over URL params. Chosen arbitrarily, this value should be changed according to data.
-const canSendRunDataOverUrl = (runData) => JSON.stringify(runData).length < MAX_LIGHTWEIGHT_MODE_RUN_DATA_SIZE;
-
-class TestRun {
-    /**
-     * @param {string} executionId
-     * @param {string} executionName
-     * @param {import('./runners/TestPlanRunner').ExecutionList[number]} test
-     * @param {import('./runOptions').RunnerOptions} options
-     * @param {string} branchToUse
-     * @param {import('./testRunStatus')} testRunStatus
-     */
-    constructor(executionId, executionName, test, options, branchToUse, testRunStatus) {
-        this._executionId = executionId;
-        this._executionName = executionName;
-        this._testStatus = test.testStatus;
-        this._testId = test.testId;
-        this._testName = test.name;
-        this._testResultId = test.resultId;
-        this._code = test.code;
-        this._baseUrl = options.baseUrl || test.baseUrl || test.testConfig.baseUrl;
-        this._nativeApp = test.nativeApp;
-        this._overrideTestConfigId = test.overrideTestConfig?.id;
-        this._options = options;
-        this._branch = branchToUse;
-        this._maxRetryCount = options.retries;
-        this._remoteRunId = options.remoteRunId;
-        this._retryCount = 1;
-        this._timeoutRetryCount = 1;
-        this._totalRetryCount = 1;
-
-        this._testRunStatus = testRunStatus;
-        const shouldUpdateConfig = !test.runConfig?.isMobileWeb && options.browser;
-        this._runConfig = shouldUpdateConfig ? utils.getRunConfigByBrowserName(options.browser, options.saucelabs, options.browserstack) : test.runConfig;
-        this.clearTestResultFinished = Promise.resolve();
-
-        this.seleniumPerfStats = new SeleniumPerfStats();
-    }
-
-    async waitForExecutionStartedFinished() {
-        return await this._testRunStatus.waitForExecutionStartedFinished() && await this.clearTestResultFinished;
-    }
-
-    get testStatus() {
-        return this._testStatus;
-    }
-
-    get runMode() {
-        return this._options.mode;
-    }
-
-    get automationMode() {
-        return this._code ? 'codeful' : 'codeless';
-    }
-
-    get code() {
-        return this._code;
-    }
-
-    get runConfig() {
-        return this._runConfig;
-    }
-
-    get testResultId() {
-        return this._testResultId;
-    }
-
-    get baseUrl() {
-        return this._baseUrl;
-    }
-
-    get executionId() {
-        return this._executionId;
-    }
-
-    get executionName() {
-        return this._executionName;
-    }
-
-    get androidActivityWait() {
-        const appData = 'appMetadata' in this._nativeApp ? this._nativeApp.appMetadata : this._nativeApp;
-        if (appData.activity.includes(appData.id)) {
-            return `${appData.id}.*`;
-        }
-        const { activity } = appData;
-        const lastValue = activity.split('.').pop();
-        return activity.replace(lastValue, '*');
-    }
-
-    get nativeAppLink() {
-        let link = null;
-        if ('filePath' in this._nativeApp) {
-            link = `${config.SERVICES_HOST}/storage${this._nativeApp.filePath}?access_token=${this._options.authData.token}`;
-        }
-        return link;
-    }
-
-    async getNativeAppData() {
-        const { appId, baseUrl } = this._options;
-        if (baseUrl && !this._nativeApp && !appId) {
-            const url = baseUrl || this.baseUrl;
-            if (!url) {
-                return null;
-            }
-            const [packageName, activity] = url.split(':');
-            return {
-                packageName,
-                activity,
-            };
-        }
-        if (this._nativeApp && !appId) {
-            if ('appMetadata' in this._nativeApp) {
-                return this._nativeApp.appMetadata;
-            }
-            return this._nativeApp;
-        }
-        return null;
-    }
-
-    get branch() {
-        return this._branch;
-    }
-
-    get sfdcCredential() {
-        return this._options.sfdcCredential;
-    }
-
-    get remoteRunId() {
-        return this._remoteRunId;
-    }
-
-    get overrideTestConfigId() {
-        return this._overrideTestConfigId;
-    }
-
-    markClearBrowser() {
-        this.clearBrowser = true;
-    }
-
-    async getRunRequestParams() {
-        const customTokenV3 = await testimCustomToken.getCustomTokenV3();
-        const runRequestParams = {
-            tokenV3: customTokenV3,
-            refreshToken: testimCustomToken.getRefreshToken(),
-            projectId: this._options.project,
-            executionId: this._executionId,
-            executionName: this._executionName,
-            testId: this._testId,
-            resultId: this._testResultId,
-            baseUrl: this._baseUrl,
-            branch: this._branch,
-            servicesUrl: config.EXTENSION_SERVICES_HOST,
-            remoteRunId: this.remoteRunId,
-            previousTestResultId: this.previousTestResultId,
-            testRetryCount: this.retryCount,
-            ...(this.code && { isCodeMode: true, testName: this.testName }),
-            ...(this._options.shouldMonitorPerformance && { shouldMonitorPerformance: true }),
-            ...(this._options.company && {
-                companyId: this._options.company.companyId,
-                onprem: this._options.company.onprem,
-                storageBaseUrl: this._options.company.storageBaseUrl,
-                storageType: this._options.company.storageType,
-                planType: this._options.company.planType,
-                isPOC: this._options.company.isPOC,
-                isStartUp: this._options.company.isStartUp,
-            }),
-            ...(this._options.collectCodeCoverage && { codeCoverageUrlFilter: this._options.codeCoverageUrlFilter || `${this.baseUrl}*` }),
-            ...(this._options.disableMockNetwork && { disableMockNetwork: this._options.disableMockNetwork }),
-            ...(this._options.lightweightMode && { lightweightMode: this._options.lightweightMode }),
-            ...(this.clearBrowser && { clearBrowser: true }),
-            ...(this._options.localRCASaver && { localRCASaver: this._options.localRCASaver }),
-            ...(this.sfdcCredential && { sfdcCredential: this.sfdcCredential }),
-        };
-
-        if (this._options.disableMockNetwork) {
-            analytics.trackWithCIUser('user-disable-mock');
-        }
-
-        if (this._options.lightweightMode?.general) {
-            runRequestParams.company = this._options.company;
-            const runData = this.runData;
-            runRequestParams.lightweightMode.isRunDataSentInUrl = canSendRunDataOverUrl(runData);
-            const stringifiedLength = JSON.stringify(runData).length;
-            const testId = this.testId;
-            if (runRequestParams.lightweightMode.isRunDataSentInUrl) {
-                runRequestParams.runData = runData;
-                logger.info(`Run data sent as URL param, test id: ${testId} run data length: ${stringifiedLength}`);
-            } else {
-                logger.warn(`Run data is too big to be sent as a URL param. Test id: ${testId}, run data size: ${stringifiedLength} (limit: ${MAX_LIGHTWEIGHT_MODE_RUN_DATA_SIZE} characters)`);
-            }
-            runRequestParams.isLocalRun = Boolean(this._options.useLocalChromeDriver || this._options.useChromeLauncher);
-        }
-
-        if (this._options.lightweightMode?.preloadTests && this._options.useChromeLauncher) {
-            const preloadedTests = await preloadTests(this._options);
-            runRequestParams.preloadedTest = preloadedTests[runRequestParams.testId];
-        }
-
-        return runRequestParams;
-    }
-
-    async getRunTestUrl() {
-        const runRequestParams = await this.getRunRequestParams();
-        const url = `https://run.testim.io/?params=${encodeURIComponent(JSON.stringify(runRequestParams))}`;
-        logger.info(`Test (${this.testId}) run URL length: ${url.length}`);
-        return url;
-    }
-
-    set sessionId(sessionId) {
-        this._sessionId = sessionId;
-    }
-
-    get sessionId() {
-        return this._sessionId;
-    }
-
-    get testId() {
-        return this._testId;
-    }
-
-    get testName() {
-        return this._testName;
-    }
-
-    get runParams() {
-        return this._options.runParams[this._testResultId] || {};
-    }
-
-    get runData() {
-        return {
-            userParamsData: this.runParams,
-            overrideTestConfigId: this._overrideTestConfigId || null,
-        };
-    }
-
-    async clearTestResult() {
-        const runData = this.runData;
-        if (this.runMode === CLI_MODE.EXTENSION) {
-            runData.code = this.code;
-        }
-
-        if (this._options.mockNetworkRules) {
-            runData.mockNetworkRules = this._options.mockNetworkRules;
-        }
-        const mustClearPreviousStepResults = (this._timeoutRetryCount > 1 || this._retryCount > 1);
-
-        if (this._options.lightweightMode?.disableResults && !mustClearPreviousStepResults && canSendRunDataOverUrl(runData)) {
-            return undefined;
-        }
-
-        const uploadRunDataArtifactAndFallbackToEmptyString = async () => {
-            try {
-                return await testimServicesApi.uploadRunDataArtifact(this._options.project, this._testId, this._testResultId, runData);
-            } catch (err) {
-                logger.error('failed to upload run data artifact (runner)', { err });
-                return '';
-            }
-        };
-        const clearTestResultFinished = async () => {
-            const runDataUrl = await uploadRunDataArtifactAndFallbackToEmptyString();
-            // make sure the execution is created by now
-            await this._testRunStatus.waitForExecutionStartedFinished();
-            // we probably can save this backend call by initializing the execution
-            return testimServicesApi.clearTestResult(this._options.project, this._testResultId, this._testId, {
-                name: this._testName,
-                resultId: this._testResultId,
-                status: 'pending',
-                retryCount: this._retryCount,
-                runDataUrl, // links the run data url to the test.
-                runData: runDataUrl ? undefined : runData, // put runData in mongo if we fail to upload to S3.
-                testRetryKey: this.retryKey,
-            });
-        };
-        this.clearTestResultFinished = clearTestResultFinished();
-        return this.clearTestResultFinished;
-    }
-
-    hasMoreRetries() {
-        return this._retryCount < this._maxRetryCount;
-    }
-
-    get retryKey() {
-        return `${this._retryCount}:${this._timeoutRetryCount}`;
-    }
-
-    startNewRetry() {
-        this._retryCount++;
-        this._timeoutRetryCount = 1;
-        return this.onRetry();
-    }
-
-    async runTestUsingCDP(cdpTestRunner) {
-        perf.log('runTestUsingCDP');
-        const { targetInfos } = await cdpTestRunner.cdpCommand('Target.getTargets') || { targetInfos: [] };
-        const { targetId: extensionTargetId } = targetInfos.find(target => target.type === 'background_page' && target.title === 'Testim Editor') || {};
-        const { targetId: AUTTargetId } = targetInfos.find(target => target.type === 'page') || {};
-        if (!extensionTargetId) {
-            throw new Error('testim extension not found');
-        }
-        if (!AUTTargetId) {
-            throw new Error('AUT target not found');
-        }
-
-        try {
-            perf.log('before Target.attachToTarget');
-            const [extensionSession, runRequestParams] = await Promise.all([
-                cdpTestRunner.cdpCommand('Target.attachToTarget', { targetId: extensionTargetId, flatten: true }),
-                this.getRunRequestParams(),
-            ]);
-            const { sessionId: extensionSessionId } = extensionSession || {};
-            perf.log('before Runtime.evaluate');
-
-            await pRetry(async () => {
-                const { result } = await cdpTestRunner.cdpCommand('Runtime.evaluate', { expression: 'typeof runTestimTest !== \'undefined\'', returnByValue: true }, extensionSessionId);
-                if (!result.value) {
-                    throw new Error('runTestimTest not available on global scope');
-                }
-            }, { retries: 100, minTimeout: 30, factor: 1 });
-
-            perf.log('after wait for runTestimTest function');
-            const { result } = await cdpTestRunner.cdpCommand(
-                'Runtime.evaluate',
-                { expression: `runTestimTest(${JSON.stringify(runRequestParams)})`, awaitPromise: true, returnByValue: true },
-                extensionSessionId,
-            );
-            if (result.subtype === 'error') {
-                throw new Error(result.description);
-            }
-            perf.log('after Runtime.evaluate');
-            return result.value;
-        } catch (err) {
-            logger.error('error running test using CDP', { err });
-            throw new Error('Error running test using CDP');
-        }
-    }
-
-    isRetryKeyMismatch(testResult) {
-        return testResult.testRetryKey && (testResult.testRetryKey !== this.retryKey);
-    }
-
-    validateRunConfig() {
-        const { baseUrl, runConfig: { browserValue } } = this;
-
-        if (baseUrl && browserValue === 'safari') {
-            let parsedUrl;
-            try {
-                parsedUrl = new URL(baseUrl);
-            } catch (err) {
-                // ignore invalid URLs (missing http:// or https:// prefix)
-                return;
-            }
-            const { username, password } = parsedUrl;
-
-            if (username || password) {
-                throw new Error('Basic authentication in URL is not supported in Safari');
-            }
-        }
-    }
-
-    /** @param {number} startTimeout */
-    onStarted(startTimeout) {
-        return new Promise(resolve => {
-            // We can't leave the test result as it may remove other listeners as well
-            // We need to implement an .off(resultId, listener) method
-            let reportedStart = false;
-            const resolveResult = testResult => {
-                if (reportedStart) {
-                    return;
-                }
-                if (this.isRetryKeyMismatch(testResult)) {
-                    logger.warn(`ignoring result update for on started due to retry key mismatch, got ${testResult.testRetryKey}, current is ${this.retryKey}`, {
-                        resultId: this.testResultId,
-                        testId: this.testId,
-                    });
-                    return;
-                }
-                if (['running', 'completed'].includes(testResult.status)) {
-                    testResult.resultId = this.testResultId;
-                    if (testResult.status === 'completed') {
-                        logger.info('setting _wasCompletedOnStartedCheck to true', {
-                            testResult,
-                            resultId: this.testResultId,
-                            testId: this.testId,
-                            testRetryKey: this.retryKey,
-                        });
-                        this._wasCompletedOnStartedCheck = testResult;
-                    }
-                    reportedStart = true;
-                    resolve(testResult);
-                }
-            };
-            if (this._options.disableSockets) {
-                const timeLimit = Date.now() + startTimeout;
-                const checkIfDone = async () => {
-                    if (Date.now() > timeLimit) {
-                        return;
-                    }
-                    const { testId, testResultId, branch, _options: { project: projectId } } = this;
-
-                    try {
-                        const restResult = await testimServicesApi.getTestResults(testId, testResultId, projectId, branch);
-                        resolveResult(restResult);
-                        if (!reportedStart) {
-                            setTimeout(checkIfDone, 3000);
-                        }
-                    } catch (err) {
-                        logger.error('failed to check if done', { err });
-                        setTimeout(checkIfDone, 3000);
-                    }
-                };
-                setTimeout(checkIfDone, 3000);
-            } else {
-                testResultService.listenToTestResult(this._testResultId, this._testId, resolveResult);
-            }
-        });
-    }
-
-    async checkViaRestAPIIfTestStarted() {
-        const { testId, testResultId, _options: { project: projectId }, branch } = this;
-        try {
-            const testResult = testimServicesApi.getTestResults(testId, testResultId, projectId, branch);
-            const expectedStatuses = ['running', 'completed'];
-            if (expectedStatuses.includes(testResult.status)) {
-                logger.info(`get status: ${testResult.status} after not get test started status`, { testId, testResultId, branch });
-                return testResult;
-            }
-            logger.error(`test not start test status: ${testResult.status} (expected [${expectedStatuses.join(', ')}])`, { testId, testResultId, branch });
-            throw new Error(timeoutMessages.TEST_START_TIMEOUT_MSG);
-        } catch (err) {
-            logger.error('failed to get test result after test start timeout', { err, testId, testResultId, branch });
-            throw new Error(timeoutMessages.TEST_START_TIMEOUT_MSG);
-        }
-    }
-
-    async onCompletedCleanup() {
-        if (this._options.disableSockets) {
-            return undefined;
-        }
-        return await testResultService.leaveTestResult(this._testResultId, this._testId);
-    }
-
-    async onCompleted() {
-        let onConnected;
-        try {
-            const res = await new Promise(resolve => {
-                if (this._wasCompletedOnStartedCheck && !this.isRetryKeyMismatch(this._wasCompletedOnStartedCheck)) {
-                    logger.info('test was already completed in on started check', {
-                        resultId: this.testResultId,
-                        testId: this.testId,
-                    });
-                    resolve(this._wasCompletedOnStartedCheck);
-                    return;
-                }
-
-                if (!this._options.disableSockets) {
-                    testResultService.listenToTestResult(this._testResultId, this._testId, testResult => {
-                        if (this.isRetryKeyMismatch(testResult)) {
-                            logger.warn(`ignoring result update for on completed due to retry key mismatch, got ${testResult.testRetryKey}, current is ${this.retryKey}`, {
-                                resultId: this.testResultId,
-                                testId: this.testId,
-                            });
-                            return;
-                        }
-                        if (testResult.status === 'completed') {
-                            testResult.resultId = this._testResultId;
-                            resolve(testResult);
-                        }
-                    });
-                }
-                const debounceDelay = this._options.disableSockets ? 0 : Math.floor(10000 + (Math.random() * 5000));
-                const maxWait = this._options.disableSockets ? 0 : Math.floor(60000 + (Math.random() * 15000));
-                onConnected = _.debounce(async () => {
-                    try {
-                        const testResult = await testimServicesApi.getTestResults(this._testId, this._testResultId, this._options.project, this.branch);
-                        if (this.isRetryKeyMismatch(testResult)) {
-                            logger.warn(`ignoring result update for on completed (in reconnect) due to retry key mismatch, got ${testResult.testRetryKey}, current is ${this.retryKey}`, {
-                                resultId: this.testResultId,
-                                testId: this.testId,
-                            });
-                            return false;
-                        }
-                        if (testResult?.status === 'completed') {
-                            logger.info('Socket reconnected - Test complete', { testId: this._testId, resultId: this._testResultId, projectId: this._options.project });
-                            testResult.resultId = this._testResultId;
-                            resolve(testResult);
-                            return true;
-                        }
-                        return false;
-                    } catch (err) {
-                        logger.warn('Error while trying to check status on socket connect', err);
-                        return false;
-                    }
-                }, debounceDelay, { maxWait });
-                if (!this._options.disableSockets) {
-                    testResultService.on('socket-connected', onConnected);
-                } else {
-                    const waitForTestEnd = () => {
-                        setTimeout(async () => {
-                            try {
-                                const { isComplete } = await testimServicesApi.isTestResultCompleted(this._testResultId, this._options.project, this.retryKey);
-                                if (isComplete) {
-                                    const isDone = await onConnected();
-                                    if (!isDone) {
-                                        logger.warn('onConnected returned false even though isComplete was true');
-                                        waitForTestEnd();
-                                    }
-                                } else {
-                                    waitForTestEnd();
-                                }
-                            } catch (err) {
-                                logger.error('failed to check is complete', { err });
-                                waitForTestEnd();
-                            }
-                        }, 3000);
-                    };
-                    waitForTestEnd();
-                }
-            });
-            await this.onCompletedCleanup();
-            return res;
-        } finally {
-            if (onConnected && !this._options.disableSockets) {
-                testResultService.off('socket-connected', onConnected);
-            }
-        }
-    }
-
-    listenToRemoteStep(browser) {
-        remoteStepService.listenToRemoteStep(this.testResultId, step => {
-            remoteStepPlayback.executeStep(this._options, browser, step, this.testResultId);
-        });
-    }
-
-    hasMoreTimeoutRetries() {
-        const maxRetryCount = this._options.disableTimeoutRetry ? 1 : RETRIES_ON_TIMEOUT;
-        return this._timeoutRetryCount < maxRetryCount;
-    }
-
-    startNewTimeoutRetry() {
-        this._timeoutRetryCount++;
-        return this.onRetry();
-    }
-
-    get retryCount() {
-        return this._retryCount;
-    }
-
-    get previousTestResultId() {
-        return this._previousTestResultId;
-    }
-
-    isAllowReportTestResultRetries() {
-        return Boolean(this._options.company?.activePlan?.premiumFeatures?.allowReportTestResultRetries);
-    }
-
-    async onRetry() {
-        this._previousTestResultId = this.testResultId;
-
-        if (!this.isAllowReportTestResultRetries()) {
-            return;
-        }
-
-        this._totalRetryCount++;
-        this._originalTestResultId = this._originalTestResultId || this._previousTestResultId;
-        this._testResultId = utils.guid();
-
-        if (this._options.lightweightMode?.onlyTestIdsNoSuite) {
-            return;
-        }
-
-        await this._testRunStatus.addRetryTestResult({
-            retryCount: this._totalRetryCount,
-            executionId: this._executionId,
-            projectId: this._options.project,
-            newResultId: this._testResultId,
-            originalTestResultId: this._originalTestResultId,
-            previousTestResultId: this._previousTestResultId,
-        });
-    }
-}
-
-module.exports = TestRun;