@testim/testim-cli 3.200.0 → 3.204.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testim/testim-cli",
-  "version": "3.200.0",
+  "version": "3.204.0",
   "description": "Command line interface for running Testing on your CI",
   "author": "Oren Rubin",
   "contributors": [{
@@ -36,7 +36,7 @@
     "typescript": "3.9.3"
   },
   "lazyDependencies": {
-    "ngrok": "3.2.3",
+    "ngrok": "3.4.0",
     "webpack": "4.43.0",
     "playwright": "0.12.1",
     "puppeteer": "2.1.1",
@@ -60,8 +60,8 @@
     "bluebird": "3.7.2",
     "bluebird-retry": "0.11.0",
     "body-parser": "1.19.0",
-    "bunyan": "1.8.15",
-    "bunyan-prettystream": "0.1.3",
+    "winston": "3.3.3",
+    "winston-transport": "4.4.0",
     "chalk": "4.1.0",
     "chrome-launcher": "0.13.4",
     "combine-source-map": "0.8.0",
@@ -79,10 +79,10 @@
     "glob": "7.1.6",
     "istanbul-lib-report": "3.0.0",
     "istanbul-reports": "3.0.2",
-    "jimp": "0.2.28",
+    "jimp": "0.16.1",
     "jsdom": "16.7.0",
     "jsonwebtoken": "8.5.1",
-    "lodash": "4.17.20",
+    "lodash": "4.17.21",
     "memory-fs": "0.5.0",
     "memorystream": "0.3.1",
     "mkdirp": "1.0.4",
@@ -105,9 +105,9 @@
     "superagent-proxy": "3.0.0",
     "test-exclude": "6.0.0",
     "threads": "0.12.0",
-    "ua-parser-js": "0.7.22",
+    "ua-parser-js": "0.7.28",
     "validate-npm-package-name": "3.0.0",
-    "ws": "7.3.0",
+    "ws": "8.2.3",
     "xml2js": "0.4.23",
     "yaml": "1.10.0"
   },

package/player/stepActions/pixelValidationStepAction.js

@@ -7,21 +7,24 @@ const logger = require('../../commons/logger').getLogger('pixel-validation-step-
 
 class PixelValidationStepAction extends StepAction {
     async performAction() {
-        const { shouldUseVisualGrid, applitoolsSdkConfig: config } = this.context;
+        const { shouldUseVisualGrid, applitoolsSdkConfig: config, testResultId } = this.context;
         this.runContext = this.context.getRunContext(undefined);
-        const eyeManager = await eyeSdkService.getManager(shouldUseVisualGrid, this.context.config.applitoolsConcurrency || 5);
+        const batchId = (config.batch && config.batch.id) || testResultId;
+        const eyeManager = await eyeSdkService.getManager(shouldUseVisualGrid, this.context.config.applitoolsConcurrency || 5, batchId, this.runContext.applitoolsIntegrationData);
         const targetElementData = this.getTarget() || {};
+        let result;
         try {
             const openedEye = await eyeManager.openEyes({ driver: this.driver.client, config });
             const region = (this.step.action === 'element' && targetElementData.seleniumElement) || undefined;
             await openedEye.check({ settings: { region, fully: this.step.action === 'stitched' } });
             const eyesResults = await openedEye.close();
 
-            return { isApplitoolsSdkResult: true, success: true, eyesResults };
+            result = { isApplitoolsSdkResult: true, success: true, eyesResults };
         } catch (err) {
             logger.error('Applitools SDK step failed', { err, info: err.info });
-            return { isApplitoolsSdkResult: true, success: false, err };
+            result = { isApplitoolsSdkResult: true, success: false, err };
         }
+        return await eyeSdkService.handleApplitoolsSdkResult(this.context, result);
     }
 }
 

package/player/utils/eyeSdkService.js

@@ -1,5 +1,10 @@
-// https://github.com/applitools/eyes.sdk.javascript1/blob/master/packages/eyes-webdriverio-4/src/spec-driver.ts
+/**
+ * @typedef {typeof import('../../../../clickim/src/background/eyeSdkBuilder').EyeSdkBuilder} EyeSdkBuilder
+ * @typedef {import('@applitools/types').SpecDriver} SpecDriver
+ * @typedef {import('@applitools/types').Core} Core
+ */
 
+const { EyeSdkBuilder } = require('../../commons/getSessionPlayerRequire');
 const { makeSDK } = require('@applitools/eyes-sdk-core');
 const { W3C_ELEMENT_ID } = require('../constants');
 const _ = require('lodash');
@@ -27,7 +32,8 @@ function getValueOrFallbackIfNullOrUndefined(value, fallback) {
 }
 
 /**
- * @typedef {import('@applitools/types').SpecDriver} SpecDriver
+ * Applitools Spec Driver for webdriverIO 4.
+ * @see https://github.com/applitools/eyes.sdk.javascript1/blob/master/packages/eyes-webdriverio-4/src/spec-driver.ts
  * @implements {SpecDriver}
  */
 class EyesSpec {
@@ -211,19 +217,20 @@ class EyesSpec {
 
 class EyeSdkService {
     constructor() {
-        /**
-         * @typedef {import('@applitools/types').Core} Core
-         * @type {Core}
-         */
+        /** @type {Core} */
         this.sdk = makeSDK({
             name: 'Testim.io',
             version: '4.0.0',
             spec: new EyesSpec(),
             VisualGridClient: require('@applitools/visual-grid-client'),
         });
+        /** @type {EyeSdkBuilder['handleApplitoolsSdkResult']} */
+        this.handleApplitoolsSdkResult = EyeSdkBuilder.handleApplitoolsSdkResult;
     }
-    getManager(useVisualGrid, concurrency) {
-        return this.sdk.makeManager({ type: useVisualGrid ? 'vg' : 'classic', concurrency });
+    async getManager(useVisualGrid, concurrency, batchId, applitoolsIntegrationData) {
+        const manager = await this.sdk.makeManager({ type: useVisualGrid ? 'vg' : 'classic', concurrency });
+        EyeSdkBuilder.rememberCreatedBatch(batchId, applitoolsIntegrationData);
+        return manager;
     }
 }
 

package/reports/junitReporter.js

@@ -50,9 +50,6 @@ class JunitReporter {
 }
 
 function getPrintName(testResult) {
-    if (!featureFlags.flags.testNameTestDataInJunitReport.isEnabled()) {
-        return testResult.name;
-    }
     const testData = testResult.testData || {};
     const testDataNumber = typeof testData.total === 'number' ? ` - ${testData.index} / ${testData.total} Data set` : '';
     return `${testResult.name}${testDataNumber}`;

package/runner.js

@@ -286,16 +286,7 @@ async function init(options) {
     reporter.setOptions(options, branchToUse);
 }
 
-function run(options, customExtensionLocalLocation) {
-    perf.log('in runner.js run');
-
-    if (options.files.length > 0 && !featureFlags.flags.enableTDKRun.isEnabled()) {
-        throw new ArgError('run command is not supported, please sign up to the Testim Development Kit beta - https://go.testim.io/testims-new-beta-testim-tdk-testim-development-kit-testautomation');
-    }
-    return runRunner(options, customExtensionLocalLocation);
-}
-
 module.exports = {
-    run,
+    run: runRunner,
     init: Promise.method(init),
 };

package/runners/ParallelWorkerManager.js

@@ -46,14 +46,14 @@ class ParallelWorkerManager {
         return Array.from(new Array(count), createWorker);
     }
 
-    async runTests(testList, testStatus, executionId, options, branchToUse, authData, workerCount, stopOnError) {
+    async runTests(testList, testStatus, executionId, executionName, options, branchToUse, authData, workerCount, stopOnError) {
         if (testList && testList.length === 0) {
             return undefined;
         }
 
         const runAndWaitToComplete = token => new Promise((resolve, reject) => {
             const projectId = options.project;
-            const executionQueue = new ExecutionQueue(executionId, testList, options, branchToUse, testStatus);
+            const executionQueue = new ExecutionQueue(executionId, executionName, testList, options, branchToUse, testStatus);
 
             const combinedTestResults = {};
             const testCount = testList.length;

package/runners/TestPlanRunner.js

@@ -19,6 +19,7 @@ const { getSuite, calcTestResultStatus, validateConfig } = require('./runnerUtil
 const { StopRunOnError, ArgError } = require('../errors');
 const Logger = require('../commons/logger');
 const perf = require('../commons/performance-logger');
+const featureFlags = require('../commons/featureFlags');
 
 const guid = utils.guid;
 const logger = Logger.getLogger('test-plan-runner');
@@ -29,30 +30,30 @@ class TestPlanRunner {
         this.workerManager = new ParallelWorkerManager(customExtensionLocalLocation);
         this.startTime = Date.now();
     }
-    runTestAllPhases(beforeTests, tests, afterTests, branchToUse, tpOptions, executionId, testStatus) {
+    runTestAllPhases(beforeTests, tests, afterTests, branchToUse, tpOptions, executionId, executionName, testStatus) {
         const executionResults = {};
         const authData = testimCustomToken.getTokenV3UserData();
 
-        const runBeforeTests = (beforeTests, testStatus, executionId, tpOptions, branchToUse, authData) => {
+        const runBeforeTests = (beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData) => {
             const workerCount = 1;
             const stopOnError = true;
-            return this.workerManager.runTests(beforeTests, testStatus, executionId, tpOptions, branchToUse, authData, workerCount, stopOnError)
+            return this.workerManager.runTests(beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError)
                 .then(beforeTestsResults => Object.assign(executionResults, beforeTestsResults));
         };
 
-        const runTestPlanTests = (tests, testStatus, executionId, tpOptions, branchToUse, authData) => {
+        const runTestPlanTests = (tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData) => {
             const workerCount = config.TESTIM_CONCURRENT_WORKER_COUNT || tpOptions.parallel;
             const stopOnError = false;
             perf.log('right before this.workerManager.runTests');
-            return this.workerManager.runTests(tests, testStatus, executionId, tpOptions, branchToUse, authData, workerCount, stopOnError)
+            return this.workerManager.runTests(tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError)
                 .log('right after this.workerManager.runTests')
                 .then(testsResults => Object.assign(executionResults, testsResults));
         };
 
-        const runAfterTests = (afterTests, testStatus, executionId, tpOptions, branchToUse, authData) => {
+        const runAfterTests = (afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData) => {
             const workerCount = 1;
             const stopOnError = false;
-            return this.workerManager.runTests(afterTests, testStatus, executionId, tpOptions, branchToUse, authData, workerCount, stopOnError)
+            return this.workerManager.runTests(afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError)
                 .then(afterTestsResults => Object.assign(executionResults, afterTestsResults));
         };
 
@@ -63,11 +64,11 @@ class TestPlanRunner {
         const sessionType = utils.getSessionType(tpOptions);
         analyticsService.analyticsExecsStart({ authData, executionId, projectId: tpOptions.project, sessionType });
         perf.log('right before runBeforeTests');
-        return runBeforeTests(beforeTests, testStatus, executionId, tpOptions, branchToUse, authData)
+        return runBeforeTests(beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData)
             .log('right before runTestPlanTests')
-            .then(() => runTestPlanTests(tests, testStatus, executionId, tpOptions, branchToUse, authData))
+            .then(() => runTestPlanTests(tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData))
             .log('right after runTestPlanTests')
-            .then(() => runAfterTests(afterTests, testStatus, executionId, tpOptions, branchToUse, authData))
+            .then(() => runAfterTests(afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData))
             .then(() => executionResults)
             .catch(err => {
                 logger.error('error running test plan', { err });
@@ -75,6 +76,34 @@ class TestPlanRunner {
                     return catchBeforeTestsFailed(executionId);
                 }
                 throw err;
+            })
+            .finally(async () => {
+                if ((tpOptions.lightweightMode && tpOptions.lightweightMode.disablePixelValidation) || !featureFlags.flags.applitoolsNewIntegration.isEnabled()) {
+                    return;
+                }
+                // When sessionPlayer is available, use it - as it only attempts to close batches that exist.
+                if (tpOptions.mode === constants.CLI_MODE.SELENIUM) {
+                    const { EyeSdkBuilder } = require('../commons/getSessionPlayerRequire');
+                    await EyeSdkBuilder.closeBatch(executionId);
+                    return;
+                }
+                try {
+                    if (!tpOptions.company || !tpOptions.company.activePlan || !tpOptions.company.activePlan.premiumFeatures || !tpOptions.company.activePlan.premiumFeatures.applitools) {
+                        return;
+                    }
+                    const applitoolsIntegrationData = await testimServicesApi.getApplitoolsIntegrationData(tpOptions.project);
+                    if (_.isEmpty(applitoolsIntegrationData)) {
+                        return;
+                    }
+                    const { runKey: apiKey, url: serverUrl } = applitoolsIntegrationData;
+                    const tmpSDK = require('@applitools/eyes-sdk-core').makeSDK({ name: 'Testim.io', version: '4.0.0', spec: {} });
+                    await tmpSDK.closeBatches({ batchIds: [executionId], serverUrl, apiKey });
+                } catch (err) {
+                    if (err.message && err.message.startsWith('Request failed with status code 404')) { // If a batch with this name did not exist, do not log an error.
+                        return;
+                    }
+                    logger.error('Failed closing batch in extension mode', { err, projectId: tpOptions.project });
+                }
             });
     }
 
@@ -167,7 +196,7 @@ class TestPlanRunner {
         }
 
         perf.log('before runTestAllPhases');
-        const results = await this.runTestAllPhases(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, branch, tpOptions, executionId, testStatus);
+        const results = await this.runTestAllPhases(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, branch, tpOptions, executionId, testPlanName || 'All Tests', testStatus);
         const childResults = await Bluebird.resolve(childTestResults)
             .timeout(TDK_CHILD_RESULTS_TIMEOUT)
             .catch(async () => {

package/services/gridService.js

@@ -50,13 +50,12 @@ function getSerializableObject(grid) {
         (tunnel && grid.hybrid.external && grid.hybrid.external[grid.hybrid.tunnel] && grid.hybrid.external[grid.hybrid.tunnel].user) : user;
     const tunnelKey = type === gridTypes.HYBRID ?
         (tunnel && grid.hybrid.external && grid.hybrid.external[grid.hybrid.tunnel] && grid.hybrid.external[grid.hybrid.tunnel].key) : key;
-    const arn = (grid && grid.external && grid.external.arn) || (tunnel && grid.hybrid.external && grid.hybrid.external[grid.hybrid.tunnel] && grid.hybrid.external[grid.hybrid.tunnel].arn);
     const name = grid && grid.name;
     const gridId = grid && (grid._id || grid.gridId);
     const provider = grid && grid.provider;
 
     return {
-        host, port, path, protocol, accessToken, slotId, gridId, tunnel, user, key, type, name, arn, provider, tunnelUser, tunnelKey,
+        host, port, path, protocol, accessToken, slotId, gridId, tunnel, user, key, type, name, provider, tunnelUser, tunnelKey,
     };
 }
 

package/testRunHandler.js

@@ -15,13 +15,13 @@ const utils = require('./utils');
 const config = require('./commons/config');
 const analytics = require('./commons/testimAnalytics');
 const { SeleniumPerfStats } = require('./commons/SeleniumPerfStats');
-const featureFlags = require('./commons/featureFlags');
 const { preloadTests } = require('./commons/preloadTests');
 
 const RETRIES_ON_TIMEOUT = 3;
 
-const TestRun = function (executionId, test, options, branchToUse, testRunStatus) {
+const TestRun = function (executionId, executionName, test, options, branchToUse, testRunStatus) {
     this._executionId = executionId;
+    this._executionName = executionName;
     this._testStatus = test.testStatus;
     this._testId = test.testId;
     this._testName = test.name;
@@ -81,6 +81,10 @@ TestRun.prototype.getExecutionId = function () {
     return this._executionId;
 };
 
+TestRun.prototype.getExecutionName = function () {
+    return this._executionName;
+};
+
 TestRun.prototype.getNativeAppData = function () {
     if (this._nativeApp && !this._options.baseUrl) {
         return this._nativeApp;
@@ -122,6 +126,7 @@ TestRun.prototype.getRunRequestParams = async function () {
         refreshToken: testimCustomToken.getRefreshToken(),
         projectId: this._options.project,
         executionId: this._executionId,
+        executionName: this._executionName,
         testId: this._testId,
         resultId: this._testResultId,
         baseUrl: this._baseUrl,
@@ -232,6 +237,12 @@ TestRun.prototype.clearTestResult = function () {
     if (this._options.mockNetworkRules) {
         runData.mockNetworkRules = this._options.mockNetworkRules;
     }
+    const mustClearPreviousStepResults = !this.isAllowReportTestResultRetries() && (this._timeoutRetryCount > 1 || this._retryCount > 1);
+
+    if (this._options.lightweightMode && this._options.lightweightMode.disableResults &&
+        !mustClearPreviousStepResults) {
+        return Promise.resolve();
+    }
 
     this.clearTestResultFinished = testimServicesApi.uploadRunDataArtifact(this._options.project, this._testId, this._testResultId, runData)
         .catch(err => {
@@ -528,7 +539,7 @@ TestRun.prototype.isAllowReportTestResultRetries = function () {
 TestRun.prototype.onRetry = async function () {
     this._previousTestResultId = this._testResultId;
 
-    if (!featureFlags.flags.countRetries.isEnabled() || !this.isAllowReportTestResultRetries()) {
+    if (!this.isAllowReportTestResultRetries()) {
         return;
     }
 

package/workers/BaseWorker.js

@@ -90,9 +90,7 @@ class BaseWorker {
             resultId: testRunHandler.getTestResultId(),
             seleniumSession: player.getSessionId(),
         });
-        if (this.options.lightweightMode && this.options.lightweightMode.disableResults) {
-            return undefined;
-        }
+
         return await testRunHandler.clearTestResult();
     }
 

package/workers/WorkerSelenium.js

@@ -88,6 +88,9 @@ class WorkerSelenium extends BaseWorker {
 
         setupCliPerformanceMonitoring(sessionPlayer);
 
+        sessionPlayer.playbackManager.executionId = testRunHandler.getExecutionId();
+        sessionPlayer.playbackManager.executionName = testRunHandler.getExecutionName();
+
         sessionPlayer.setLightweightMode(this.options.lightweightMode);
         if (sessionPlayerInit.localAssetService) {
             sessionPlayerInit.localAssetService.initialize({ serverUrl: this.options.localRCASaver });

package/lib/coralogix-bunyan.stream.js

@@ -1,56 +0,0 @@
-'use strict';
-
-// This is copied from coralogix-bunyan-stream pacakge and altered to expose waitForFlush
-
-const coralogixLogger = require('@testim/coralogix-logger');
-
-const sevMap = {
-    10: coralogixLogger.Severity.debug,
-    30: coralogixLogger.Severity.info,
-    40: coralogixLogger.Severity.warning,
-    50: coralogixLogger.Severity.error,
-    60: coralogixLogger.Severity.critical,
-    20: coralogixLogger.Severity.debug,
-};
-const CoralogixStream = (function () {
-    function CoralogixStream(options) {
-        if (!options) options = {};
-        CoralogixStream.logger = this.logger = new coralogixLogger.CoralogixLogger(options.category);
-    }
-    CoralogixStream.waitForFlush = function () {
-        return CoralogixStream.logger.waitForFlush();
-    };
-    CoralogixStream.prototype.write = function (rec) {
-        const log = new coralogixLogger.Log();
-        log.severity = sevMap[rec.level];
-        log.category = rec.category;
-        if (rec.className) log.className = rec.className;
-        if (rec.methodName) log.methodName = rec.methodName;
-        if (rec.threadId) log.threadId = rec.threadId;
-        log.text = this.removeStaticFields(rec);
-        this.logger.addLog(log);
-    };
-    CoralogixStream.prototype.removeStaticFields = function (json) {
-        delete json.threadId;
-        delete json.methodName;
-        delete json.className;
-        delete json.category;
-        return JSON.stringify(json, this.replaceErrors);
-    };
-    CoralogixStream.prototype.replaceErrors = function (key, value) {
-        if (value instanceof Error) {
-            const error = {};
-            Object.getOwnPropertyNames(value).forEach((key) => {
-                error[key] = value[key];
-            });
-            return error;
-        }
-        return value;
-    };
-    CoralogixStream.configure = function (config) {
-        coralogixLogger.CoralogixLogger.configure(config);
-    };
-    return CoralogixStream;
-}());
-
-module.exports = { CoralogixStream };