fastlane-plugin-retry_tests 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c836a5f93edff7bdd7d09d6e5d7a4e62e075352b
+  data.tar.gz: aacd3dcc668bb57dbf6e99162e23cd78e95ba49a
+SHA512:
+  metadata.gz: 6f37a54bedb06e4a9bb3027e51781294c5df4d0465fbe1ccb7c717e9eb2aafe6890a63d7040693e41e91c103bb431bf00806ac04c34ac4f3fe86558640ffa7f5
+  data.tar.gz: 4bb410cacd5a8e72187d27e449e04369a56b3a19b44e181b4f1d34257df19d910cc1c2bd4288016162bfdcaad8c5a394b2f700d9e31002e4462917a4f9e6d95d

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Lyndsey Ferguson <lyndsey.ferguson@appian.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,2 @@
+# fastlane-plugin-retry
+A fastlane plugin for retrying iOS automation tests. Based on the "fastlane-plugin-test_center" plugin repository by Github user lyndsey-ferguson. 

data/lib/fastlane/plugin/retry_tests.rb

@@ -0,0 +1,16 @@
+require 'fastlane/plugin/retry_tests/version'
+
+module Fastlane
+  module RetryTests
+    # Return all .rb files inside the "actions" and "helper" directory
+    def self.all_classes
+      Dir[File.expand_path('**/{actions,helper}/*.rb', File.dirname(__FILE__))]
+    end
+  end
+end
+
+# By default we want to import all available actions and helpers
+# A plugin can contain any number of actions and plugins
+Fastlane::RetryTests.all_classes.each do |current|
+  require current
+end

data/lib/fastlane/plugin/retry_tests/actions/collate_junit_reports.rb

@@ -0,0 +1,177 @@
+#Adapted from the "fastlane-plugin-test_center" plugin
+
+require 'nokogiri'
+require 'nokogiri-plist'
+require 'FileUtils'
+module Fastlane
+  module Actions
+    class CollateJunitReportsAction < Action
+
+      def self.run(params)
+        report_filepaths = params[:reports].reverse
+        if report_filepaths.size == 1
+          FileUtils.cp(report_filepaths[0], params[:collated_report])
+        else
+          #Convert .plist reports to Nokogiri XML objects
+          target_report = File.open(report_filepaths.shift) {|f| Nokogiri::XML(f)}
+          file_name = params[:collated_report] + "/#{params[:scheme]}.test_result/1_Test/action_TestSummaries.plist"
+          reports = report_filepaths.map { |report_filepath| Nokogiri::XML(Nokogiri::PList(open(report_filepath)).to_plist_xml) }
+
+          #Merge .plist reports
+          reports.each do |retry_report|
+            mergeLists(target_report, retry_report, params)
+          end
+        end
+        merge_assets(params[:assets], params[:collated_report] + "/Attachments")
+        merge_logs(params[:logs], params[:collated_report] + "/")
+      end
+
+      def self.mergeLists(target_report, retry_report, params)
+        UI.verbose("Merging retried results...")
+        Dir.mkdir(params[:collated_report]) unless File.exists?(params[:collated_report])
+        file_name = params[:collated_report] + "/action_TestSummaries.plist"
+        retried_tests = retry_report.xpath("//key[contains(.,'TestSummaryGUID')]/..")
+        current_node = retried_tests.shift
+
+        #Copy retried test results into the base report
+        while (current_node != nil)
+          testName = get_test_name(current_node)
+          matching_node = target_report.at_xpath("//string[contains(.,'#{testName}')]/..")
+
+          if (!matching_node.nil?)
+            matching_node.previous.next.replace(current_node)
+            write_report_to_file(target_report, file_name)
+          end
+          current_node = retried_tests.shift
+        end
+      end
+
+      def self.merge_assets(asset_files, assets_folder)
+        UI.verbose ("Merging screenshot folders...")
+
+        #copy screenshots from all retries into the results folder
+        Dir.mkdir(assets_folder) unless File.exists?(assets_folder)
+        asset_files.each do |folder|
+          FileUtils.cp_r(Dir[folder + '/*'], assets_folder)
+        end
+      end
+
+      def self.merge_logs(log_files, logs_folder)
+        UI.verbose("Merging console logs...")
+
+        #copy console logs from all retries into the results folder
+        target_log = log_files.shift
+        log_files.each do |log|
+          to_append = File.read(log)
+          File.open(target_log, "a") do |handle|
+            handle.puts to_append
+          end
+          FileUtils.cp_r(target_log, logs_folder)
+        end
+      end
+
+      def self.write_report_to_file(report, file_name)
+        UI.verbose("Writing merged results to file...")
+        File.new(file_name, 'w')
+        File.open(file_name, 'w') do |f|
+          f.write(report.to_xml)
+        end
+      end
+
+      def self.get_test_name(test_data)
+        test_name = test_data.xpath("(//key[contains(.,'TestSummaryGUID')])/../key[contains(.,'TestName')]/following-sibling::string").to_a[0].to_s
+        test_name = test_name[8..-10]
+        test_name
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "Combines and combines tests from multiple junit report files"
+      end
+
+      def self.details
+        "The first junit report is used as the base report. Testcases " \
+        "from other reports are added if they do not already exist, or " \
+        "if the testcases already exist, they are replaced." \
+        "" \
+        "This is done because it is assumed that fragile tests, when " \
+        "re-run will often succeed due to less interference from other " \
+        "tests and the subsequent junit reports will have more passed tests." \
+        "" \
+        "Inspired by Derek Yang's fastlane-plugin-merge_junit_report"
+      end
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(
+            key: :scheme,
+            env_name: 'SCHEME',
+            description: 'The test scheme being run',
+            optional: false,
+            default_value: '',
+            type: String
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :reports,
+            env_name: 'COLLATE_JUNIT_REPORTS_REPORTS',
+            description: 'An array of junit reports to collate. The first report is used as the base into which other reports are merged in',
+            optional: false,
+            type: Array,
+            verify_block: proc do |reports|
+              UI.user_error!('No junit report files found') if reports.empty?
+              reports.each do |report|
+                UI.user_error!("Error: junit report not found: '#{report}'") unless File.exist?(report)
+              end
+            end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :collated_report,
+            env_name: 'COLLATE_JUNIT_REPORTS_COLLATED_REPORT',
+            description: 'The final junit report file where all testcases will be merged into',
+            optional: true,
+            default_value: 'result.xml',
+            type: String
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :assets,
+            env_name: 'COLLATE_JUNIT_REPORTS_ASSETS',
+            description: 'An array of junit reports to collate. The first report is used as the base into which other reports are merged in',
+            optional: false,
+            type: Array,
+            verify_block: proc do |assets|
+              UI.user_error!('No junit report files found') if assets.empty?
+              assets.each do |asset|
+                UI.user_error!("Error: junit report not found: '#{asset}'") unless File.exist?(asset)
+              end
+            end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :logs,
+            env_name: 'COLLATE_JUNIT_REPORTS_LOGS',
+            description: 'An array of junit reports to collate. The first report is used as the base into which other reports are merged in',
+            optional: false,
+            type: Array,
+            verify_block: proc do |logs|
+              UI.user_error!('No junit report files found') if logs.empty?
+              logs.each do |log|
+                UI.user_error!("Error: junit report not found: '#{log}'") unless File.exist?(log)
+              end
+            end
+          )
+        ]
+      end
+
+      def self.authors
+        #Adapted from the "fastlane-plugin-test_center" plugin by:
+        ["lyndsey-ferguson/@lyndseydf"]
+      end
+
+      def self.is_supported?(platform)
+        platform == :ios
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/retry_tests/actions/multi_scan.rb

@@ -0,0 +1,146 @@
+#Adapted from the "fastlane-plugin-test_center" plugin
+
+module Fastlane
+  module Actions
+    require 'fastlane/actions/scan'
+    require 'shellwords'
+
+    class MultiScanAction < Action
+      def self.run(params)
+        try_count = 0
+        scan_options = params.values.reject { |k| k == :try_count }
+        final_report_path = scan_options[:result_bundle]
+        unless Helper.test?
+          FastlaneCore::PrintTable.print_values(
+            config: params._values.reject { |k, v| scan_options.key?(k) },
+            title: "Summary for multi_scan (test_center v#{Fastlane::TestCenter::VERSION})"
+          )
+        end
+
+        scan_options = config_with_junit_report(scan_options)
+
+        begin
+          try_count += 1
+          scan_options = config_with_retry(scan_options, try_count)
+          config = FastlaneCore::Configuration.create(Fastlane::Actions::ScanAction.available_options, scan_options)
+          Fastlane::Actions::ScanAction.run(config)
+        rescue FastlaneCore::Interface::FastlaneTestFailure => e
+          UI.verbose("Scan failed with error #{e}")
+          #Retry for the specified number of times if there were failed tests
+          if try_count < params[:try_count]
+            report_filepath = junit_report_filepath(scan_options)
+            failed_tests = other_action.tests_from_junit(junit: report_filepath)[:failed]
+            scan_options[:only_testing] = failed_tests.map(&:shellescape)
+            retry
+          end
+        end
+        merge_reports(scan_options, final_report_path)
+      end
+
+      def self.merge_reports(scan_options, final_report_path)
+        folder = get_folder_root(scan_options[:output_directory])
+        report_files = Dir.glob("#{folder}*/#{scan_options[:scheme]}.test_result/1_Test/action_TestSummaries.plist")
+        asset_files = Dir.glob("#{folder}*/#{scan_options[:scheme]}.test_result/1_Test/Attachments")
+        log_files = Dir.glob("#{folder}*/#{scan_options[:scheme]}.test_result/1_Test/action.xcactivitylog")
+        #Merge all reports, screenshots, and logs if there were retried tests
+        if report_files.size > 1
+          other_action.collate_junit_reports(
+            scheme: scan_options[:scheme],
+            reports: report_files,
+            collated_report: final_report_path,
+            assets: asset_files,
+            logs: log_files,
+          )
+        end
+      end
+
+      def self.config_has_junit_report(config)
+        output_types = config.fetch(:output_types, '').to_s.split(',')
+        output_filenames = config.fetch(:output_files, '').to_s.split(',')
+
+        output_type_file_count_match = output_types.size == output_filenames.size
+        output_types.include?('junit') && (output_type_file_count_match || config[:custom_report_file_name].to_s.strip.length > 0)
+      end
+
+      def self.config_with_retry(config, count)
+        folder = get_folder_root(config[:result_bundle])
+        config[:result_bundle] = (folder + count.to_s)
+        config[:output_directory] = (folder + count.to_s)
+        config
+      end
+
+      def self.get_folder_root(folder)
+        folder = folder.gsub(/ *\d+$/, '')
+        folder
+      end
+
+      def self.config_with_junit_report(config)
+        return config if config_has_junit_report(config)
+
+        if config[:output_types].to_s.strip.empty? || config[:custom_report_file_name]
+          config[:custom_report_file_name] ||= 'report.xml'
+          config[:output_types] = 'junit'
+        elsif config[:output_types].strip == 'junit' && config[:output_files].to_s.strip.empty?
+          config[:custom_report_file_name] ||= 'report.xml'
+        elsif !config[:output_types].split(',').include?('junit')
+          config[:output_types] << ',junit'
+          config[:output_files] << ',report.xml'
+        elsif config[:output_files].nil?
+          config[:output_files] = config[:output_types].split(',').map { |type| "report.#{type}" }.join(',')
+        end
+        config
+      end
+
+      def self.junit_report_filename(config)
+        report_filename = config[:custom_report_file_name]
+        if report_filename.nil?
+          junit_index = config[:output_types].split(',').find_index('junit')
+          report_filename = config[:output_files].to_s.split(',')[junit_index]
+        end
+        report_filename
+      end
+
+      def self.junit_report_filepath(config)
+        File.absolute_path(File.join(config[:output_directory], junit_report_filename(config)))
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "Uses scan to run Xcode tests a given number of times: only re-testing failing tests."
+      end
+
+      def self.details
+        "Use this action to run your tests if you have fragile tests that fail sporadically."
+      end
+
+      def self.scan_options
+        ScanAction.available_options
+      end
+
+      def self.available_options
+        scan_options + [
+          FastlaneCore::ConfigItem.new(
+            key: :try_count,
+            env_name: "FL_MULTI_SCAN_TRY_COUNT",
+            description: "The number of times to retry running tests via scan",
+            type: Integer,
+            is_string: false,
+            default_value: 1
+          )
+        ]
+      end
+
+      def self.authors
+        #Adapted from the "fastlane-plugin-test_center" plugin by:
+        ["lyndsey-ferguson/@lyndseydf"]
+      end
+
+      def self.is_supported?(platform)
+        platform == :ios
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/retry_tests/actions/suppress_tests.rb

@@ -0,0 +1,93 @@
+module Fastlane
+  module Actions
+    class SuppressTestsAction < Action
+      require 'xcodeproj'
+
+      def self.run(params)
+        project_path = params[:xcodeproj]
+        all_tests_to_skip = params[:tests]
+        scheme = params[:scheme]
+
+        scheme_filepaths = Dir.glob("#{project_path}/{xcshareddata,xcuserdata}/**/xcschemes/#{scheme || '*'}.xcscheme")
+        if scheme_filepaths.length.zero?
+          UI.user_error!("Error: cannot find any scheme named #{scheme}") unless scheme.nil?
+          UI.user_error!("Error: cannot find any schemes in the Xcode project")
+        end
+
+        scheme_filepaths.each do |scheme_filepath|
+          is_dirty = false
+          xcscheme = Xcodeproj::XCScheme.new(scheme_filepath)
+          xcscheme.test_action.testables.each do |testable|
+            buildable_name = File.basename(testable.buildable_references[0].buildable_name, '.xctest')
+
+            tests_to_skip = all_tests_to_skip.select { |test| test.start_with?(buildable_name) }
+                                             .map { |test| test.sub("#{buildable_name}/", '') }
+
+            tests_to_skip.each do |test_to_skip|
+              skipped_test = Xcodeproj::XCScheme::TestAction::TestableReference::SkippedTest.new
+              skipped_test.identifier = test_to_skip
+              testable.add_skipped_test(skipped_test)
+              is_dirty = true
+            end
+          end
+          xcscheme.save! if is_dirty
+        end
+        nil
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "Suppresses specific tests in a specific or all Xcode Schemes in a given project"
+      end
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(
+            key: :xcodeproj,
+            env_name: "FL_SUPPRESS_TESTS_XCODE_PROJECT",
+            description: "The file path to the Xcode project file to modify",
+            verify_block: proc do |path|
+              UI.user_error!("Error: Xcode project file path not given!") unless path and !path.empty?
+              UI.user_error!("Error: Xcode project '#{path}' not found!") unless Dir.exist?(path)
+            end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :tests,
+            env_name: "FL_SUPPRESS_TESTS_TESTS_TO_SUPPRESS",
+            description: "A list of tests to suppress",
+            verify_block: proc do |tests|
+              UI.user_error!("Error: no tests were given to suppress!") unless tests and !tests.empty?
+              tests.each do |test_identifier|
+                is_valid_test_identifier = %r{^[a-zA-Z][a-zA-Z0-9]+\/[a-zA-Z][a-zA-Z0-9]+(\/test[a-zA-Z0-9]+)?$} =~ test_identifier
+                unless is_valid_test_identifier
+                  UI.user_error!("Error: invalid test identifier '#{test_identifier}'. It must be in the format of 'Testable/TestSuiteToSuppress' or 'Testable/TestSuiteToSuppress/testToSuppress'")
+                end
+              end
+            end,
+            type: Array
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :scheme,
+            optional: true,
+            env_name: "FL_SUPPRESS_TESTS_SCHEME_TO_UPDATE",
+            description: "The Xcode scheme where the tests should be suppressed",
+            verify_block: proc do |scheme_name|
+              UI.user_error!("Error: Xcode Scheme '#{scheme_name}' is not valid!") if scheme_name and scheme_name.empty?
+            end
+          )
+        ]
+      end
+
+      def self.authors
+        ["lyndsey-ferguson/@lyndseydf"]
+      end
+
+      def self.is_supported?(platform)
+        platform == :ios
+      end
+    end
+  end
+end