fastlane-plugin-retry_tests 1.0.1

data/lib/fastlane/plugin/retry_tests/actions/suppress_tests_from_junit.rb

@@ -0,0 +1,102 @@
+module Fastlane
+  module Actions
+    class SuppressTestsFromJunitAction < Action
+      def self.run(params)
+        project_path = params[:xcodeproj]
+        scheme = params[:scheme]
+
+        scheme_filepaths = Dir.glob("#{project_path}/{xcshareddata,xcuserdata}/**/xcschemes/#{scheme || '*'}.xcscheme")
+        if scheme_filepaths.length.zero?
+          UI.user_error!("Error: cannot find any scheme named #{scheme}") unless scheme.nil?
+          UI.user_error!("Error: cannot find any schemes in the Xcode project")
+        end
+
+        testables = Hash.new([])
+        desired_passed_status = (params[:suppress_type] == :passing)
+
+        report = ::TestCenter::Helper::XcodeJunit::Report.new(params[:junit])
+
+        report.testables.each do |testable|
+          testables[testable.name] = []
+          testable.testsuites.each do |testsuite|
+            testsuite.testcases.each do |testcase|
+              if testcase.passed? == desired_passed_status
+                testables[testable.name] << testcase.skipped_test
+              end
+            end
+          end
+        end
+
+        scheme_filepaths.each do |scheme_filepath|
+          is_dirty = false
+          xcscheme = Xcodeproj::XCScheme.new(scheme_filepath)
+
+          xcscheme.test_action.testables.each do |testable|
+            buildable_name = testable.buildable_references[0].buildable_name
+            testables[buildable_name].each do |skipped_test|
+              testable.add_skipped_test(skipped_test)
+              is_dirty = true
+            end
+          end
+          xcscheme.save! if is_dirty
+        end
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "Uses a junit xml report file to suppress either passing or failing tests in an Xcode Scheme"
+      end
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(
+            key: :xcodeproj,
+            env_name: "FL_SUPPRESS_TESTS_FROM_JUNIT_XCODE_PROJECT",
+            description: "The file path to the Xcode project file to modify",
+            verify_block: proc do |path|
+              UI.user_error!("Error: Xcode project file path not given!") unless path and !path.empty?
+              UI.user_error!("Error: Xcode project '#{path}' not found!") unless Dir.exist?(path)
+            end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :junit,
+            env_name: "FL_SUPPRESS_TESTS_FROM_JUNIT_JUNIT_REPORT",
+            description: "The junit xml report file from which to collect the tests to suppress",
+            verify_block: proc do |path|
+              UI.user_error!("Error: cannot find the junit xml report file '#{path}'") unless File.exist?(path)
+            end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :scheme,
+            optional: true,
+            env_name: "FL_SUPPRESS_TESTS_FROM_JUNIT_SCHEME_TO_UPDATE",
+            description: "The Xcode scheme where the tests should be suppressed",
+            verify_block: proc do |scheme_name|
+              UI.user_error!("Error: Xcode Scheme '#{scheme_name}' is not valid!") if scheme_name and scheme_name.empty?
+            end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :suppress_type,
+            type: Symbol,
+            env_name: "FL_SUPPRESS_TESTS_FROM_JUNIT_SUPPRESS_TYPE",
+            description: "Tests to suppress are either :failed or :passing",
+            verify_block: proc do |type|
+              UI.user_error!("Error: suppress type ':#{type}' is invalid! Only :failed or :passing are valid types") unless %i[failed passing].include?(type)
+            end
+          )
+        ]
+      end
+
+      def self.authors
+        ["lyndsey-ferguson/@lyndseydf"]
+      end
+
+      def self.is_supported?(platform)
+        platform == :ios
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/retry_tests/actions/suppressed_tests.rb

@@ -0,0 +1,72 @@
+module Fastlane
+  module Actions
+    class SuppressedTestsAction < Action
+      require 'set'
+
+      def self.run(params)
+        project_path = params[:xcodeproj]
+        scheme = params[:scheme]
+
+        scheme_filepaths = Dir.glob("#{project_path}/{xcshareddata,xcuserdata}/**/xcschemes/#{scheme || '*'}.xcscheme")
+        if scheme_filepaths.length.zero?
+          UI.user_error!("Error: cannot find any scheme named #{scheme}") unless scheme.nil?
+          UI.user_error!("Error: cannot find any schemes in the Xcode project")
+        end
+
+        skipped_tests = Set.new
+        scheme_filepaths.each do |scheme_filepath|
+          xcscheme = Xcodeproj::XCScheme.new(scheme_filepath)
+          xcscheme.test_action.testables.each do |testable|
+            buildable_name = testable.buildable_references[0]
+                                     .buildable_name
+
+            buildable_name = File.basename(buildable_name, '.xctest')
+            testable.skipped_tests.map do |skipped_test|
+              skipped_tests.add("#{buildable_name}/#{skipped_test.identifier}")
+            end
+          end
+        end
+        skipped_tests.to_a
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "Retrieves a list of tests that are suppressed in a specific or all Xcode Schemes in a project"
+      end
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(
+            key: :xcodeproj,
+            env_name: "FL_SUPPRESSED_TESTS_XCODE_PROJECT",
+            description: "The file path to the Xcode project file to read the skipped tests from",
+            verify_block: proc do |path|
+              UI.user_error!("Error: Xcode project file path not given!") unless path and !path.empty?
+              UI.user_error!("Error: Xcode project '#{path}' not found!") unless Dir.exist?(path)
+            end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :scheme,
+            optional: true,
+            env_name: "FL_SUPPRESSED_TESTS_SCHEME_TO_UPDATE",
+            description: "The Xcode scheme where the suppressed tests may be",
+            verify_block: proc do |scheme_name|
+              UI.user_error!("Error: Xcode Scheme '#{scheme_name}' is not valid!") if scheme_name and scheme_name.empty?
+            end
+          )
+        ]
+      end
+
+      def self.authors
+        ["lyndsey-ferguson/@lyndseydf"]
+      end
+
+      def self.is_supported?(platform)
+        platform == :ios
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/retry_tests/actions/tests_from_junit.rb

@@ -0,0 +1,59 @@
+module Fastlane
+  module Actions
+    class TestsFromJunitAction < Action
+      def self.run(params)
+        report = ::TestCenter::Helper::XcodeJunit::Report.new(params[:junit])
+        passing = []
+        failed = []
+        report.testables.each do |testable|
+          testable.testsuites.each do |testsuite|
+            testsuite.testcases.each do |testcase|
+              if testcase.passed?
+                passing << testcase.identifier
+              else
+                failed << testcase.identifier
+              end
+            end
+          end
+        end
+        {
+          failed: failed,
+          passing: passing
+        }
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "Retrieves the failing and passing tests as reported in a junit xml file"
+      end
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(
+            key: :junit,
+            env_name: "FL_SUPPRESS_TESTS_FROM_JUNIT_JUNIT_REPORT", # The name of the environment variable
+            description: "The junit xml report file from which to collect the tests to suppress",
+            verify_block: proc do |path|
+              UI.user_error!("Error: cannot find the junit xml report file '#{path}'") unless File.exist?(path)
+            end
+          )
+        ]
+      end
+
+      def self.return_value
+        "A Hash with an Array of :passing and :failed tests"
+      end
+
+      def self.authors
+        ["lyndsey-ferguson/lyndseydf"]
+      end
+
+      def self.is_supported?(platform)
+        platform == :ios
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/retry_tests/actions/tests_from_xctestrun.rb

@@ -0,0 +1,65 @@
+require 'plist'
+
+module Fastlane
+  module Actions
+    class TestsFromXctestrunAction < Action
+      def self.run(params)
+        return xctestrun_tests(params[:xctestrun])
+      end
+
+      def self.xctestrun_tests(xctestrun_path)
+        xctestrun = Plist.parse_xml(xctestrun_path)
+        xctestrun_rootpath = File.dirname(xctestrun_path)
+        tests = Hash.new([])
+        xctestrun.each do |testable_name, xctestrun_config|
+          test_identifiers = XCTestList.tests(xctest_bundle_path(xctestrun_rootpath, xctestrun_config))
+          if xctestrun_config.key?('SkipTestIdentifiers')
+            test_identifiers.reject! { |test_identifier| xctestrun_config['SkipTestIdentifiers'].include?(test_identifier) }
+          end
+          tests[testable_name] = test_identifiers.map do |test_identifier|
+            "#{testable_name.shellescape}/#{test_identifier}"
+          end
+        end
+        tests
+      end
+
+      def self.xctest_bundle_path(xctestrun_rootpath, xctestrun_config)
+        xctest_host_path = xctestrun_config['TestHostPath'].sub('__TESTROOT__', xctestrun_rootpath)
+        xctestrun_config['TestBundlePath'].sub('__TESTHOST__', xctest_host_path).sub('__TESTROOT__', xctestrun_rootpath)
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "Retrieves all of the tests from xctest bundles referenced by the xctestrun file"
+      end
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(
+            key: :xctestrun,
+            env_name: "FL_SUPPRESS_TESTS_FROM_XCTESTRUN_FILE",
+            description: "The xctestrun file to use to find where the xctest bundle file is for test retrieval",
+            verify_block: proc do |path|
+              UI.user_error!("Error: cannot find the xctestrun file '#{path}'") unless File.exist?(path)
+            end
+          )
+        ]
+      end
+
+      def self.return_value
+        "A Hash of testable => tests, where testable is the name of the test target and tests is an array of test identifiers"
+      end
+
+      def self.authors
+        ["lyndsey-ferguson/lyndseydf"]
+      end
+
+      def self.is_supported?(platform)
+        platform == :ios
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/retry_tests/helper/correcting_scan_helper.rb

@@ -0,0 +1,164 @@
+module TestCenter
+  module Helper
+    require 'fastlane_core/ui/ui.rb'
+    require 'plist'
+
+    class CorrectingScanHelper
+      attr_reader :retry_total_count
+
+      def initialize(multi_scan_options)
+        @batch_count = multi_scan_options[:batch_count] || 1
+        @output_directory = multi_scan_options[:output_directory] || 'test_results'
+        @try_count = multi_scan_options[:try_count]
+        @retry_total_count = 0
+        @testrun_completed_block = multi_scan_options[:testrun_completed_block]
+        @given_custom_report_file_name = multi_scan_options[:custom_report_file_name]
+        @given_output_types = multi_scan_options[:output_types]
+        @given_output_files = multi_scan_options[:output_files]
+        @scan_options = multi_scan_options.reject do |option, _|
+          %i[
+            output_directory
+            only_testing
+            skip_testing
+            clean
+            try_count
+            batch_count
+            custom_report_file_name
+            fail_build
+            testrun_completed_block
+          ].include?(option)
+        end
+        @test_collector = TestCollector.new(multi_scan_options)
+      end
+
+      def scan
+        tests_passed = true
+        @testables_count = @test_collector.testables.size
+        @test_collector.testables.each do |testable|
+          tests_passed = scan_testable(testable) && tests_passed
+        end
+        tests_passed
+      end
+
+      def scan_testable(testable)
+        tests_passed = true
+        reportnamer = ReportNameHelper.new(
+          @given_output_types,
+          @given_output_files,
+          @given_custom_report_file_name
+        )
+        output_directory = @output_directory
+        testable_tests = @test_collector.testables_tests[testable]
+        if @batch_count > 1 || @testables_count > 1
+          current_batch = 1
+          testable_tests.each_slice((testable_tests.length / @batch_count.to_f).round).to_a.each do |tests_batch|
+            if @testables_count > 1
+              output_directory = File.join(@output_directory, "results-#{testable}")
+            end
+            FastlaneCore::UI.header("Starting test run on testable '#{testable}'")
+            tests_passed = correcting_scan(
+              {
+                only_testing: tests_batch,
+                output_directory: output_directory
+              },
+              current_batch,
+              reportnamer
+            ) && tests_passed
+            current_batch += 1
+            reportnamer.increment
+          end
+        else
+          options = {
+            output_directory: output_directory,
+            only_testing: testable_tests
+          }
+          tests_passed = correcting_scan(options, 1, reportnamer) && tests_passed
+        end
+        collate_reports(output_directory, reportnamer)
+        tests_passed
+      end
+
+      def collate_reports(output_directory, reportnamer)
+        report_files = Dir.glob("#{output_directory}/*#{reportnamer.junit_filextension}").map do |relative_filepath|
+          File.absolute_path(relative_filepath)
+        end
+        if report_files.size > 1
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::CollateJunitReportsAction.available_options,
+            {
+              reports: report_files.sort { |f1, f2| File.mtime(f1) <=> File.mtime(f2) },
+              collated_report: File.absolute_path(File.join(output_directory, reportnamer.junit_reportname))
+            }
+          )
+          Fastlane::Actions::CollateJunitReportsAction.run(config)
+        end
+        retried_junit_reportfiles = Dir.glob("#{output_directory}/**/*-[1-9]*#{reportnamer.junit_filextension}")
+        FileUtils.rm_f(retried_junit_reportfiles)
+      end
+
+      def correcting_scan(scan_run_options, batch, reportnamer)
+        scan_options = @scan_options.merge(scan_run_options)
+        try_count = 0
+        tests_passed = true
+        begin
+          try_count += 1
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::ScanAction.available_options,
+            scan_options.merge(reportnamer.scan_options)
+          )
+          quit_simulators
+          Fastlane::Actions::ScanAction.run(config)
+          @testrun_completed_block && @testrun_completed_block.call(
+            testrun_info(batch, try_count, reportnamer, scan_options[:output_directory])
+          )
+          tests_passed = true
+        rescue FastlaneCore::Interface::FastlaneTestFailure => e
+          FastlaneCore::UI.verbose("Scan failed with #{e}")
+          if try_count < @try_count
+            @retry_total_count += 1
+
+            info = testrun_info(batch, try_count, reportnamer, scan_options[:output_directory])
+            @testrun_completed_block && @testrun_completed_block.call(
+              info
+            )
+            scan_options[:only_testing] = info[:failed].map(&:shellescape)
+            FastlaneCore::UI.message('Re-running scan on only failed tests')
+            reportnamer.increment
+            retry
+          end
+          tests_passed = false
+        end
+        tests_passed
+      end
+
+      def testrun_info(batch, try_count, reportnamer, output_directory)
+        report_filepath = File.join(output_directory, reportnamer.junit_last_reportname)
+        config = FastlaneCore::Configuration.create(
+          Fastlane::Actions::TestsFromJunitAction.available_options,
+          {
+            junit: File.absolute_path(report_filepath)
+          }
+        )
+        junit_results = Fastlane::Actions::TestsFromJunitAction.run(config)
+
+        {
+          failed: junit_results[:failed],
+          passing: junit_results[:passing],
+          batch: batch,
+          try_count: try_count,
+          report_filepath: report_filepath
+        }
+      end
+
+      def quit_simulators
+        Fastlane::Actions.sh("killall -9 'iPhone Simulator' 'Simulator' 'SimulatorBridge' &> /dev/null || true", log: false)
+        launchctl_list_count = 0
+        while Fastlane::Actions.sh('launchctl list | grep com.apple.CoreSimulator.CoreSimulatorService || true', log: false) != ''
+          break if (launchctl_list_count += 1) > 10
+          Fastlane::Actions.sh('launchctl remove com.apple.CoreSimulator.CoreSimulatorService &> /dev/null || true', log: false)
+          sleep(1)
+        end
+      end
+    end
+  end
+end