fastlane-plugin-test_center 3.7.0.parallelizing.alpha.4 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac1d3616de1a2ad7f95123d43b899227325b485c
-  data.tar.gz: 6cdd4e31e41ef0c998b52ad8666c8eb08c0c5e8d
+  metadata.gz: dd7fbaf52ee79115ad9ea8f260bdd35e90868c14
+  data.tar.gz: 03ab87b0d261fde47d37edfa7d997e33a6f3faa7
 SHA512:
-  metadata.gz: '09d7060687e873c399e8d77833db3ba1c7a804e94b9de5ec7b4cd32fda0a5b9813a3b02942d3f9dbe2c26d7df9cab3f369842dc246ac9dd6efac8f94b339133a'
-  data.tar.gz: 8b027123c0ff018e9227326652a6a298e9ed2a90b8b795d1a6c6841a35188f5cf1efb148f156c4c7e706dc72c3ab4d8a693e4833487301763ba90b0a45bb0618
+  metadata.gz: 9cd9189973cddeab19e82f2bee16577eca64b0eea355d0229cf17fa555b6231f536d558ac44a086ddbfb22ae6877cf857f45db20a9fedbe80a392cc0594db7b6
+  data.tar.gz: 3d3747b5497960d65c97b44b20291eea6133c68752449f29d442b6f2e144b4dfc853a7b44eb09523f383749b11d143abbdab028ff73ee314e2bef1e056725ef7

data/lib/fastlane/plugin/test_center.rb

@@ -4,7 +4,7 @@ module Fastlane
   module TestCenter
     # Return all .rb files inside the "actions" and "helper" directory
     def self.all_classes
-      Dir[File.expand_path('**/{actions,helper}/**/*.rb', File.dirname(__FILE__))]
+      Dir[File.expand_path('**/{actions,helper}/*.rb', File.dirname(__FILE__))]
     end
   end
 end

data/lib/fastlane/plugin/test_center/actions/collate_test_result_bundles.rb

@@ -14,7 +14,7 @@ module Fastlane
             collate_bundles(base_bundle_path, other_bundlepath)
           end
         end
-        # FileUtils.rm_rf(params[:collated_bundle])
+        FileUtils.rm_rf(params[:collated_bundle])
         FileUtils.cp_r(base_bundle_path, params[:collated_bundle])
         UI.message("Finished collating test_result bundle to '#{params[:collated_bundle]}'")
       end

data/lib/fastlane/plugin/test_center/actions/multi_scan.rb

@@ -5,7 +5,6 @@ module Fastlane
     require 'shellwords'
     require 'xctest_list'
     require 'plist'
-    require_relative '../helper/multi_scan_manager/runner'
 
     class MultiScanAction < Action
       def self.run(params)
@@ -15,16 +14,14 @@ module Fastlane
             title: "Summary for multi_scan (test_center v#{Fastlane::TestCenter::VERSION})"
           )
         end
-        
-        prepare_for_testing(params._values)
-        
-        runner = ::TestCenter::Helper::MultiScanManager::Runner.new(params.values)
-        tests_passed = runner.run
-        if params[:fail_build] && !tests_passed
-          raise UI.test_failure!('Tests have failed')
+        unless params[:test_without_building] || params[:skip_build]
+          build_for_testing(
+            params._values
+          )
         end
-
-        summary = run_summary(params, tests_passed, runner.retry_total_count)
+        smart_scanner = ::TestCenter::Helper::CorrectingScanHelper.new(params.values)
+        tests_passed = smart_scanner.scan
+        summary = run_summary(params, tests_passed, smart_scanner.retry_total_count)
         unless Helper.test?
           FastlaneCore::PrintTable.print_values(
             config: summary,
@@ -83,51 +80,27 @@ module Fastlane
         }
       end
 
-      def self.prepare_for_testing(scan_options)
-        if scan_options[:test_without_building] || scan_options[:skip_build]
-          prepare_scan_config(scan_options)
-        else
-          build_for_testing(scan_options)
-        end
-      end
-
-      def self.prepare_scan_config(scan_options)
-        Scan.config ||= FastlaneCore::Configuration.create(
+      def self.build_for_testing(scan_options)
+        options_to_remove = %i[
+          try_count
+          batch_count
+          quit_simulators
+          testrun_completed_block
+          test_without_building
+          invocation_based_tests
+          output_types
+          output_files
+        ]
+        config = FastlaneCore::Configuration.create(
           Fastlane::Actions::ScanAction.available_options,
-          scan_options.select { |k,v| %i[project workspace scheme].include?(k) }
+          scan_options.merge(build_for_testing: true).reject { |k, _| options_to_remove.include?(k) }
         )
-      end
-
-      def self.build_for_testing(scan_options)
-          valid_scan_keys = Fastlane::Actions::ScanAction.available_options.map(&:key)
-          scan_options = scan_options.select { |k,v| valid_scan_keys.include?(k) }
-
-          Scan.config = FastlaneCore::Configuration.create(
-            Fastlane::Actions::ScanAction.available_options,
-            scan_options.merge(build_for_testing: true)
-          )
-          values = Scan.config.values(ask: false)
-          values[:xcode_path] = File.expand_path("../..", FastlaneCore::Helper.xcode_path)
-          FastlaneCore::PrintTable.print_values(
-            config: values,
-            hide_keys: [:destination, :slack_url],
-            title: "Summary for scan #{Fastlane::VERSION}"
-          )
-          Scan::Runner.new.run
-          remove_build_report_files
+        Fastlane::Actions::ScanAction.run(config)
 
-          Scan.config._values.delete(:build_for_testing)
-          scan_options[:derived_data_path] = Scan.config[:derived_data_path]
-      end
-
-      def self.remove_build_report_files
-        report_options = Scan::XCPrettyReporterOptionsGenerator.generate_from_scan_config
-        output_files = report_options.instance_variable_get(:@output_files)
-        output_directory = report_options.instance_variable_get(:@output_directory)
-
-        output_files.each do |output_file|
-          FileUtils.rm_f(File.join(output_directory, output_file))
-        end
+        scan_options.merge!(
+          test_without_building: true,
+          derived_data_path: Scan.config[:derived_data_path]
+        ).delete(:build_for_testing)
       end
 
       #####################################################
@@ -214,28 +187,6 @@ module Fastlane
             description: "Comma separated list of the output types (e.g. html, junit, json, json-compilation-database)",
             default_value: "html,junit"
           ),
-          FastlaneCore::ConfigItem.new(
-            key: :parallelize,
-            description: 'Run each batch of tests and/or each test target in parallel on its own Simulator',
-            optional: true,
-            is_string: false,
-            default_value: false,
-            conflicting_options: [:parallel_simulator_count],
-            conflict_block: proc do |value|
-              UI.user_error!("You can't use 'parallelize' and 'parallel_simulator_count' options in one run")
-            end
-          ),
-          FastlaneCore::ConfigItem.new(
-            key: :parallel_simulator_fork_count,
-            description: 'Run  simulators each batch of tests and/or each test target in parallel on its own Simulator',
-            optional: true,
-            is_string: false,
-            default_value: false,
-            conflicting_options: [:parallelize],
-            conflict_block: proc do |value|
-              UI.user_error!("You can't use 'parallelize' and 'parallel_simulator_count' options in one run")
-            end
-          ),
           FastlaneCore::ConfigItem.new(
             key: :testrun_completed_block,
             description: 'A block invoked each time a test run completes',
@@ -385,22 +336,16 @@ module Fastlane
           "
           UI.important(
             'example: ' \\
-            'use the :xctestrun parameter instead of the :project parameter to find, ' \\
-            'build, and test the iOS app.'
+            'multi_scan also works with invocation based tests.'
+          )
+          cocoapods
+          multi_scan(
+            workspace: File.absolute_path('../KiwiDemo/KiwiDemo.xcworkspace'),
+            scheme: 'KiwiDemoTests',
+            try_count: 3,
+            invocation_based_tests: true,
+            fail_build: false
           )
-          Dir.mktmpdir do |derived_data_path|
-            project_path = '/Users/lyndsey.ferguson/repo/fastlane-plugin-test_center/AtomicBoy/AtomicBoy.xcodeproj'
-            command = \"bundle exec fastlane scan --build_for_testing true --project '#{project_path}' --derived_data_path #{derived_data_path} --scheme AtomicBoy\"
-            `#{command}`  
-            xctestrun_file = Dir.glob(\"#{derived_data_path}/Build/Products/AtomicBoy*.xctestrun\").first
-            multi_scan(
-              scheme: 'AtomicBoy',
-              try_count: 3,
-              fail_build: false,
-              xctestrun: xctestrun_file,
-              test_without_building: true
-            )
-          end
           "
         ]
       end

data/lib/fastlane/plugin/test_center/helper/correcting_scan_helper.rb

@@ -0,0 +1,315 @@
+module TestCenter
+  module Helper
+    require 'fastlane_core/ui/ui.rb'
+    require 'plist'
+    require 'json'
+    class CorrectingScanHelper
+      attr_reader :retry_total_count
+
+      def initialize(multi_scan_options)
+        @batch_count = multi_scan_options[:batch_count] || 1
+        @output_directory = multi_scan_options[:output_directory] || 'test_results'
+        @try_count = multi_scan_options[:try_count]
+        @quit_simulators = multi_scan_options[:quit_simulators]
+        @retry_total_count = 0
+        @testrun_completed_block = multi_scan_options[:testrun_completed_block]
+        @given_custom_report_file_name = multi_scan_options[:custom_report_file_name]
+        @given_output_types = multi_scan_options[:output_types]
+        @given_output_files = multi_scan_options[:output_files]
+        @invocation_based_tests = multi_scan_options[:invocation_based_tests]
+        @scan_options = multi_scan_options.reject do |option, _|
+          %i[
+            output_directory
+            only_testing
+            skip_testing
+            clean
+            try_count
+            batch_count
+            invocation_based_tests
+            quit_simulators
+            custom_report_file_name
+            fail_build
+            testrun_completed_block
+            output_types
+            output_files
+          ].include?(option)
+        end
+        @scan_options[:clean] = false
+        @test_collector = TestCollector.new(multi_scan_options)
+      end
+
+      def scan
+        tests_passed = true
+        @testables_count = @test_collector.testables.size
+        @test_collector.testables.each do |testable|
+          tests_passed = scan_testable(testable) && tests_passed
+        end
+        tests_passed
+      end
+
+      def scan_testable(testable)
+        tests_passed = true
+        reportnamer = ReportNameHelper.new(
+          @given_output_types,
+          @given_output_files,
+          @given_custom_report_file_name
+        )
+        output_directory = @output_directory
+
+        if @invocation_based_tests
+          FastlaneCore::UI.header("Starting test run on testable '#{testable}'")
+          if @testables_count > 1
+            output_directory = File.join(@output_directory, "results-#{testable}")
+          end
+          tests_passed = correcting_scan(
+            {
+              output_directory: output_directory,
+              only_testing: [testable]
+            },
+            1,
+            reportnamer
+          ) && tests_passed
+        else
+          testable_tests = @test_collector.testables_tests[testable]
+          if testable_tests.empty?
+            FastlaneCore::UI.important("There are no tests to run in testable '#{testable}'. Skipping")
+            return true
+          end
+
+          if @batch_count > 1 || @testables_count > 1 
+            current_batch = 1
+  
+            testable_tests.each_slice((testable_tests.length / @batch_count.to_f).round).to_a.each do |tests_batch|
+              if @testables_count > 1
+                output_directory = File.join(@output_directory, "results-#{testable}")
+              end
+              FastlaneCore::UI.header("Starting test run on testable '#{testable}'")
+              if @scan_options[:result_bundle]
+                FastlaneCore::UI.message("Clearing out previous test_result bundles in #{output_directory}")
+                FileUtils.rm_rf(Dir.glob("#{output_directory}/*.test_result"))
+              end
+  
+              options =  {
+                output_directory: output_directory,
+                only_testing: tests_batch
+              }
+              tests_passed = correcting_scan(
+                options,
+                current_batch,
+                reportnamer
+              ) && tests_passed
+              current_batch += 1
+              reportnamer.increment
+            end
+          else
+            options = {
+              output_directory: output_directory,
+              only_testing: testable_tests
+            }
+            tests_passed = correcting_scan(options, 1, reportnamer) && tests_passed
+          end
+        end
+        collate_reports(output_directory, reportnamer)
+        tests_passed
+      end
+
+      def test_result_bundlepaths(output_directory, reportnamer)
+        [
+          File.join(output_directory, @scan_options[:scheme]) + '.test_result',
+          File.join(output_directory, @scan_options[:scheme]) + "_#{reportnamer.report_count}.test_result"
+        ]
+      end
+
+      def collate_reports(output_directory, reportnamer)
+        collate_junit_reports(output_directory, reportnamer)
+
+        if reportnamer.includes_html?
+          collate_html_reports(output_directory, reportnamer)
+        end
+
+        if reportnamer.includes_json?
+          collate_json_reports(output_directory, reportnamer)
+        end
+
+        if @scan_options[:result_bundle]
+          collate_test_result_bundles(output_directory, reportnamer)
+        end
+      end
+
+      def passed_test_count_from_summary(summary)
+        /.*Executed (?<test_count>\d+) test, with (?<test_failures>\d+) failure/ =~ summary
+        test_count.to_i - test_failures.to_i
+      end
+
+      def collate_test_result_bundles(output_directory, reportnamer)
+        test_result_bundlepaths = Dir.glob("#{output_directory}/#{@scan_options[:scheme]}*.test_result").map do |relative_test_result_bundle_filepath|
+          File.absolute_path(relative_test_result_bundle_filepath)
+        end
+        if test_result_bundlepaths.size > 1
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::CollateTestResultBundlesAction.available_options,
+            {
+              bundles: test_result_bundlepaths.sort { |f1, f2| File.mtime(f1) <=> File.mtime(f2) },
+              collated_bundle: File.join(output_directory, @scan_options[:scheme]) + '.test_result'
+            }
+          )
+          Fastlane::Actions::CollateTestResultBundlesAction.run(config)
+        end
+        retried_test_result_bundlepaths = Dir.glob("#{output_directory}/#{@scan_options[:scheme]}_*.test_result")
+        FileUtils.rm_rf(retried_test_result_bundlepaths)
+      end
+
+      def collate_json_reports(output_directory, reportnamer)
+        report_filepaths = Dir.glob("#{output_directory}/#{reportnamer.json_fileglob}").map do |relative_filepath|
+          File.absolute_path(relative_filepath)
+        end
+        if report_filepaths.size > 1
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::CollateJsonReportsAction.available_options,
+            {
+              reports: report_filepaths.sort { |f1, f2| File.mtime(f1) <=> File.mtime(f2) },
+              collated_report: File.absolute_path(File.join(output_directory, reportnamer.json_reportname))
+            }
+          )
+          Fastlane::Actions::CollateJsonReportsAction.run(config)
+        end
+        retried_json_reportfiles = Dir.glob("#{output_directory}/#{reportnamer.json_numbered_fileglob}")
+        FileUtils.rm_f(retried_json_reportfiles)
+      end
+
+      def collate_html_reports(output_directory, reportnamer)
+        report_files = Dir.glob("#{output_directory}/#{reportnamer.html_fileglob}").map do |relative_filepath|
+          File.absolute_path(relative_filepath)
+        end
+        if report_files.size > 1
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::CollateHtmlReportsAction.available_options,
+            {
+              reports: report_files.sort { |f1, f2| File.mtime(f1) <=> File.mtime(f2) },
+              collated_report: File.absolute_path(File.join(output_directory, reportnamer.html_reportname))
+            }
+          )
+          Fastlane::Actions::CollateHtmlReportsAction.run(config)
+        end
+        retried_html_reportfiles = Dir.glob("#{output_directory}/#{reportnamer.html_numbered_fileglob}")
+        FileUtils.rm_f(retried_html_reportfiles)
+      end
+
+      def collate_junit_reports(output_directory, reportnamer)
+        report_files = Dir.glob("#{output_directory}/#{reportnamer.junit_fileglob}").map do |relative_filepath|
+          File.absolute_path(relative_filepath)
+        end
+        if report_files.size > 1
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::CollateJunitReportsAction.available_options,
+            {
+              reports: report_files.sort { |f1, f2| File.mtime(f1) <=> File.mtime(f2) },
+              collated_report: File.absolute_path(File.join(output_directory, reportnamer.junit_reportname))
+            }
+          )
+          Fastlane::Actions::CollateJunitReportsAction.run(config)
+        end
+        retried_junit_reportfiles = Dir.glob("#{output_directory}/#{reportnamer.junit_numbered_fileglob}")
+        FileUtils.rm_f(retried_junit_reportfiles)
+      end
+
+      def correcting_scan(scan_run_options, batch, reportnamer)
+        scan_options = @scan_options.merge(scan_run_options)
+        try_count = 0
+        tests_passed = true
+        xcpretty_json_file_output = ENV['XCPRETTY_JSON_FILE_OUTPUT']
+        begin
+          try_count += 1
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::ScanAction.available_options,
+            scan_options.merge(reportnamer.scan_options)
+          )
+          quit_simulators
+          if reportnamer.includes_json?
+            ENV['XCPRETTY_JSON_FILE_OUTPUT'] = File.join(
+              scan_options[:output_directory],
+              reportnamer.json_last_reportname
+            )
+          end
+          Fastlane::Actions::ScanAction.run(config)
+          @testrun_completed_block && @testrun_completed_block.call(
+            testrun_info(batch, try_count, reportnamer, scan_options[:output_directory])
+          )
+          tests_passed = true
+        rescue FastlaneCore::Interface::FastlaneTestFailure => e
+          FastlaneCore::UI.verbose("Scan failed with #{e}")
+          info = testrun_info(batch, try_count, reportnamer, scan_options[:output_directory])
+          @testrun_completed_block && @testrun_completed_block.call(
+            info
+          )
+          if try_count < @try_count
+            @retry_total_count += 1
+            scan_options.delete(:code_coverage)
+            last_failed_tests = info[:failed].map(&:shellsafe_testidentifier)
+            last_failed_tests = last_failed_tests.map(&:strip_testcase) if @invocation_based_tests
+            scan_options[:only_testing] = last_failed_tests.uniq
+            FastlaneCore::UI.message('Re-running scan on only failed tests')
+            reportnamer.increment
+            if @scan_options[:result_bundle]
+              built_test_result, moved_test_result = test_result_bundlepaths(
+                scan_options[:output_directory], reportnamer
+              )
+              FileUtils.mv(built_test_result, moved_test_result)
+            end
+            retry
+          end
+          tests_passed = false
+        ensure
+          ENV['XCPRETTY_JSON_FILE_OUTPUT'] = xcpretty_json_file_output
+        end
+        tests_passed
+      end
+
+      def testrun_info(batch, try_count, reportnamer, output_directory)
+        report_filepath = File.join(output_directory, reportnamer.junit_last_reportname)
+        config = FastlaneCore::Configuration.create(
+          Fastlane::Actions::TestsFromJunitAction.available_options,
+          {
+            junit: File.absolute_path(report_filepath)
+          }
+        )
+        junit_results = Fastlane::Actions::TestsFromJunitAction.run(config)
+
+        info = {
+          failed: junit_results[:failed],
+          passing: junit_results[:passing],
+          batch: batch,
+          try_count: try_count,
+          report_filepath: report_filepath
+        }
+        if reportnamer.includes_html?
+          html_report_filepath = File.join(output_directory, reportnamer.html_last_reportname)
+          info[:html_report_filepath] = html_report_filepath
+        end
+        if reportnamer.includes_json?
+          json_report_filepath = File.join(output_directory, reportnamer.json_last_reportname)
+          info[:json_report_filepath] = json_report_filepath
+        end
+        if @scan_options[:result_bundle]
+          test_result_suffix = '.test_result'
+          test_result_suffix.prepend("_#{reportnamer.report_count}") unless reportnamer.report_count.zero?
+          test_result_bundlepath = File.join(output_directory, @scan_options[:scheme]) + test_result_suffix
+          info[:test_result_bundlepath] = test_result_bundlepath
+        end
+        info
+      end
+
+      def quit_simulators
+        return unless @quit_simulators
+
+        Fastlane::Actions.sh("killall -9 'iPhone Simulator' 'Simulator' 'SimulatorBridge' &> /dev/null || true", log: false)
+        launchctl_list_count = 0
+        while Fastlane::Actions.sh('launchctl list | grep com.apple.CoreSimulator.CoreSimulatorService || true', log: false) != ''
+          break if (launchctl_list_count += 1) > 10
+          Fastlane::Actions.sh('launchctl remove com.apple.CoreSimulator.CoreSimulatorService &> /dev/null || true', log: false)
+          sleep(1)
+        end
+      end
+    end
+  end
+end