fastlane-plugin-test_center 3.7.0 → 3.8.0.parallelizing.beta.1

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/report_collator.rb

@@ -0,0 +1,129 @@
+module TestCenter
+  module Helper
+    module MultiScanManager
+      class ReportCollator
+        CollateJunitReportsAction = Fastlane::Actions::CollateJunitReportsAction
+        CollateHtmlReportsAction = Fastlane::Actions::CollateHtmlReportsAction
+        CollateJsonReportsAction = Fastlane::Actions::CollateJsonReportsAction
+        CollateTestResultBundlesAction = Fastlane::Actions::CollateTestResultBundlesAction
+
+        def initialize(params)
+          @source_reports_directory_glob = params[:source_reports_directory_glob]
+          @output_directory = params[:output_directory]
+          @reportnamer = params[:reportnamer]
+          @scheme = params[:scheme]
+          @result_bundle = params[:result_bundle]
+          @suffix = params[:suffix] || ''
+        end
+
+        def collate
+          collate_junit_reports
+          collate_html_reports
+          collate_json_reports
+          collate_test_result_bundles
+        end
+
+        def sort_globbed_files(glob)
+          files = Dir.glob(glob).map do |relative_filepath|
+            File.absolute_path(relative_filepath)
+          end
+          files.sort! { |f1, f2| File.mtime(f1) <=> File.mtime(f2) }
+        end
+
+        def delete_globbed_intermediatefiles(glob)
+          retried_reportfiles = Dir.glob(glob)
+          FileUtils.rm_f(retried_reportfiles)
+        end
+
+        # :nocov:
+        def create_config(klass, options)
+          FastlaneCore::Configuration.create(klass.available_options, options)
+        end
+        # :nocov:
+
+        def collate_junit_reports
+          glob = "#{@source_reports_directory_glob}/#{@reportnamer.junit_fileglob}"
+          report_files = sort_globbed_files(glob)
+          collated_file =  File.absolute_path(File.join(@output_directory, @reportnamer.junit_reportname(@suffix)))
+          if report_files.size > 1
+            config = create_config(
+              CollateJunitReportsAction,
+              {
+                reports: report_files,
+                collated_report: collated_file
+              }
+            )
+            CollateJunitReportsAction.run(config)
+            FileUtils.rm_rf(report_files - [collated_file])
+          elsif report_files.size == 1 && report_files.first != collated_file
+            FileUtils.mkdir_p(File.dirname(collated_file))
+            FileUtils.mv(report_files.first, collated_file)
+          end
+        end
+
+        def collate_html_reports
+          return unless @reportnamer.includes_html?
+
+          report_files = sort_globbed_files("#{@source_reports_directory_glob}/#{@reportnamer.html_fileglob}")
+          collated_file = File.absolute_path(File.join(@output_directory, @reportnamer.html_reportname(@suffix)))
+          if report_files.size > 1
+            config = create_config(
+              CollateJunitReportsAction,
+              {
+                reports: report_files,
+                collated_report: collated_file
+              }
+            )
+            CollateHtmlReportsAction.run(config)
+            FileUtils.rm_rf(report_files - [collated_file])
+          elsif report_files.size == 1 && report_files.first != collated_file
+            FileUtils.mkdir_p(File.dirname(collated_file))
+            FileUtils.mv(report_files.first, collated_file)
+          end
+        end
+
+        def collate_json_reports
+          return unless @reportnamer.includes_json?
+
+          report_files = sort_globbed_files("#{@source_reports_directory_glob}/#{@reportnamer.json_fileglob}")
+          collated_file = File.absolute_path(File.join(@output_directory, @reportnamer.json_reportname(@suffix)))
+          if report_files.size > 1
+            config = create_config(
+              CollateJsonReportsAction,
+              {
+                reports: report_files,
+                collated_report: collated_file
+              }
+            )
+            CollateJsonReportsAction.run(config)
+            FileUtils.rm_rf(report_files - [collated_file])
+          elsif report_files.size == 1 && report_files.first != collated_file
+            FileUtils.mkdir_p(File.dirname(collated_file))
+            FileUtils.mv(report_files.first, collated_file)
+          end
+        end
+
+        def collate_test_result_bundles
+          return unless @result_bundle
+
+          test_result_bundlepaths = sort_globbed_files("#{@source_reports_directory_glob}/#{@scheme}*.test_result")
+          collated_test_result_bundlepath = File.absolute_path("#{File.join(@output_directory, @scheme)}.test_result")
+          if test_result_bundlepaths.size > 1
+            config = create_config(
+              CollateTestResultBundlesAction,
+              {
+                bundles: test_result_bundlepaths,
+                collated_bundle: collated_test_result_bundlepath
+              }
+            )
+            CollateTestResultBundlesAction.run(config)
+            FileUtils.rm_rf(test_result_bundlepaths - [collated_test_result_bundlepath])
+          elsif test_result_bundlepaths.size == 1 && test_result_bundlepaths.first != collated_test_result_bundlepath
+            FileUtils.mkdir_p(File.dirname(collated_test_result_bundlepath))
+            FileUtils.mv(test_result_bundlepaths.first, collated_test_result_bundlepath)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/retrying_scan.rb

@@ -0,0 +1,77 @@
+module TestCenter
+  module Helper
+    module MultiScanManager
+      class RetryingScan
+        def initialize(options = {})
+          @options = options
+          @retrying_scan_helper = RetryingScanHelper.new(@options)
+        end
+
+        # :nocov:
+        def scan_config
+          Scan.config
+        end
+
+        def scan_cache
+          Scan.cache
+        end
+        # :nocov:
+
+        def prepare_scan_config_for_destination
+          # this allows multi_scan's `destination` option to be picked up by `scan`
+          scan_config._values.delete(:device)
+          scan_config._values.delete(:devices)
+          scan_cache.clear
+        end
+
+        def update_scan_options
+          valid_scan_keys = Fastlane::Actions::ScanAction.available_options.map(&:key)
+          scan_options = @options.select { |k,v| valid_scan_keys.include?(k) }
+                                  .merge(@retrying_scan_helper.scan_options)
+
+          prepare_scan_config_for_destination
+          scan_options.each do |k,v|
+            scan_config.set(k,v) unless v.nil?
+          end
+        end
+
+        # :nocov:
+        def self.run(options)
+          RetryingScan.new(options).run
+        end
+        # :nocov:
+
+        def run
+          try_count = @options[:try_count] || 1
+          begin
+            # TODO move delete_xcresults to `before_testrun`
+            @retrying_scan_helper.before_testrun
+            update_scan_options
+            
+            values = scan_config.values(ask: false)
+            values[:xcode_path] = File.expand_path("../..", FastlaneCore::Helper.xcode_path)
+            FastlaneCore::PrintTable.print_values(
+              config: values,
+              hide_keys: [:destination, :slack_url],
+              title: "Summary for scan #{Fastlane::VERSION}"
+            ) unless FastlaneCore::Helper.test?
+
+            Scan::Runner.new.run
+            @retrying_scan_helper.after_testrun
+            true
+          rescue FastlaneCore::Interface::FastlaneTestFailure => e
+            FastlaneCore::UI.message("retrying_scan after test failure")
+            @retrying_scan_helper.after_testrun(e)
+            retry if @retrying_scan_helper.testrun_count < try_count
+            false
+          rescue FastlaneCore::Interface::FastlaneBuildFailure => e
+            FastlaneCore::UI.message("retrying_scan after build failure")
+            @retrying_scan_helper.after_testrun(e)
+            retry if @retrying_scan_helper.testrun_count < try_count
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/retrying_scan_helper.rb

@@ -0,0 +1,272 @@
+module TestCenter
+  module Helper
+    module MultiScanManager
+      require_relative 'device_manager'
+
+      class RetryingScanHelper
+
+        attr_reader :testrun_count
+
+        def initialize(options)
+          raise ArgumentError, 'Do not use the :device or :devices option. Instead use the :destination option.' if (options.key?(:device) or options.key?(:devices))
+
+          @options = options
+          @testrun_count = 0
+          @xcpretty_json_file_output = ENV['XCPRETTY_JSON_FILE_OUTPUT']
+          @reportnamer = ReportNameHelper.new(
+            @options[:output_types],
+            @options[:output_files],
+            @options[:custom_report_file_name]
+          )
+        end
+        
+        def before_testrun
+          remove_preexisting_test_result_bundles
+          delete_xcresults # has to be performed _after_ moving a *.test_result
+          quit_simulator
+          set_json_env
+          print_starting_scan_message
+        end
+
+        def quit_simulator
+          return unless @options[:quit_simulators]
+
+          @options.fetch(:destination).each do |destination|
+            if /id=(?<udid>\w+),?/ =~ destination
+              `xcrun simctl shutdown #{udid} 2>/dev/null`
+            end
+          end
+        end
+
+        def delete_xcresults
+          derived_data_path = File.expand_path(@options[:derived_data_path] || Scan.config[:derived_data_path])
+          xcresults = Dir.glob("#{derived_data_path}/Logs/Test/*.xcresult")
+          FastlaneCore::UI.message("Deleting xcresults: #{xcresults}")
+          FileUtils.rm_rf(xcresults)
+        end
+
+        def output_directory
+          absolute_output_directory = File.absolute_path(@options[:output_directory])
+          if @options[:batch]
+            testable = @options.fetch(:only_testing, ['']).first.split('/').first || ''
+            absolute_output_directory = File.join(absolute_output_directory, "#{testable}-batch-#{@options[:batch]}")
+          end
+          absolute_output_directory
+        end
+
+        def print_starting_scan_message
+          scan_message = "Starting scan ##{@testrun_count + 1} with #{@options.fetch(:only_testing, []).size} tests"
+          scan_message << " for batch ##{@options[:batch]}" unless @options[:batch].nil?
+          FastlaneCore::UI.message("#{scan_message}.")
+        end
+
+        def set_json_env
+          return unless @reportnamer.includes_json?
+
+          ENV['XCPRETTY_JSON_FILE_OUTPUT'] = File.join(
+            output_directory,
+            @reportnamer.json_last_reportname
+          )
+        end
+
+        def reset_json_env
+          return unless @reportnamer.includes_json?
+
+          ENV['XCPRETTY_JSON_FILE_OUTPUT'] = @xcpretty_json_file_output
+        end
+
+        def remove_preexisting_test_result_bundles
+          return unless @options[:result_bundle]
+
+          glob_pattern = "#{output_directory}/*.test_result"
+          preexisting_test_result_bundles = Dir.glob(glob_pattern)
+          FileUtils.rm_rf(preexisting_test_result_bundles)
+        end
+
+        def scan_options
+          valid_scan_keys = Fastlane::Actions::ScanAction.available_options.map(&:key)
+          xcargs = @options[:xcargs]
+          retrying_scan_options = @reportnamer.scan_options.merge(
+            {
+              output_directory: output_directory,
+              xcargs: "#{xcargs} -parallel-testing-enabled NO"
+            }
+          )
+          @options.select { |k,v| valid_scan_keys.include?(k) }
+            .merge(retrying_scan_options)
+        end
+
+        # after_testrun methods
+
+        def after_testrun(exception = nil)
+          @testrun_count = @testrun_count + 1
+          if exception.kind_of?(FastlaneCore::Interface::FastlaneTestFailure)
+            handle_test_failure
+          elsif exception.kind_of?(FastlaneCore::Interface::FastlaneBuildFailure)
+            handle_build_failure(exception)
+          else
+            handle_success
+          end
+          collate_reports
+        end
+
+        def handle_success
+          send_callback_testrun_info
+          move_test_result_bundle_for_next_run
+          reset_json_env
+        end
+        
+        def collate_reports
+          report_collator_options = {
+            source_reports_directory_glob: output_directory,
+            output_directory: output_directory,
+            reportnamer: @reportnamer,
+            scheme: @options[:scheme],
+            result_bundle: @options[:result_bundle]
+          }
+          TestCenter::Helper::MultiScanManager::ReportCollator.new(report_collator_options).collate
+        end
+
+        def handle_test_failure
+          send_callback_testrun_info
+          reset_simulators
+          move_test_result_bundle_for_next_run
+          update_scan_options
+          @reportnamer.increment
+        end
+
+        def send_callback_testrun_info(additional_info = {})
+          return unless @options[:testrun_completed_block]
+
+          report_filepath = nil
+          junit_results = {}
+          unless additional_info.key?(:test_operation_failure)
+            report_filepath = File.join(output_directory, @reportnamer.junit_last_reportname)
+  
+            config = FastlaneCore::Configuration.create(
+              Fastlane::Actions::TestsFromJunitAction.available_options,
+              {
+                junit: File.absolute_path(report_filepath)
+              }
+            )
+            junit_results = Fastlane::Actions::TestsFromJunitAction.run(config)
+          end
+
+          info = {
+            failed: junit_results[:failed],
+            passing: junit_results[:passing],
+            batch: @options[:batch] || 1,
+            try_count: @testrun_count,
+            report_filepath: report_filepath
+          }.merge(additional_info)
+
+          if @reportnamer.includes_html?
+            html_report_filepath = File.join(output_directory, @reportnamer.html_last_reportname)
+            info[:html_report_filepath] = html_report_filepath
+          end
+          if @reportnamer.includes_json?
+            json_report_filepath = File.join(output_directory, @reportnamer.json_last_reportname)
+            info[:json_report_filepath] = json_report_filepath
+          end
+          if @options[:result_bundle]
+            test_result_suffix = '.test_result'
+            test_result_suffix.prepend("-#{@reportnamer.report_count}") unless @reportnamer.report_count.zero?
+            test_result_bundlepath = File.join(output_directory, @options[:scheme]) + test_result_suffix
+            info[:test_result_bundlepath] = test_result_bundlepath
+          end
+          @options[:testrun_completed_block].call(info)
+        end
+
+        def update_scan_options
+          update_only_testing
+          turn_off_code_coverage
+        end
+
+        def turn_off_code_coverage
+          # Turn off code coverage as code coverage reports are not merged and
+          # the first, more valuable, report will be overwritten
+          @options.delete(:code_coverage)
+        end
+
+        def update_only_testing
+          report_filepath = File.join(output_directory, @reportnamer.junit_last_reportname)
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::TestsFromJunitAction.available_options,
+            {
+              junit: File.absolute_path(report_filepath)
+            }
+          )
+          @options[:only_testing] = Fastlane::Actions::TestsFromJunitAction.run(config)[:failed]
+          if @options[:invocation_based_tests]
+            @options[:only_testing] = @options[:only_testing].map(&:strip_testcase).uniq
+          end
+        end
+        
+        def reset_simulators
+          return unless @options[:reset_simulators]
+
+          @options[:simulators].each(&:reset)
+        end
+
+        def handle_build_failure(exception)
+          test_operation_failure = ''
+
+          test_session_last_messages = last_lines_of_test_session_log
+          test_operation_failure_match = /Test operation failure: (?<test_operation_failure>.*)$/ =~ test_session_last_messages
+          if test_operation_failure_match.nil?
+            test_operation_failure = 'Unknown test operation failure'
+          end
+          
+          case test_operation_failure
+          when /Test runner exited before starting test execution/
+            FastlaneCore::UI.error(test_operation_failure)
+          when /Lost connection to testmanagerd/
+            FastlaneCore::UI.error(test_operation_failure)
+            FastlaneCore::UI.important("com.apple.CoreSimulator.CoreSimulatorService may have become corrupt, consider quitting it")
+            if @options[:quit_core_simulator_service]
+              Fastlane::Actions::RestartCoreSimulatorServiceAction.run
+            end
+          else
+            FastlaneCore::UI.error(test_operation_failure)
+            send_callback_testrun_info(test_operation_failure: test_operation_failure)
+            raise exception
+          end
+          if @options[:reset_simulators]
+            @options[:simulators].each do |simulator|
+              simulator.reset
+            end
+          end
+          send_callback_testrun_info(test_operation_failure: test_operation_failure)
+        end
+
+        def last_lines_of_test_session_log
+          derived_data_path = File.expand_path(@options[:derived_data_path])
+          test_session_logs = Dir.glob("#{derived_data_path}/Logs/Test/*.xcresult/*_Test/Diagnostics/**/Session-*.log")
+          return '' if test_session_logs.empty?
+          
+          test_session_logs.sort! { |logfile1, logfile2| File.mtime(logfile1) <=> File.mtime(logfile2) }
+          test_session = File.open(test_session_logs.last)
+          backwards_seek_offset = -1 * [1000, test_session.stat.size].min
+          test_session.seek(backwards_seek_offset, IO::SEEK_END)
+          test_session_last_messages = test_session.read
+        end
+
+        def move_test_result_bundle_for_next_run
+          return unless @options[:result_bundle]
+
+          glob_pattern = "#{output_directory}/*.test_result"
+          preexisting_test_result_bundles = Dir.glob(glob_pattern)
+          unnumbered_test_result_bundles = preexisting_test_result_bundles.reject do |test_result|
+            test_result =~ /.*-\d+\.test_result/
+          end
+          src_test_bundle = unnumbered_test_result_bundles.first
+          dst_test_bundle_parent_dir = File.dirname(src_test_bundle)
+          dst_test_bundle_basename = File.basename(src_test_bundle, '.test_result')
+          dst_test_bundle = "#{dst_test_bundle_parent_dir}/#{dst_test_bundle_basename}-#{@testrun_count}.test_result"
+          FileUtils.mkdir_p(dst_test_bundle)
+          FileUtils.mv(src_test_bundle, dst_test_bundle)
+        end
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/runner.rb

@@ -0,0 +1,109 @@
+
+module TestCenter
+  module Helper
+    module MultiScanManager
+      require 'fastlane_core/ui/ui.rb'
+      require 'plist'
+      require 'json'
+      require 'shellwords'
+      require 'snapshot/reset_simulators'
+      
+      class Runner
+        attr_reader :retry_total_count
+
+        def initialize(multi_scan_options)
+          @test_collector = TestCenter::Helper::TestCollector.new(multi_scan_options)
+          @options = multi_scan_options.merge(
+            clean: false,
+            disable_concurrent_testing: true
+          )
+          @batch_count = @test_collector.test_batches.size
+        end
+
+        def output_directory
+          @options.fetch(:output_directory, 'test_results')
+        end
+
+        def run
+          if @options[:invocation_based_tests]
+            run_invocation_based_tests
+          else
+            run_test_batches
+          end
+        end
+        
+        def run_invocation_based_tests
+          @options[:only_testing] = @options[:only_testing]&.map(&:strip_testcase)&.uniq
+          @options[:skip_testing] = @options[:skip_testing]&.map(&:strip_testcase)&.uniq
+          
+          RetryingScan.run(@options.reject { |key| %i[device devices force_quit_simulator].include?(key) } )
+        end
+        
+        def run_test_batches
+          test_batch_results = []
+          pool_options = @options.reject { |key| %i[device devices force_quit_simulator].include?(key) }
+          pool_options[:test_batch_results] = test_batch_results
+
+          pool = TestBatchWorkerPool.new(pool_options)
+          pool.setup_workers
+          
+          remaining_test_batches = @test_collector.test_batches.clone
+          remaining_test_batches.each_with_index do |test_batch, current_batch_index|
+            worker = pool.wait_for_worker              
+            FastlaneCore::UI.message("Starting test run #{current_batch_index}")
+            worker.run(scan_options_for_worker(test_batch, current_batch_index))
+          end
+          pool.wait_for_all_workers
+          collate_batched_reports
+          test_batch_results.reduce(true) { |a, t| a && t }
+        end
+
+        def scan_options_for_worker(test_batch, batch_index)
+          {
+            only_testing: test_batch.map(&:shellsafe_testidentifier),
+            output_directory: output_directory,
+            try_count: @options[:try_count],
+            batch: batch_index + 1,
+            xctestrun: @test_collector.xctestrun_path
+          }
+        end
+  
+        def collate_batched_reports
+          return false unless @batch_count > 1
+
+          @test_collector.testables.each do |testable|
+            collate_batched_reports_for_testable(testable)
+          end
+          true
+        end
+
+        def collate_batched_reports_for_testable(testable)
+          absolute_output_directory = File.join(
+            File.absolute_path(output_directory),
+            testable
+          )
+          source_reports_directory_glob = "#{absolute_output_directory}-batch-*"
+
+          given_custom_report_file_name = @options[:custom_report_file_name]
+          given_output_types = @options[:output_types]
+          given_output_files = @options[:output_files]
+
+          TestCenter::Helper::MultiScanManager::ReportCollator.new(
+            source_reports_directory_glob: source_reports_directory_glob,
+            output_directory: absolute_output_directory,
+            reportnamer: ReportNameHelper.new(
+              given_output_types,
+              given_output_files,
+              given_custom_report_file_name
+            ),
+            scheme: @options[:scheme],
+            result_bundle: @options[:result_bundle]
+          ).collate
+          FileUtils.rm_rf(Dir.glob(source_reports_directory_glob))
+          true
+        end
+      end
+    end
+  end
+end
+

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/simulator_helper.rb

@@ -0,0 +1,49 @@
+module TestCenter
+  module Helper
+    module MultiScanManager
+      class SimulatorHelper
+        def initialize(options)
+          @options = options
+        end
+
+        def setup
+          if @options[:parallel_testrun_count] > 1
+            delete_multi_scan_cloned_simulators
+          end
+        end
+
+        def clone_destination_simulators
+          cloned_simulators = []
+
+          run_count = @options[:parallel_testrun_count] || 0
+          destination_simulator_ids = Scan.config[:destination].map do |destination|
+            destination.split(',id=').last
+          end
+          original_simulators = FastlaneCore::DeviceManager.simulators('iOS').find_all do |simulator|
+            destination_simulator_ids.include?(simulator.udid)
+          end
+          original_simulators.each(&:shutdown)
+          (0...run_count).each do |batch_index|
+            cloned_simulators << []
+            original_simulators.each do |simulator|
+              FastlaneCore::UI.verbose("Cloning simulator")
+              cloned_simulator = simulator.clone
+              new_first_name = simulator.name.sub(/( ?\(.*| ?$)/, " Clone #{batch_index}")
+              new_last_name = "#{self.class.name}<#{self.object_id}>"
+              cloned_simulator.rename("#{new_first_name} #{new_last_name}")
+
+              cloned_simulators.last << cloned_simulator
+            end
+          end
+          cloned_simulators
+        end
+
+        def delete_multi_scan_cloned_simulators
+          FastlaneCore::DeviceManager.simulators('iOS').each do |simulator|
+            simulator.delete if /#{self.class.name}<\d+>/ =~ simulator.name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/test_batch_worker.rb

@@ -0,0 +1,24 @@
+module TestCenter
+  module Helper
+    module MultiScanManager
+      class TestBatchWorker
+        attr_accessor :state
+
+        def initialize(options)
+          @options = options
+          @state = :ready_to_work
+        end
+        
+        def process_results
+          @state = :ready_to_work
+        end
+
+        def run(run_options)
+          self.state = :working
+          @options[:test_batch_results] << RetryingScan.run(@options.merge(run_options))
+          self.state = :ready_to_work
+        end
+      end
+    end
+  end
+end