fastlane-plugin-test_center 3.6.3 → 3.7.0.parallelizing.alpha.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: d4b6e643411a6c7dc79cc1ffdf6f2e92d6eca01a2a31c14c8f2904290f0792a3
-  data.tar.gz: d57683f16900bd9460adfe2c613dff614dc11af4a3cccbb315391a773f7ffd12
+SHA1:
+  metadata.gz: ac1d3616de1a2ad7f95123d43b899227325b485c
+  data.tar.gz: 6cdd4e31e41ef0c998b52ad8666c8eb08c0c5e8d
 SHA512:
-  metadata.gz: 2662f68881f0cfd6e75721f7fd13e9ee229981d1ddaf6b01ba3c1effd1510ba3e0df7542cf04b89ac7f36d9a83c94f11f201bed01246f376a30ad561c806d610
-  data.tar.gz: 5855c0fd93652ac1a034a8a268268c9587ffe5de8420b65d4c1fa4fe614198d4ab9190f63589abaacd961e20517b879b8508453db719c1aff78dd57337b21495
+  metadata.gz: '09d7060687e873c399e8d77833db3ba1c7a804e94b9de5ec7b4cd32fda0a5b9813a3b02942d3f9dbe2c26d7df9cab3f369842dc246ac9dd6efac8f94b339133a'
+  data.tar.gz: 8b027123c0ff018e9227326652a6a298e9ed2a90b8b795d1a6c6841a35188f5cf1efb148f156c4c7e706dc72c3ab4d8a693e4833487301763ba90b0a45bb0618

data/README.md

@@ -107,6 +107,10 @@ Do you have multiple test targets and the normal operation of `:scan` is providi
 
 `test_result` bundles are particularly useful because they contain screenshots of the UI when a UI test fails so you can review what was actually there compared to what you expected.
 
+#### Invocation based tests
+
+If your tests are invocation based like [Kiwi](https://github.com/kiwi-bdd/Kiwi) you need to set `:invocation_based_tests` to handle these tests, because unlike `XCTest`s the list of tests cannot be deterimned before running and also you can't try an exact test (The reruns run the whole file where the test failed).
+
 <details>
     <summary>Example Code (expand to view):</summary>
 <!-- multi_scan examples: begin -->

data/lib/fastlane/plugin/test_center.rb

@@ -4,7 +4,7 @@ module Fastlane
   module TestCenter
     # Return all .rb files inside the "actions" and "helper" directory
     def self.all_classes
-      Dir[File.expand_path('**/{actions,helper}/*.rb', File.dirname(__FILE__))]
+      Dir[File.expand_path('**/{actions,helper}/**/*.rb', File.dirname(__FILE__))]
     end
   end
 end

data/lib/fastlane/plugin/test_center/actions/collate_test_result_bundles.rb

@@ -14,7 +14,7 @@ module Fastlane
             collate_bundles(base_bundle_path, other_bundlepath)
           end
         end
-        FileUtils.rm_rf(params[:collated_bundle])
+        # FileUtils.rm_rf(params[:collated_bundle])
         FileUtils.cp_r(base_bundle_path, params[:collated_bundle])
         UI.message("Finished collating test_result bundle to '#{params[:collated_bundle]}'")
       end

data/lib/fastlane/plugin/test_center/actions/multi_scan.rb

@@ -5,23 +5,26 @@ module Fastlane
     require 'shellwords'
     require 'xctest_list'
     require 'plist'
+    require_relative '../helper/multi_scan_manager/runner'
 
     class MultiScanAction < Action
       def self.run(params)
         unless Helper.test?
           FastlaneCore::PrintTable.print_values(
-            config: params._values.select { |k, _| %i[try_count batch_count fail_build quit_simulators].include?(k) },
+            config: params._values.select { |k, _| %i[try_count batch_count invocation_based_tests fail_build quit_simulators].include?(k) },
             title: "Summary for multi_scan (test_center v#{Fastlane::TestCenter::VERSION})"
           )
         end
-        unless params[:test_without_building] || params[:skip_build]
-          build_for_testing(
-            params._values
-          )
+        
+        prepare_for_testing(params._values)
+        
+        runner = ::TestCenter::Helper::MultiScanManager::Runner.new(params.values)
+        tests_passed = runner.run
+        if params[:fail_build] && !tests_passed
+          raise UI.test_failure!('Tests have failed')
         end
-        smart_scanner = ::TestCenter::Helper::CorrectingScanHelper.new(params.values)
-        tests_passed = smart_scanner.scan
-        summary = run_summary(params, tests_passed, smart_scanner.retry_total_count)
+
+        summary = run_summary(params, tests_passed, runner.retry_total_count)
         unless Helper.test?
           FastlaneCore::PrintTable.print_values(
             config: summary,
@@ -80,26 +83,51 @@ module Fastlane
         }
       end
 
-      def self.build_for_testing(scan_options)
-        options_to_remove = %i[
-          try_count
-          batch_count
-          quit_simulators
-          testrun_completed_block
-          test_without_building
-          output_types
-          output_files
-        ]
-        config = FastlaneCore::Configuration.create(
+      def self.prepare_for_testing(scan_options)
+        if scan_options[:test_without_building] || scan_options[:skip_build]
+          prepare_scan_config(scan_options)
+        else
+          build_for_testing(scan_options)
+        end
+      end
+
+      def self.prepare_scan_config(scan_options)
+        Scan.config ||= FastlaneCore::Configuration.create(
           Fastlane::Actions::ScanAction.available_options,
-          scan_options.merge(build_for_testing: true).reject { |k, _| options_to_remove.include?(k) }
+          scan_options.select { |k,v| %i[project workspace scheme].include?(k) }
         )
-        Fastlane::Actions::ScanAction.run(config)
+      end
+
+      def self.build_for_testing(scan_options)
+          valid_scan_keys = Fastlane::Actions::ScanAction.available_options.map(&:key)
+          scan_options = scan_options.select { |k,v| valid_scan_keys.include?(k) }
 
-        scan_options.merge!(
-          test_without_building: true,
-          derived_data_path: Scan.config[:derived_data_path]
-        ).delete(:build_for_testing)
+          Scan.config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::ScanAction.available_options,
+            scan_options.merge(build_for_testing: true)
+          )
+          values = Scan.config.values(ask: false)
+          values[:xcode_path] = File.expand_path("../..", FastlaneCore::Helper.xcode_path)
+          FastlaneCore::PrintTable.print_values(
+            config: values,
+            hide_keys: [:destination, :slack_url],
+            title: "Summary for scan #{Fastlane::VERSION}"
+          )
+          Scan::Runner.new.run
+          remove_build_report_files
+
+          Scan.config._values.delete(:build_for_testing)
+          scan_options[:derived_data_path] = Scan.config[:derived_data_path]
+      end
+
+      def self.remove_build_report_files
+        report_options = Scan::XCPrettyReporterOptionsGenerator.generate_from_scan_config
+        output_files = report_options.instance_variable_get(:@output_files)
+        output_directory = report_options.instance_variable_get(:@output_directory)
+
+        output_files.each do |output_file|
+          FileUtils.rm_f(File.join(output_directory, output_file))
+        end
       end
 
       #####################################################
@@ -156,6 +184,20 @@ module Fastlane
               UI.user_error!("Error: Batch counts must be greater than zero") unless count > 0
             end
           ),
+          FastlaneCore::ConfigItem.new(
+            key: :invocation_based_tests,
+            description: "Set to true If your test suit have invocation based tests like Kiwi",
+            type: Boolean,
+            is_string: false,
+            default_value: false,
+            optional: true,
+            conflicting_options: [:batch_count],
+            conflict_block: proc do |value|
+              UI.user_error!(
+                "Error: Can't use 'invocation_based_tests' and 'batch_count' options in one run, "\
+                "because the number of tests is unkown.")
+            end
+          ),
           FastlaneCore::ConfigItem.new(
             key: :quit_simulators,
             env_name: "FL_MULTI_SCAN_QUIT_SIMULATORS",
@@ -172,6 +214,28 @@ module Fastlane
             description: "Comma separated list of the output types (e.g. html, junit, json, json-compilation-database)",
             default_value: "html,junit"
           ),
+          FastlaneCore::ConfigItem.new(
+            key: :parallelize,
+            description: 'Run each batch of tests and/or each test target in parallel on its own Simulator',
+            optional: true,
+            is_string: false,
+            default_value: false,
+            conflicting_options: [:parallel_simulator_count],
+            conflict_block: proc do |value|
+              UI.user_error!("You can't use 'parallelize' and 'parallel_simulator_count' options in one run")
+            end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :parallel_simulator_fork_count,
+            description: 'Run  simulators each batch of tests and/or each test target in parallel on its own Simulator',
+            optional: true,
+            is_string: false,
+            default_value: false,
+            conflicting_options: [:parallelize],
+            conflict_block: proc do |value|
+              UI.user_error!("You can't use 'parallelize' and 'parallel_simulator_count' options in one run")
+            end
+          ),
           FastlaneCore::ConfigItem.new(
             key: :testrun_completed_block,
             description: 'A block invoked each time a test run completes',
@@ -317,6 +381,26 @@ module Fastlane
             output_files: 'report.json',
             fail_build: false
           )
+          ",
+          "
+          UI.important(
+            'example: ' \\
+            'use the :xctestrun parameter instead of the :project parameter to find, ' \\
+            'build, and test the iOS app.'
+          )
+          Dir.mktmpdir do |derived_data_path|
+            project_path = '/Users/lyndsey.ferguson/repo/fastlane-plugin-test_center/AtomicBoy/AtomicBoy.xcodeproj'
+            command = \"bundle exec fastlane scan --build_for_testing true --project '#{project_path}' --derived_data_path #{derived_data_path} --scheme AtomicBoy\"
+            `#{command}`  
+            xctestrun_file = Dir.glob(\"#{derived_data_path}/Build/Products/AtomicBoy*.xctestrun\").first
+            multi_scan(
+              scheme: 'AtomicBoy',
+              try_count: 3,
+              fail_build: false,
+              xctestrun: xctestrun_file,
+              test_without_building: true
+            )
+          end
           "
         ]
       end

data/lib/fastlane/plugin/test_center/actions/restart_core_simulator_service.rb

@@ -0,0 +1,38 @@
+module Fastlane
+  module Actions
+    class QuitCoreSimulatorServiceAction < Action
+      def self.run(params)
+        launchctl_list_count = 0
+        commands = []
+        while Actions.sh('launchctl list | grep com.apple.CoreSimulator.CoreSimulatorService || true', log: false) != ''
+          UI.crash!('Unable to quit com.apple.CoreSimulator.CoreSimulatorService after 10 tries') if (launchctl_list_count += 1) > 10
+          commands << Actions.sh('launchctl remove com.apple.CoreSimulator.CoreSimulatorService &> /dev/null || true', log: false)
+          UI.verbose('Waiting for com.apple.CoreSimulator.CoreSimulatorService to quit')
+          sleep(0.5)
+        end
+        commands
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "Force-quits the com.apple.CoreSimulator.CoreSimulatorService."
+      end
+
+      def self.details
+        "Sometimes the com.apple.CoreSimulator.CoreSimulatorService can hang. " \
+        "Use this action to force-quit it."
+      end
+
+      def self.authors
+        ["lyndsey-ferguson/@lyndseydf"]
+      end
+
+      def self.is_supported?(platform)
+        platform == :ios
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/test_center/actions/tests_from_xctestrun.rb

@@ -5,15 +5,15 @@ module Fastlane
     class TestsFromXctestrunAction < Action
       def self.run(params)
         UI.verbose("Getting tests from xctestrun file at '#{params[:xctestrun]}'")
-        return xctestrun_tests(params[:xctestrun])
+        return xctestrun_tests(params[:xctestrun], params[:invocation_based_tests])
       end
 
-      def self.xctestrun_tests(xctestrun_path)
+      def self.xctestrun_tests(xctestrun_path, invocation_based_tests)
         xctestrun = Plist.parse_xml(xctestrun_path)
         xctestrun_rootpath = File.dirname(xctestrun_path)
         tests = Hash.new([])
         xctestrun.each do |testable_name, xctestrun_config|
-          next if testable_name == '__xctestrun_metadata__'
+          next if ignoredTestables.include? testable_name
 
           xctest_path = xctest_bundle_path(xctestrun_rootpath, xctestrun_config)
           test_identifiers = XCTestList.tests(xctest_path)
@@ -23,7 +23,7 @@ module Fastlane
             UI.verbose("Removing skipped tests: #{skipped_tests.join("\n\t")}")
             test_identifiers.reject! { |test_identifier| skipped_tests.include?(test_identifier) }
           end
-          if test_identifiers.empty?
+          if test_identifiers.empty? && !invocation_based_tests
             UI.error("No tests found in '#{xctest_path}'!")
             UI.important("Is the Build Setting, `ENABLE_TESTABILITY` enabled for the test target #{testable_name}?")
           end
@@ -39,6 +39,10 @@ module Fastlane
         xctestrun_config['TestBundlePath'].sub('__TESTHOST__', xctest_host_path).sub('__TESTROOT__', xctestrun_rootpath)
       end
 
+      def self.ignoredTestables
+        return ['__xctestrun_metadata__']
+      end
+
       #####################################################
       # @!group Documentation
       #####################################################
@@ -56,6 +60,14 @@ module Fastlane
             verify_block: proc do |path|
               UI.user_error!("Error: cannot find the xctestrun file '#{path}'") unless File.exist?(path)
             end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :invocation_based_tests,
+            description: "Set to true If your test suit have invocation based tests like Kiwi",
+            type: Boolean,
+            is_string: false,
+            default_value: false,
+            optional: true
           )
         ]
       end

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager.rb

@@ -0,0 +1,5 @@
+require_relative 'multi_scan_manager/interstitial'
+require_relative 'multi_scan_manager/report_collator'
+require_relative 'multi_scan_manager/simulator_manager'
+require_relative 'multi_scan_manager/runner'
+require_relative 'xctestrun_info'

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/device_manager.rb

@@ -0,0 +1,30 @@
+module FastlaneCore
+  class DeviceManager
+    class Device
+      def clone
+        raise 'Can only clone iOS Simulators' unless self.is_simulator
+        Device.new(
+          name: self.name,
+          udid: `xcrun simctl clone #{self.udid} '#{self.name}'`.chomp,
+          os_type: self.os_type,
+          os_version: self.os_version,
+          state: self.state,
+          is_simulator: self.is_simulator
+        )
+      end
+
+      def rename(newname)
+        `xcrun simctl rename #{self.udid} '#{newname}'`
+        self.name = newname
+      end
+
+      def boot
+        `xcrun simctl boot #{self.udid}`
+      end
+
+      def shutdown
+        `xcrun simctl shutdown #{self.udid}` unless self.state == "Shutdown"
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/interstitial.rb

@@ -0,0 +1,143 @@
+module TestCenter
+  module Helper
+    module MultiScanManager
+      class Interstitial
+
+        attr_writer :output_directory
+        attr_writer :batch
+
+        def initialize(options)
+          @output_directory = options[:output_directory]
+          @testrun_completed_block = options[:testrun_completed_block]
+          @result_bundle = options[:result_bundle]
+          @scheme = options[:scheme]
+          @batch = options[:batch]
+          @reportnamer = options[:reportnamer]
+          @xcpretty_json_file_output = ENV['XCPRETTY_JSON_FILE_OUTPUT']
+          @parallelize = options[:parallelize]
+
+          before_all
+        end
+
+        # TODO: Should we be creating a new interstitial for each batch? yes.
+        # Should we clear out the result bundles before each batch? --> should
+        # it not be done before all batches? Same with env var for json resports.
+        def before_all
+          if @result_bundle
+            remove_preexisting_test_result_bundles
+          end
+          set_json_env_if_necessary
+          if @parallelize
+            @original_derived_data_path = ENV['SCAN_DERIVED_DATA_PATH']
+            FileUtils.mkdir_p(@output_directory)
+            ENV['SCAN_DERIVED_DATA_PATH'] = Dir.mktmpdir(nil, @output_directory)
+          end
+        end
+
+        def after_all
+          FastlaneCore::UI.message("resetting JSON ENV var to #{@xcpretty_json_file_output}")
+          ENV['XCPRETTY_JSON_FILE_OUTPUT'] = @xcpretty_json_file_output
+          if @parallelize
+            ENV['SCAN_DERIVED_DATA_PATH'] = @original_derived_data_path
+          end
+        end
+
+        def remove_preexisting_test_result_bundles
+          glob_pattern = "#{@output_directory}/.*\.test_result"
+          preexisting_test_result_bundles = Dir.glob(glob_pattern)
+          FileUtils.rm_rf(preexisting_test_result_bundles)
+        end
+
+        def move_test_result_bundle_for_next_run
+          if @result_bundle
+            built_test_result, moved_test_result = test_result_bundlepaths
+            FileUtils.mv(built_test_result, moved_test_result)
+          end
+        end
+
+        def test_result_bundlepaths
+          [
+            File.join(@output_directory, @scheme) + '.test_result',
+            File.join(@output_directory, @scheme) + "_#{@reportnamer.report_count}.test_result"
+          ]
+        end
+
+        def reset_simulators
+          destinations = Scan.config[:destination]
+          simulators = FastlaneCore::DeviceManager.simulators('iOS')
+          simulator_ids_to_reset = []
+          destinations.each do |destination|
+            destination.split(',').each do |destination_pair|
+              key, value = destination_pair.split('=')
+              if key == 'id'
+                simulator_ids_to_reset << value
+              end
+            end
+          end
+          simulators_to_reset = simulators.each.select { |simulator| simulator_ids_to_reset.include?(simulator.udid) }
+          simulators_to_reset.each do |simulator|
+            simulator.reset
+          end
+        end
+
+        def send_info_for_try(try_count)
+          puts "in send_info_for_try for #{@batch}"
+          return unless @testrun_completed_block
+
+          report_filepath = File.join(@output_directory, @reportnamer.junit_last_reportname)
+
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::TestsFromJunitAction.available_options,
+            {
+              junit: File.absolute_path(report_filepath)
+            }
+          )
+          junit_results = Fastlane::Actions::TestsFromJunitAction.run(config)
+          info = {
+            failed: junit_results[:failed],
+            passing: junit_results[:passing],
+            batch: @batch,
+            try_count: try_count,
+            report_filepath: report_filepath
+          }
+
+          if @reportnamer.includes_html?
+            html_report_filepath = File.join(@output_directory, @reportnamer.html_last_reportname)
+            info[:html_report_filepath] = html_report_filepath
+          end
+          if @reportnamer.includes_json?
+            json_report_filepath = File.join(@output_directory, @reportnamer.json_last_reportname)
+            info[:json_report_filepath] = json_report_filepath
+          end
+          if @result_bundle
+            test_result_suffix = '.test_result'
+            test_result_suffix.prepend("-#{@reportnamer.report_count}") unless @reportnamer.report_count.zero?
+            test_result_bundlepath = File.join(@output_directory, @scheme) + test_result_suffix
+            info[:test_result_bundlepath] = test_result_bundlepath
+          end
+          puts "interstitial about to call #{@testrun_completed_block} for batch #{@batch}"
+          @testrun_completed_block.call(info)
+        end
+
+        def set_json_env_if_necessary
+          if @reportnamer && @reportnamer.includes_json?
+            ENV['XCPRETTY_JSON_FILE_OUTPUT'] = File.join(
+              @output_directory,
+              @reportnamer.json_last_reportname
+            )
+          end
+        end
+
+        def finish_try(try_count)
+          send_info_for_try(try_count)
+          reset_simulators
+          ENV['SCAN_DERIVED_DATA_PATH'] = Dir.mktmpdir(nil, @output_directory) if @parallelize
+          move_test_result_bundle_for_next_run
+          set_json_env_if_necessary
+          @reportnamer && @reportnamer.increment
+        end
+      end
+    end
+  end
+end
+