fastlane-plugin-test_center 3.14.3 → 3.14.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e081e2a0d9d6bcb59c38f24072eb0059edef4457f03fed6e2318930260e9f5a4
-  data.tar.gz: 681403d6200f5d943e28f9c3fa6b90a6f450d7a9252c476eb7cf77c7e0c6101d
+  metadata.gz: 4364b9e3e49e3aa1a3bbd1e26feb07e449de72900bcd1ec3341ff8d1b0ca62ad
+  data.tar.gz: f983745b33c34838839d546aff4c4e407673c8241e72ab14b695d288792bdac4
 SHA512:
-  metadata.gz: 98e979a4fbafe31e6dc33bd1a3a94a1e1da316ce8226c3688e9d223461a33c5d75c9291dc837e256c227915959547f8964df38c524e40b45742408235ee94713
-  data.tar.gz: 3049728d26e108dea44480f7445e41be419f4189e3bb2b63b8cef7871a9d7bf58b5edcee01351dc40496209f6805706bfcb0b6cf08b9fee6c43bba1decbfecda
+  metadata.gz: 1bb52a9bcc82781df58b2ab2b6dd33c1831da0b345b9b51842959b8a372e5341b773cf639e3cb884bccc426336dad01c8ff1e7fc418fad0baefe201e12c46bab
+  data.tar.gz: 8d6e6a9321e13aada993f1099f45b7317f4d4c30e8bbdfa55a35a49a1a0dd3aa6c7e06c7dd244132e37d4c149ac0a6b9f94d9198d3f0bd8ee912853fa44f773d

data/README.md

@@ -99,7 +99,7 @@ _fastlane_ is the easiest way to automate beta deployments and releases for your
 ## Supporters
 
 ![Владислав Давыдов](https://avatars1.githubusercontent.com/u/47553334?s=44&u=4691860dba898943b947180b3d28bb85851b0d7c&v=4)
-[vdavydovHH](https://github.com/vdavydovHH)
+[vdavydovHH](https://github.com/vdavydovHH)  ![amelendezSGY](https://avatars1.githubusercontent.com/u/28657466?s=44&v=4)[amelendezSGY](https://github.com/amelendezSGY)
 
 ## License
 

data/lib/fastlane/plugin/test_center/actions/multi_scan.rb

@@ -361,6 +361,14 @@ module Fastlane
               UI.user_error!("Error: Batch counts must be greater than zero") unless count > 0
             end
           ),
+          FastlaneCore::ConfigItem.new(
+            key: :batches,
+            env_name: "FL_MULTI_SCAN_BATCHES",
+            description: "The explicit batches (an Array of Arrays of test identifiers) to run either serially, or each batch on a simulator in parallel if :parallel_testrun_count is given",
+            type: Array,
+            optional: true,
+            conflicting_options: [:batch_count]
+          ),
           FastlaneCore::ConfigItem.new(
             key: :retry_test_runner_failures,
             description: "Set to true If you want to treat build failures during testing, like 'Test runner exited before starting test execution', as 'all tests failed'",
@@ -375,7 +383,7 @@ module Fastlane
             is_string: false,
             default_value: false,
             optional: true,
-            conflicting_options: [:batch_count],
+            conflicting_options: %i[batch_count batches],
             conflict_block: proc do |value|
               UI.user_error!(
                 "Error: Can't use 'invocation_based_tests' and 'batch_count' options in one run, "\
@@ -433,7 +441,7 @@ module Fastlane
           ),
           FastlaneCore::ConfigItem.new(
             key: :testrun_completed_block,
-            description: 'A block invoked each time a test run completes. When combined with :parallel_testrun_count, will be called separately in each child process',
+            description: 'A block invoked each time a test run completes. When combined with :parallel_testrun_count, will be called separately in each child process. Return a Hash with :continue set to false to stop retrying tests, or :only_testing to change which tests will be run in the next try',
             optional: true,
             is_string: false,
             default_value: nil,
@@ -461,6 +469,10 @@ module Fastlane
 
             # UI.abort_with_message!('You could conditionally abort')
             UI.message(\"\\\u1F60A everything is fine, let's continue try \#{try_attempt + 1} for batch \#{batch}\")
+            {
+              continue: true,
+              only_testing: ['AtomicBoyUITests/AtomicBoyUITests/testExample17']
+            }
           end
 
           multi_scan(
@@ -486,6 +498,120 @@ module Fastlane
             output_files: 'report.json',
             fail_build: false
           )
+          ",
+          "
+          UI.header('batches feature')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            try_count: 3,
+            fail_build: false,
+            batches: [
+              [
+                'AtomicBoyUITests/AtomicBoyUITests/testExample5',
+                'AtomicBoyUITests/AtomicBoyUITests/testExample10',
+                'AtomicBoyUITests/AtomicBoyUITests/testExample15'
+              ],
+              [
+                'AtomicBoyUITests/AtomicBoyUITests/testExample6',
+                'AtomicBoyUITests/AtomicBoyUITests/testExample12',
+                'AtomicBoyUITests/AtomicBoyUITests/testExample18'
+              ]
+            ]
+          )
+          "
+        ]
+      end
+
+      def self.integration_tests
+        [
+          "
+          UI.header('Basic test')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            disable_xcpretty: true
+          )
+          ",
+          "
+          UI.header('Basic test with 1 specific test')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            only_testing: ['AtomicBoyUITests/AtomicBoyUITests/testExample']
+          )
+          ",
+          "
+          UI.header('Basic test with test target expansion')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            only_testing: ['AtomicBoyUITests', 'AtomicBoyTests']
+          )
+          ",
+          "
+          UI.header('Parallel test run')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            parallel_testrun_count: 2
+          )
+          ",
+          "
+          UI.header('Parallel test run with fewer tests than parallel test runs')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            parallel_testrun_count: 4,
+            only_testing: ['AtomicBoyUITests/AtomicBoyUITests/testExample']
+          )
+          ",
+          "
+          UI.header('Basic test with batch count')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            batch_count: 2
+          )
+          ",
+          "
+          UI.header('Basic test with batches')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            batches: [
+              ['AtomicBoyUITests/AtomicBoyUITests/testExample', 'AtomicBoyUITests/AtomicBoyUITests/testExample2'],
+              ['AtomicBoyUITests/AtomicBoyUITests/testExample3', 'AtomicBoyUITests/AtomicBoyUITests/testExample4']
+            ],
+            parallel_testrun_count: 2
+          )
+          ",
+          "
+          UI.header('Basic test with xcresult')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            output_types: 'xcresult',
+            output_files: 'result.xcresult',
+            collate_reports: false,
+            fail_build: false,
+            try_count: 2,
+            batch_count: 2
+          )
           "
         ]
       end

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/retrying_scan_helper.rb

@@ -16,6 +16,7 @@ module TestCenter
             @options[:output_files],
             @options[:custom_report_file_name]
           )
+          @callback_overrides_only_testing = false
         end
 
         def before_testrun
@@ -189,7 +190,26 @@ module TestCenter
           update_json_failure_details(info)
           update_test_result_bundle_details(info)
 
-          @options[:testrun_completed_block].call(info)
+          @callback_overrides_only_testing = false
+          callback_result = @options[:testrun_completed_block].call(info)
+          if callback_result.kind_of?(Hash)
+            should_continue = callback_result.fetch(:continue, true)
+            if !should_continue
+              discontinue_message = 'Following testrun_completed_block\'s request to discontinue testing'
+              discontinue_message << " for batch ##{@options[:batch]}" unless @options[:batch].nil?
+              FastlaneCore::UI.verbose(discontinue_message)
+              @testrun_count = options[:try_count]
+            end
+            overridden_only_testing = callback_result.fetch(:only_testing, nil)
+            if overridden_only_testing && should_continue
+              override_only_testing_message = 'Following testrun_completed_block\'s request to change :only_testing to '
+              override_only_testing_message << overridden_only_testing.to_s
+              override_only_testing_message << " for batch ##{@options[:batch]}" unless @options[:batch].nil?
+              FastlaneCore::UI.verbose(override_only_testing_message)
+              @callback_overrides_only_testing = true
+              @options[:only_testing] = overridden_only_testing
+            end
+          end
         end
 
         def failure_details(additional_info)
@@ -242,6 +262,8 @@ module TestCenter
         end
 
         def update_only_testing
+          return if @callback_overrides_only_testing
+
           report_filepath = File.join(output_directory, @reportnamer.junit_last_reportname)
           config = FastlaneCore::Configuration.create(
             Fastlane::Actions::TestsFromJunitAction.available_options,

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/runner.rb

@@ -65,8 +65,8 @@ module TestCenter
 
           @test_collector = TestCollector.new(@options)
           @options.reject! { |key| %i[testplan].include?(key) }
-          @batch_count = @test_collector.test_batches.size
-          tests = @test_collector.test_batches.flatten
+          @batch_count = @test_collector.batches.size
+          tests = @test_collector.batches.flatten
           if tests.size < @options[:parallel_testrun_count].to_i
             FastlaneCore::UI.important(":parallel_testrun_count greater than the number of tests (#{tests.size}). Reducing to that number.")
             @options[:parallel_testrun_count] = tests.size
@@ -196,7 +196,7 @@ module TestCenter
           pool = TestBatchWorkerPool.new(pool_options)
           pool.setup_workers
 
-          remaining_test_batches = @test_collector.test_batches.clone
+          remaining_test_batches = @test_collector.batches.clone
           remaining_test_batches.each_with_index do |test_batch, current_batch_index|
             worker = pool.wait_for_worker
             FastlaneCore::UI.message("Starting test run #{current_batch_index + 1}")
@@ -209,7 +209,7 @@ module TestCenter
         end
 
         def scan_options_for_worker(test_batch, batch_index)
-          if @test_collector.test_batches.size > 1
+          if @test_collector.batches.size > 1
             # If there are more than 1 batch, then we want each batch result
             # sent to a "batch index" output folder to be collated later
             # into the requested output_folder.

data/lib/fastlane/plugin/test_center/helper/test_collector.rb

@@ -3,34 +3,154 @@ module TestCenter
     require 'fastlane_core/ui/ui.rb'
     require 'fastlane/actions/scan'
     require 'plist'
+    require 'set'
 
     class TestCollector
       attr_reader :xctestrun_path
-      attr_reader :only_testing
+      attr_reader :batches
+      attr_reader :testables
 
       def initialize(options)
-        unless options[:xctestrun] || options[:derived_data_path]
-          options[:derived_data_path] = default_derived_data_path(options)
+        @invocation_based_tests = options[:invocation_based_tests]
+        @swift_test_prefix = options[:swift_test_prefix]
+
+        @xctestrun_path = self.class.xctestrun_filepath(options)
+        initialize_batches(options)
+      end
+
+      private
+
+      def initialize_batches(options)
+        if options[:batches]
+          expand_given_batches_to_full_test_identifiers(options)
+        else
+          derive_batches_from_tests(options)
         end
-        @xctestrun_path = options[:xctestrun] || derived_testrun_path(options[:derived_data_path], options[:scheme])
-        unless @xctestrun_path && File.exist?(@xctestrun_path)
-          FastlaneCore::UI.user_error!("Error: cannot find xctestrun file '#{@xctestrun_path}'")
+      end
+
+      def expand_given_batches_to_full_test_identifiers(options)
+        @batches = options[:batches]
+        testables = Set.new
+        @batches.each do |batch|
+          expand_test_identifiers(batch)
+          batch.each { |t| testables << t.split('/')[0] }
         end
-        @only_testing = options[:only_testing] || only_testing_from_testplan(options)
-        if @only_testing.kind_of?(String)
-          @only_testing = @only_testing.split(',')
+        @testables = testables.to_a
+      end
+
+      def derive_batch_count(options)
+        batch_count = options.fetch(:batch_count, 1)
+        if batch_count == 1 && options.fetch(:parallel_testrun_count, 0) > 1
+          # if the batch count is 1, and the users wants parallel runs
+          # we *must* set the batch count to the same number of parallel
+          # runs or else the desired reports will not be written
+          batch_count = options[:parallel_testrun_count]
         end
-        @skip_testing = options[:skip_testing]
-        @invocation_based_tests = options[:invocation_based_tests]
-        @batch_count = options[:batch_count]
-        if @batch_count == 1 && options[:parallel_testrun_count] > 1
-          @batch_count = options[:parallel_testrun_count]
+        batch_count
+      end
+
+      def derive_only_testing(options)
+        only_testing = options[:only_testing] || self.class.only_testing_from_testplan(options)
+        if only_testing && only_testing.kind_of?(String)
+          only_testing = only_testing.split(',').map(&:strip)
         end
+        only_testing
+      end
 
-        @swift_test_prefix = options[:swift_test_prefix]
+      def testable_tests_hash_from_options(options)
+        testable_tests_hash = Hash.new { |h, k| h[k] = [] }
+        only_testing = derive_only_testing(options)
+        if only_testing
+          expand_test_identifiers(only_testing)
+          only_testing.each do |test_identifier|
+            testable = test_identifier.split('/')[0]
+            testable_tests_hash[testable] << test_identifier
+          end
+        else
+          testable_tests_hash = xctestrun_known_tests.clone
+          if options[:skip_testing]
+            expand_test_identifiers(options[:skip_testing])
+            testable_tests_hash.each do |testable, test_identifiers|
+              test_identifiers.replace(test_identifiers - options[:skip_testing])
+              testable_tests_hash.delete(testable) if test_identifiers.empty?
+            end
+          end
+        end
+        testable_tests_hash
+      end
+
+      def derive_batches_from_tests(options)
+        @batches = []
+        testable_tests_hash = testable_tests_hash_from_options(options)
+        @testables = testable_tests_hash.keys
+        batch_count = derive_batch_count(options)
+        testable_tests_hash.each do |testable, test_identifiers|
+          next if test_identifiers.empty?
+
+          if batch_count > 1
+            slice_count = [(test_identifiers.length / batch_count.to_f).ceil, 1].max
+            test_identifiers.each_slice(slice_count).to_a.each do |batch|
+              @batches << batch
+            end
+          else
+            @batches << test_identifiers
+          end
+        end
+      end
+
+      def expand_test_identifiers(test_identifiers)
+        all_known_tests = nil
+        test_identifiers.each_with_index do |test_identifier, index|
+          test_components = test_identifier.split('/')
+          is_full_test_identifier = (test_components.size == 3)
+          next if is_full_test_identifier
+
+          all_known_tests ||= xctestrun_known_tests.clone
+
+          testsuite = ''
+          testable = test_components[0]
+          expanded_test_identifiers = []
+          if test_components.size == 1
+            # this is a testable, also known as a test target. Let's expand it out
+            # to all of its tests. Note: a test target can have many testSuites, each
+            # with their own testCases.
+            if all_known_tests[testable].to_a.empty?
+              FastlaneCore::UI.verbose("Unable to expand #{testable} to constituent tests")
+              expand_test_identifiers = [testable]
+            else
+              expand_test_identifiers = all_known_tests[testable]
+            end
+          else
+            # this is a testable and a test suite, let's expand it out to all of
+            # its testCases. Note: if the user put the same test identifier in more than
+            # one place in this array, this could lead to multiple repititions of the same
+            # set of test identifiers
+            testsuite = test_components[1]
+            expanded_test_identifiers = all_known_tests[testable].select do |known_test|
+              known_test.split('/')[1] == testsuite
+            end
+          end
+          test_identifiers.delete_at(index)
+          test_identifiers.insert(index, *expand_test_identifiers)
+        end
+      end
+
+      def xctestrun_known_tests
+        unless @known_tests
+          config = FastlaneCore::Configuration.create(
+            ::Fastlane::Actions::TestsFromXctestrunAction.available_options,
+            {
+              xctestrun: @xctestrun_path,
+              invocation_based_tests: @invocation_based_tests,
+              swift_test_prefix: @swift_test_prefix
+            }
+          )
+          @known_tests = ::Fastlane::Actions::TestsFromXctestrunAction.run(config)
+        end
+        @known_tests
       end
 
-      def only_testing_from_testplan(options)
+      def self.only_testing_from_testplan(options)
         return unless options[:testplan] && options[:scheme]
 
         config = FastlaneCore::Configuration.create(
@@ -60,153 +180,26 @@ module TestCenter
         return test_options[:only_testing]
       end
 
-      def default_derived_data_path(options)
+      def self.default_derived_data_path
         project_derived_data_path = Scan.project.build_settings(key: "BUILT_PRODUCTS_DIR")
         File.expand_path("../../..", project_derived_data_path)
       end
 
-      def derived_testrun_path(derived_data_path, scheme)
+      def self.derived_testrun_path(derived_data_path)
         xctestrun_files = Dir.glob("#{derived_data_path}/Build/Products/*.xctestrun")
         xctestrun_files.sort { |f1, f2| File.mtime(f1) <=> File.mtime(f2) }.last
       end
 
-      def testables
-        unless @testables
-          if @only_testing
-            @testables ||= only_testing_to_testables_tests.keys
-          else
-            @testables = xctestrun_known_tests.keys
-          end
-        end
-        @testables
-      end
-
-      def only_testing_to_testables_tests
-        tests = Hash.new { |h, k| h[k] = [] }
-        @only_testing.sort.each do |test_identifier|
-          testable = test_identifier.split('/', 2)[0]
-          tests[testable] << test_identifier
-        end
-        tests
-      end
-
-      def xctestrun_known_tests
-        unless @known_tests
-          config = FastlaneCore::Configuration.create(
-            ::Fastlane::Actions::TestsFromXctestrunAction.available_options,
-            {
-              xctestrun: @xctestrun_path,
-              invocation_based_tests: @invocation_based_tests,
-              swift_test_prefix: @swift_test_prefix
-            }
-          )
-          @known_tests = ::Fastlane::Actions::TestsFromXctestrunAction.run(config)
-        end
-        @known_tests
-      end
-
-      def expand_short_testidentifiers_to_tests(testables_tests)
-        # Remember, testable_tests is of the format:
-        # {
-        #   'testable1' => [
-        #     'testsuite1/testcase1',
-        #     'testsuite1/testcase2',
-        #     'testsuite2/testcase1',
-        #     'testsuite2/testcase2',
-        #     ...
-        #     'testsuiteN/testcase1', ... 'testsuiteN/testcaseM'
-        #   ],
-        #   ...
-        #   'testableO' => [
-        #     'testsuite1/testcase1',
-        #     'testsuite1/testcase2',
-        #     'testsuite2/testcase1',
-        #     'testsuite2/testcase2',
-        #     ...
-        #     'testsuiteN/testcase1', ... 'testsuiteN/testcaseM'
-        #   ]
-        # }
-        return if @invocation_based_tests
-
-        # iterate among all the test identifers for each testable
-        # A test identifier is seperated into components by '/'
-        # if a test identifier has only 2 separators, it probably is
-        # 'testable/testsuite' (but it could be 'testsuite/testcase' )
-        all_known_tests = nil
-        known_tests = []
-        testables_tests.each do |testable, tests|
-          tests.each_with_index do |test, index|
-            test_components = test.split('/')
-            is_full_test_identifier = (test_components.size == 3)
-            next if is_full_test_identifier
-
-            all_known_tests ||= xctestrun_known_tests.clone
-
-            testsuite = ''
-            if test_components.size == 1
-              if test_components[0] == testable
-                # The following || [] is just in case the user provided multiple
-                # test targets or there are no actual tests found.
-                testables_tests[testable][index] = all_known_tests[testable] || []
-                all_known_tests.delete(testable)
-                next
-              end
-
-              testsuite = test_components[0]
-            else
-              testsuite = test_components[1]
-            end
-
-            testables_tests[testable][index], all_known_tests[testable] = all_known_tests[testable].partition do |known_test|
-              known_test.split('/')[1] == testsuite
-            end
-          end
-          testables_tests[testable].flatten!
-        end
-      end
-
-      def testables_tests
-        unless @testables_tests
-          if @only_testing
-            @testables_tests = only_testing_to_testables_tests
-            expand_short_testidentifiers_to_tests(@testables_tests)
-          else
-            @testables_tests = xctestrun_known_tests
-            if @skip_testing
-              skipped_testable_tests = Hash.new { |h, k| h[k] = [] }
-              @skip_testing.sort.each do |skipped_test_identifier|
-                testable = skipped_test_identifier.split('/', 2)[0]
-                skipped_testable_tests[testable] << skipped_test_identifier
-              end
-              @testables_tests.each_key do |testable|
-                @testables_tests[testable] -= skipped_testable_tests[testable]
-              end
-            end
-          end
+      def self.xctestrun_filepath(options)
+        unless options[:xctestrun] || options[:derived_data_path]
+          options[:derived_data_path] = default_derived_data_path
         end
+        path = (options[:xctestrun] || derived_testrun_path(options[:derived_data_path]))
 
-        @testables_tests
-      end
-
-      def test_batches
-        if @batches.nil?
-          @batches = []
-          testables.each do |testable|
-            testable_tests = testables_tests[testable]
-            next if testable_tests.empty?
-
-            if @batch_count > 1
-              slice_count = [(testable_tests.length / @batch_count.to_f).ceil, 1].max
-              testable_tests.each_slice(slice_count).to_a.each do |tests_batch|
-                @batches << tests_batch
-              end
-            else
-              @batches << testable_tests
-            end
-          end
+        unless path && File.exist?(path)
+          FastlaneCore::UI.user_error!("Error: cannot find xctestrun file '#{path}'")
         end
-
-        @batches
+        path
       end
     end
   end

data/lib/fastlane/plugin/test_center/version.rb

@@ -1,5 +1,5 @@
 module Fastlane
   module TestCenter
-    VERSION = "3.14.3"
+    VERSION = "3.14.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fastlane-plugin-test_center
 version: !ruby/object:Gem::Version
-  version: 3.14.3
+  version: 3.14.4
 platform: ruby
 authors:
 - Lyndsey Ferguson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-15 00:00:00.000000000 Z
+date: 2020-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json