fastlane-plugin-test_center 3.11.6 → 3.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2fd204227fedab854294a870686d5f6fee7376233ebb40c3430af8b4c8ec1d0
-  data.tar.gz: 9019c35ad25ac2a73bdda24f8697e8ee0377a21baf257379a7989bb7526915a6
+  metadata.gz: 5110e6b44d55945155e5769bf07fa9a1883521d125349f0d06211eea98014d14
+  data.tar.gz: ae0ffaa0f14c026a2ffdf2413f9b119142a130ba530742746715f2338cd058d2
 SHA512:
-  metadata.gz: 7f949e7b7dd7df3d576c9816b10f8ec46b478df909256675b093477c09ae4f49243d2d80962c379d6681ff71edf208fda1ba5992738439530fd89b97bbd100b8
-  data.tar.gz: 813c14d72c63448e8e25f5916029ee850d70185d7037c7634d4675c099fe274fb0a1ce9bbaf1fd287165ed5a351ff3cf25db4a12c5fd8c06212ae44e7c784446
+  metadata.gz: 00eeb25839214b31b2cdf248f0daa5978fba3a3e1e03c31b994bf24b618e6fd56588a2e2c41c4b6610346892c5a4460db0df854c78014b2aafdf3ee1651b37cb
+  data.tar.gz: 4928830dfca2a60011d1435ee38532bebfb4d9e7b3887cd3088f1a7dcc26fe503028229b4d73d64e04d166ac21ee25d8bd8641a0a8b30e4eb96400e160d932e5

data/README.md

@@ -40,6 +40,7 @@ _read the documentation on each action by clicking on the action name_
 | [`test_options_from_testplan`](docs/feature_details/test_options_from_testplan.md) | returns the tests and test code coverage configuration for a given testplan | ios, mac |
 | [`testplans_from_scheme`](docs/feature_details/testplans_from_scheme.md) | returns the testplans that an Xcode Scheme references | ios, mac |
 | [`tests_from_junit`](docs/feature_details/tests_from_junit.md) | returns the passing and failing tests in a Junit test report | ios, mac |
+| [`tests_from_xcresult`](docs/feature_details/tests_from_xcresult.md) | returns the passing and failing tests in a xcresult bundle | ios, mac |
 | [`tests_from_xctestrun`](docs/feature_details/tests_from_xctestrun.md) | returns a list of tests for each test target in a `xctestrun` file  | ios, mac |
 | [`collate_junit_reports`](docs/feature_details/collate_junit_reports.md) | combines multiple Junit test reports into one report | ios, mac |
 | [`collate_html_reports`](docs/feature_details/collate_html_reports.md) | combines multiple HTML test reports into one report | ios, mac |

data/lib/fastlane/plugin/test_center/actions/collate_test_result_bundles.rb

@@ -221,10 +221,12 @@ module Fastlane
             'collate the test_result bundles to a temporary bundle \"result.test_result\"'
           )
           bundles = Dir['../spec/fixtures/*.test_result'].map { |relpath| File.absolute_path(relpath) }
-          collate_test_result_bundles(
-            bundles: bundles,
-            collated_bundle: File.join('test_output', 'result.test_result')
-          )
+          Dir.mktmpdir('test_output') do |dir|
+            collate_test_result_bundles(
+              bundles: bundles,
+              collated_bundle: File.join(dir, 'result.test_result')
+            )
+          end
           "
         ]
       end

data/lib/fastlane/plugin/test_center/actions/collate_xcresults.rb

@@ -86,15 +86,19 @@ module Fastlane
       def self.example_code
         [
           "
+          require 'tmpdir'
+
           UI.important(
             'example: ' \\
             'collate the xcresult bundles to a temporary xcresult bundle \"result.xcresult\"'
           )
-          xcresults = Dir['../spec/fixtures/*.xcresult'].map { |relpath| File.absolute_path(relpath) }
-          collate_xcresults(
-            xcresults: xcresults,
-            collated_xcresult: File.join('test_output', 'result.xcresult')
-          )
+          xcresults = Dir['../spec/fixtures/AtomicBoyUITests-batch-{3,4}/result.xcresult'].map { |relpath| File.absolute_path(relpath) }
+          Dir.mktmpdir('test_output') do |dir|
+            collate_xcresults(
+              xcresults: xcresults,
+              collated_xcresult: File.join(dir, 'result.xcresult')
+            )
+          end
           "
         ]
       end

data/lib/fastlane/plugin/test_center/actions/multi_scan.rb

@@ -171,7 +171,10 @@ module Fastlane
         use_scanfile_to_override_settings(scan_options)
         turn_off_concurrent_workers(scan_options)
         UI.important("Turning off :skip_build as it doesn't do anything with multi_scan") if scan_options[:skip_build]
-        scan_options.reject! { |k,v| k == :skip_build }
+        if scan_options[:disable_xcpretty]
+          UI.important("Turning off :disable_xcpretty as xcpretty is needed to generate junit reports for retrying failed tests")
+        end
+        scan_options.reject! { |k,v| %i[skip_build disable_xcpretty].include?(k) }
         ScanHelper.remove_preexisting_simulator_logs(scan_options)
         if scan_options[:test_without_building]
           UI.verbose("Preparing Scan config options for multi_scan testing")
@@ -379,6 +382,15 @@ module Fastlane
                 "because the number of tests is unknown")
             end
           ),
+          FastlaneCore::ConfigItem.new(
+            key: :swift_test_prefix,
+            description: "The prefix used to find test methods. In standard XCTests, this is `test`. If you are using Quick with Swift, set this to `spec`",
+            default_value: "test",
+            optional: true,
+            verify_block: proc do |swift_test_prefix|
+              UI.user_error!("Error: swift_test_prefix must be non-nil and non-empty") if swift_test_prefix.nil? || swift_test_prefix.empty?
+            end
+          ),
           FastlaneCore::ConfigItem.new(
             key: :quit_simulators,
             env_name: "FL_MULTI_SCAN_QUIT_SIMULATORS",
@@ -478,6 +490,85 @@ module Fastlane
         ]
       end
 
+      def self.integration_tests
+        [
+          "
+          UI.header('Basic test')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            disable_xcpretty: true
+          )
+          ",
+          "
+          UI.header('Basic test with 1 specific test')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            only_testing: ['AtomicBoyUITests/AtomicBoyUITests/testExample']
+          )
+          ",
+          "
+          UI.header('Basic test with test target expansion')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            only_testing: ['AtomicBoyUITests', 'AtomicBoyTests']
+          )
+          ",
+          "
+          UI.header('Parallel test run')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            parallel_testrun_count: 2
+          )
+          ",
+          "
+          UI.header('Parallel test run with fewer tests than parallel test runs')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            parallel_testrun_count: 4,
+            only_testing: ['AtomicBoyUITests/AtomicBoyUITests/testExample']
+          )
+          ",
+          "
+          UI.header('Basic test with batches')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            fail_build: false,
+            try_count: 2,
+            batch_count: 2
+          )
+          ",
+          "
+          UI.header('Basic test with xcresult')
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            output_types: 'xcresult',
+            output_files: 'result.xcresult',
+            collate_reports: false,
+            fail_build: false,
+            try_count: 2,
+            batch_count: 2
+          )
+          "
+        ]
+      end
+
       def self.authors
         ["lyndsey-ferguson/@lyndseydf"]
       end

data/lib/fastlane/plugin/test_center/actions/tests_from_xcresult.rb

@@ -0,0 +1,103 @@
+require 'trainer'
+require 'shellwords'
+
+module Fastlane
+  module Actions
+    class TestsFromXcresultAction < Action
+      def self.run(params)
+        unless FastlaneCore::Helper.xcode_at_least?('11.0.0')
+          UI.error("Error: tests_from_xcresult requires at least Xcode 11.0")
+          return {}
+        end
+
+        xcresult_path = File.absolute_path(params[:xcresult])
+
+        # taken from the rubygem trainer, in the test_parser.rb module
+        result_bundle_object_raw = sh("xcrun xcresulttool get --path #{xcresult_path.shellescape} --format json", print_command: false, print_command_output: false)
+        result_bundle_object = JSON.parse(result_bundle_object_raw)
+
+        # Parses JSON into ActionsInvocationRecord to find a list of all ids for ActionTestPlanRunSummaries
+        actions_invocation_record = Trainer::XCResult::ActionsInvocationRecord.new(result_bundle_object)
+        test_refs = actions_invocation_record.actions.map do |action|
+          action.action_result.tests_ref
+        end.compact
+
+        ids = test_refs.map(&:id)
+        summaries = ids.map do |id|
+          raw = sh("xcrun xcresulttool get --format json --path #{xcresult_path.shellescape} --id #{id}", print_command: false, print_command_output: false)
+          json = JSON.parse(raw)
+          Trainer::XCResult::ActionTestPlanRunSummaries.new(json)
+        end
+        failures = actions_invocation_record.issues.test_failure_summaries || []
+        all_summaries = summaries.map(&:summaries).flatten
+        testable_summaries = all_summaries.map(&:testable_summaries).flatten
+        failed = []
+        passing = []
+        failure_details = {}
+        rows = testable_summaries.map do |testable_summary|
+          all_tests = testable_summary.all_tests.flatten
+          all_tests.each do |t|
+            if t.test_status == 'Success'
+              passing << "#{t.parent.name}/#{t.identifier}"
+            else
+              test_identifier = "#{t.parent.name}/#{t.identifier}"
+              failed << test_identifier
+              failure = t.find_failure(failures)
+              if failure
+                failure_details[test_identifier] = {
+                  message: failure.failure_message
+                }
+              end
+            end
+          end
+        end
+        {
+          failed: failed.uniq,
+          passing: passing.uniq,
+          failure_details: failure_details
+        }
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "☑️ Retrieves the failing and passing tests as reportedn an xcresult bundle"
+      end
+
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(
+            key: :xcresult,
+            env_name: "FL_TESTS_FROM_XCRESULT_XCRESULT_PATH",
+            description: "The path to the xcresult bundle to retrieve the tests from",
+            verify_block: proc do |path|
+              UI.user_error!("Error: cannot find the xcresult bundle at '#{path}'") unless Dir.exist?(path)
+              UI.user_error!("Error: cannot parse files that are not in the xcresult format") unless File.extname(path) == ".xcresult"
+            end
+          )
+        ]
+      end
+
+      def self.return_value
+        "A Hash with information about the test results:\r\n" \
+        "failed: an Array of the failed test identifiers\r\n" \
+        "passing: an Array of the passing test identifiers\r\n"
+      end
+
+      def self.authors
+        ["lyndsey-ferguson/lyndseydf"]
+      end
+
+      def self.category
+        :testing
+      end
+
+      def self.is_supported?(platform)
+        %i[ios mac].include?(platform)
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/test_center/actions/tests_from_xctestrun.rb

@@ -5,10 +5,10 @@ module Fastlane
     class TestsFromXctestrunAction < Action
       def self.run(params)
         UI.verbose("Getting tests from xctestrun file at '#{params[:xctestrun]}'")
-        return xctestrun_tests(params[:xctestrun], params[:invocation_based_tests])
+        return xctestrun_tests(params[:xctestrun], params[:invocation_based_tests], swift_test_prefix: params[:swift_test_prefix])
       end
 
-      def self.xctestrun_tests(xctestrun_path, invocation_based_tests)
+      def self.xctestrun_tests(xctestrun_path, invocation_based_tests, swift_test_prefix: "test")
         xctestrun = Plist.parse_xml(xctestrun_path)
         xctestrun_rootpath = File.dirname(xctestrun_path)
         xctestrun_version = xctestrun.fetch('__xctestrun_metadata__', Hash.new).fetch('FormatVersion', 1)
@@ -39,7 +39,7 @@ module Fastlane
             test_identifiers = xctestrun_config['OnlyTestIdentifiers']
             UI.verbose("Identifiers after adding onlytest tests: #{test_identifiers.join("\n\t")}")
           else
-            test_identifiers = XCTestList.tests(xctest_path)
+            test_identifiers = XCTestList.tests(xctest_path, swift_test_prefix: swift_test_prefix)
             UI.verbose("Found the following tests: #{test_identifiers.join("\n\t")}")
           end
           if xctestrun_config.key?('SkipTestIdentifiers')
@@ -52,6 +52,7 @@ module Fastlane
           if test_identifiers.empty? && !invocation_based_tests
             UI.error("No tests found in '#{xctest_path}'!")
             UI.important("Is the Build Setting, `ENABLE_TESTABILITY` enabled for the test target #{testable_name}?")
+            UI.message("If your Swift test method names use a prefix other than `test`, consider setting `:swift_test_prefix`.")
           end
           tests[testable_name] = test_identifiers.map do |test_identifier|
             "#{testable_name}/#{test_identifier}"
@@ -116,6 +117,15 @@ module Fastlane
             is_string: false,
             default_value: false,
             optional: true
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :swift_test_prefix,
+            description: "The prefix used to find test methods. In standard XCTests, this is `test`. If you are using Quick with Swift, set this to `spec`",
+            default_value: "test",
+            optional: true,
+            verify_block: proc do |swift_test_prefix|
+              UI.user_error!("Error: swift_test_prefix must be non-nil and non-empty") if swift_test_prefix.nil? || swift_test_prefix.empty?
+            end
           )
         ]
       end

data/lib/fastlane/plugin/test_center/helper/multi_scan_manager/runner.rb

@@ -66,9 +66,10 @@ module TestCenter
           @test_collector = TestCollector.new(@options)
           @options.reject! { |key| %i[testplan].include?(key) }
           @batch_count = @test_collector.test_batches.size
-          if @test_collector.test_batches.flatten.size < @options[:parallel_testrun_count].to_i
-            FastlaneCore::UI.important(":parallel_testrun_count greater than the number of tests (#{@test_collector.only_testing.size}). Reducing to that number.")
-            @options[:parallel_testrun_count] = @test_collector.only_testing.size
+          tests = @test_collector.test_batches.flatten
+          if tests.size < @options[:parallel_testrun_count].to_i
+            FastlaneCore::UI.important(":parallel_testrun_count greater than the number of tests (#{tests.size}). Reducing to that number.")
+            @options[:parallel_testrun_count] = tests.size
           end
         end
 

data/lib/fastlane/plugin/test_center/helper/test_collector.rb

@@ -26,6 +26,8 @@ module TestCenter
         if @batch_count == 1 && options[:parallel_testrun_count] > 1
           @batch_count = options[:parallel_testrun_count]
         end
+
+        @swift_test_prefix = options[:swift_test_prefix]
       end
 
       def only_testing_from_testplan(options)
@@ -89,30 +91,77 @@ module TestCenter
       end
 
       def xctestrun_known_tests
-        config = FastlaneCore::Configuration.create(
-          ::Fastlane::Actions::TestsFromXctestrunAction.available_options,
-          {
-            xctestrun: @xctestrun_path,
-            invocation_based_tests: @invocation_based_tests
-          }
-        )
-        ::Fastlane::Actions::TestsFromXctestrunAction.run(config)
+        unless @known_tests
+          config = FastlaneCore::Configuration.create(
+            ::Fastlane::Actions::TestsFromXctestrunAction.available_options,
+            {
+              xctestrun: @xctestrun_path,
+              invocation_based_tests: @invocation_based_tests,
+              swift_test_prefix: @swift_test_prefix
+            }
+          )
+          @known_tests = ::Fastlane::Actions::TestsFromXctestrunAction.run(config)
+        end
+        @known_tests
       end
 
-      def expand_testsuites_to_tests
+      def expand_short_testidentifiers_to_tests(testables_tests)
+        # Remember, testable_tests is of the format:
+        # {
+        #   'testable1' => [
+        #     'testsuite1/testcase1',
+        #     'testsuite1/testcase2',
+        #     'testsuite2/testcase1',
+        #     'testsuite2/testcase2',
+        #     ...
+        #     'testsuiteN/testcase1', ... 'testsuiteN/testcaseM'
+        #   ],
+        #   ...
+        #   'testableO' => [
+        #     'testsuite1/testcase1',
+        #     'testsuite1/testcase2',
+        #     'testsuite2/testcase1',
+        #     'testsuite2/testcase2',
+        #     ...
+        #     'testsuiteN/testcase1', ... 'testsuiteN/testcaseM'
+        #   ]
+        # }
         return if @invocation_based_tests
 
+        # iterate among all the test identifers for each testable
+        # A test identifier is seperated into components by '/'
+        # if a test identifier has only 2 separators, it probably is
+        # 'testable/testsuite' (but it could be 'testsuite/testcase' )
+        all_known_tests = nil
         known_tests = []
-        @testables_tests.each do |testable, tests|
+        testables_tests.each do |testable, tests|
           tests.each_with_index do |test, index|
-            if test.count('/') < 2
-              known_tests += xctestrun_known_tests[testable]
-              test_components = test.split('/')
-              testsuite = test_components.size == 1 ? test_components[0] : test_components[1]
-              @testables_tests[testable][index] = known_tests.select { |known_test| known_test.include?(testsuite) }
+            test_components = test.split('/')
+            is_full_test_identifier = (test_components.size == 3)
+            next if is_full_test_identifier
+
+            all_known_tests ||= xctestrun_known_tests.clone
+
+            testsuite = ''
+            if test_components.size == 1
+              if test_components[0] == testable
+                # The following || [] is just in case the user provided multiple
+                # test targets or there are no actual tests found.
+                testables_tests[testable][index] = all_known_tests[testable] || []
+                all_known_tests.delete(testable)
+                next
+              end
+
+              testsuite = test_components[0]
+            else
+              testsuite = test_components[1]
+            end
+
+            testables_tests[testable][index], all_known_tests[testable] = all_known_tests[testable].partition do |known_test|
+              known_test.split('/')[1] == testsuite
             end
           end
-          @testables_tests[testable].flatten!
+          testables_tests[testable].flatten!
         end
       end
 
@@ -120,7 +169,7 @@ module TestCenter
         unless @testables_tests
           if @only_testing
             @testables_tests = only_testing_to_testables_tests
-            expand_testsuites_to_tests
+            expand_short_testidentifiers_to_tests(@testables_tests)
           else
             @testables_tests = xctestrun_known_tests
             if @skip_testing

data/lib/fastlane/plugin/test_center/version.rb

@@ -1,5 +1,5 @@
 module Fastlane
   module TestCenter
-    VERSION = "3.11.6"
+    VERSION = "3.14.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fastlane-plugin-test_center
 version: !ruby/object:Gem::Version
-  version: 3.11.6
+  version: 3.14.2
 platform: ruby
 authors:
 - Lyndsey Ferguson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-25 00:00:00.000000000 Z
+date: 2020-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: trainer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: xcodeproj
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +72,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.8
+        version: 1.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.8
+        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -270,6 +284,7 @@ files:
 - lib/fastlane/plugin/test_center/actions/test_options_from_testplan.rb
 - lib/fastlane/plugin/test_center/actions/testplans_from_scheme.rb
 - lib/fastlane/plugin/test_center/actions/tests_from_junit.rb
+- lib/fastlane/plugin/test_center/actions/tests_from_xcresult.rb
 - lib/fastlane/plugin/test_center/actions/tests_from_xctestrun.rb
 - lib/fastlane/plugin/test_center/helper/fastlane_core/device_manager/simulator_extensions.rb
 - lib/fastlane/plugin/test_center/helper/html_test_report.rb
@@ -295,7 +310,7 @@ homepage: https://github.com/lyndsey-ferguson/fastlane-plugin-test_center
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -310,8 +325,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Makes testing your iOS app easier
 test_files: []