fastlane-plugin-test_center 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 893544a3b0cd2e53515d862eb0fda5c3e55f6894
-  data.tar.gz: 4fb54967fc84179a6de35c8314d7018396b5170a
+  metadata.gz: '083be0187ece34a8f7019e4992c1fa2330b4732d'
+  data.tar.gz: 364b206944566949934477d518853537f27cdabb
 SHA512:
-  metadata.gz: 888c640ec758d5916be8414d1f7a5940c51c28736bdd172e3ceda86fbbf6b50ea74071c07397b4bae3ca2e5ddc9f18f853f3fd11b074eb9b26db894e1f9b67e3
-  data.tar.gz: 3b65a9bb1175eaedd4ba0fb937ea62538a037ff785f3837cd6f1061f22a52d2ac06633691ad85021fe43f6b30e7586ffce4ac14cb662fb489032c4923fe24163
+  metadata.gz: 64532eb3e5dfe6afdf6b49fe27555e58d052fc60cf3d57363becf437704e0519a696889032f0106fabafabaf5ab808f0f713b059af75a38f0e9aecc44e8f2cde
+  data.tar.gz: 301e3e7946ab86b32aa310fd73b01cb2b222c00b99d1a255575964c1bed55ca864bf66b55ff2d50690776ff399d3c57335ac3d6737de2262c50a720a565af0e5

data/README.md

@@ -69,6 +69,7 @@ This fastlane plugin includes the following actions:
 - [`tests_from_xctestrun`](#tests_from_xctestrun): from an xctestrun file, returns a list of tests for each of its test targets.
 - [`collate_junit_reports`](#collate_junit_reports): combines multiple junit test reports into one report.
 - [`collate_html_reports`](#collate_html_reports): combines multiple html test reports into one report.
+- [`collate_json_reports`](#collate_json_reports): combines multiple json test reports into one report.
 
 ### multi_scan 🎉
 
@@ -202,7 +203,8 @@ multi_scan(
   skip_build: true,
   clean: true,
   try_count: 3,
-  result_bundle: true
+  result_bundle: true,
+  fail_build: false
 )
 
 ```
@@ -235,7 +237,25 @@ multi_scan(
   workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
   scheme: 'AtomicBoy',
   try_count: 3,
-  only_testing: ['AtomicBoyTests']
+  only_testing: ['AtomicBoyTests'],
+  fail_build: false
+)
+
+```
+
+```ruby
+
+UI.important(
+  'example: ' \
+  'multi_scan also works with json.'
+)
+multi_scan(
+  workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+  scheme: 'AtomicBoy',
+  try_count: 3,
+  output_types: 'json',
+  output_files: 'report.json',
+  fail_build: false
 )
 
 ```
@@ -328,6 +348,23 @@ suppress_tests(
   scheme: 'Professor'
 )
 
+```
+
+```ruby
+
+UI.important(
+  'example: ' \
+  'suppress some tests in one Scheme from a workspace'
+)
+suppress_tests(
+  workspace: 'AtomicBoy/AtomicBoy.xcworkspace',
+  tests: [
+    'AtomicBoyUITests/HappyNapperTests/testBeepingNonExistentFriendDisplaysError',
+    'AtomicBoyUITests/GrumpyWorkerTests'
+  ],
+  scheme: 'Professor'
+)
+
 ```
 <!-- suppress_tests examples: end -->
 </details>
@@ -364,6 +401,20 @@ UI.message(
   "Suppressed tests for project: #{suppressed_tests(xcodeproj: 'AtomicBoy/AtomicBoy.xcodeproj')}"
 )
 
+```
+
+```ruby
+
+UI.important(
+  'example: ' \
+  'get the tests that are suppressed in all Schemes in a workspace'
+)
+tests = suppressed_tests(
+  workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+  scheme: 'Professor'
+)
+UI.message("tests: #{tests}")
+
 ```
 <!-- suppressed_tests examples: end -->
 </details>
@@ -438,10 +489,11 @@ Do you have multiple junit test reports coming in from different sources and nee
 
 UI.important(
   'example: ' \
-  'collate the xml reports to a temporary file 'result.xml''
+  'collate the xml reports to a temporary file "result.xml"'
 )
+reports = Dir['../spec/fixtures/*.xml'].map { |relpath| File.absolute_path(relpath) }
 collate_junit_reports(
-  reports: Dir['./spec/fixtures/*.xml'],
+  reports: reports,
   collated_report: File.join(Dir.mktmpdir, 'result.xml')
 )
 
@@ -461,10 +513,11 @@ Do you have multiple html test reports coming in from different sources and need
 
 UI.important(
   'example: ' \
-  'collate the html reports to a temporary file 'result.html''
+  'collate the html reports to a temporary file "result.html"'
 )
+reports = Dir['../spec/fixtures/*.html'].map { |relpath| File.absolute_path(relpath) }
 collate_html_reports(
-  reports: Dir['./spec/fixtures/*.html'],
+  reports: reports,
   collated_report: File.join(Dir.mktmpdir, 'result.html')
 )
 
@@ -472,6 +525,30 @@ collate_html_reports(
 <!-- collate_html_reports examples: end -->
 </details>
 
+### collate_json_reports
+
+Do you have multiple json test reports coming in from different sources and need it combined? Use this action to collate all the tests performed for a given test target into one report file.
+
+<details>
+    <summary>Example Code:</summary>
+<!-- collate_json_reports examples: begin -->
+
+```ruby
+
+UI.important(
+  'example: ' \
+  'collate the json reports to a temporary file "result.json"'
+)
+reports = Dir['../spec/fixtures/report*.json'].map { |relpath| File.absolute_path(relpath) }
+collate_json_reports(
+  reports: reports,
+  collated_report: File.join(Dir.mktmpdir, 'result.json')
+)
+
+```
+<!-- collate_json_reports examples: end -->
+</details>
+
 ## Run tests for this plugin
 
 To run both the tests, and code style validation, run

data/lib/fastlane/plugin/test_center/actions/collate_json_reports.rb

@@ -0,0 +1,168 @@
+module Fastlane
+  module Actions
+    class CollateJsonReportsAction < Action
+      require 'json'
+
+      def self.run(params)
+        report_filepaths = params[:reports]
+        if report_filepaths.size == 1
+          FileUtils.cp(report_filepaths[0], params[:collated_report])
+        else
+          base_report_json = JSON.parse(File.read(report_filepaths.shift))
+          report_filepaths.each do |report_file|
+            report_json = JSON.parse(File.read(report_file))
+            update_previous_test_failures(base_report_json)
+            merge_reports(base_report_json, report_json)
+          end
+          File.open(params[:collated_report], 'w') do |f|
+            f.write(JSON.pretty_generate(base_report_json))
+          end
+        end
+      end
+
+      def self.merge_reports(base_report, other_report)
+        base_report.each_key do |key|
+          unless %w(previous_tests_failures tests_failures tests_summary_messages).include?(key)
+            base_report[key].concat(other_report[key])
+          end
+        end
+        base_report["tests_failures"] = other_report["tests_failures"]
+        update_failed_tests_count(base_report, other_report)
+        update_time(base_report, other_report)
+        update_unexpected_failures(base_report, other_report)
+      end
+
+      def self.update_time(base_report, other_report)
+        base_test_time, base_total_time = times_from_summary(base_report['tests_summary_messages'][0])
+        other_test_time, other_total_time = times_from_summary(other_report['tests_summary_messages'][0])
+        time_regex = '(?:\d|\.)+'
+
+        test_time_sum = (base_test_time + other_test_time).round(3)
+        total_time_sum = (base_total_time + other_total_time).round(3)
+        base_report['tests_summary_messages'][0].sub!(
+          /in #{time_regex} \(#{time_regex}\) seconds/,
+          "in #{test_time_sum} (#{total_time_sum}) seconds"
+        )
+      end
+
+      def self.update_unexpected_failures(base_report, other_report)
+        base_unexpected_failures = unexpected_failures_from_summary(base_report['tests_summary_messages'][0])
+        other_unexpected_failures = unexpected_failures_from_summary(other_report['tests_summary_messages'][0])
+        base_report['tests_summary_messages'][0].sub!(
+          /\(\d+ unexpected\)/,
+          "(#{base_unexpected_failures + other_unexpected_failures} unexpected)"
+        )
+      end
+
+      def self.unexpected_failures_from_summary(summary)
+        /\((?<unexpected_failures>\d+) unexpected\)/ =~ summary
+        unexpected_failures.to_i
+      end
+
+      def self.times_from_summary(summary)
+        time_regex = '(?:\d|\.)+'
+        match = /in (?<test_time>#{time_regex}) \((?<total_time>#{time_regex})\) seconds/.match(summary)
+        return [
+          match['test_time'].to_f,
+          match['total_time'].to_f
+        ]
+      end
+
+      def self.update_failed_tests_count(base_report, other_report)
+        /\s+Executed \d+ tests?, with (?<failed_test_count>\d+) failures?/ =~ other_report['tests_summary_messages'][0]
+
+        base_report['tests_summary_messages'][0].sub!(
+          /(\d+) failure/,
+          "#{failed_test_count} failure"
+        )
+      end
+
+      def self.update_previous_test_failures(base_report)
+        previous_tests_failures = base_report['previous_tests_failures']
+        if previous_tests_failures
+          tests_failures = base_report['tests_failures']
+          tests_failures.each do |failure_suite, failures|
+            if previous_tests_failures.key?(failure_suite)
+              previous_tests_failures[failure_suite].concat(failures)
+            else
+              previous_tests_failures[failure_suite] = failures
+            end
+          end
+        else
+          base_report['previous_tests_failures'] = base_report['tests_failures'] || {}
+        end
+        base_report.delete('tests_failures')
+      end
+
+      #####################################################
+      # @!group Documentation
+      #####################################################
+
+      def self.description
+        "Combines and combines tests from multiple json report files"
+      end
+
+      def self.details
+        "The first JSON report is used as the base report. Due to the nature of " \
+        "xcpretty JSON reports, only the failing test cases are recorded. " \
+        "Testcases that failed in previous reports that no longer appear in " \
+        "later reports are assumed to have passed in a re-run, thus not appearing " \
+        "in the collated report. " \
+        "This is done because it is assumed that fragile tests, when " \
+        "re-run will often succeed due to less interference from other " \
+        "tests and the subsequent JSON reports will have fewer failing tests." \
+        ""
+      end
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(
+            key: :reports,
+            env_name: 'COLLATE_JSON_REPORTS_REPORTS',
+            description: 'An array of JSON reports to collate. The first report is used as the base into which other reports are merged in',
+            optional: false,
+            type: Array,
+            verify_block: proc do |reports|
+              UI.user_error!('No JSON report files found') if reports.empty?
+              reports.each do |report|
+                UI.user_error!("Error: JSON report not found: '#{report}'") unless File.exist?(report)
+              end
+            end
+          ),
+          FastlaneCore::ConfigItem.new(
+            key: :collated_report,
+            env_name: 'COLLATE_JSON_REPORTS_COLLATED_REPORT',
+            description: 'The final JSON report file where all testcases will be merged into',
+            optional: true,
+            default_value: 'result.json',
+            type: String
+          )
+        ]
+      end
+
+      def self.example_code
+        [
+          "
+          UI.important(
+            'example: ' \\
+            'collate the json reports to a temporary file \"result.json\"'
+          )
+          reports = Dir['../spec/fixtures/report*.json'].map { |relpath| File.absolute_path(relpath) }
+          collate_json_reports(
+            reports: reports,
+            collated_report: File.join(Dir.mktmpdir, 'result.json')
+          )
+          "
+        ]
+      end
+
+      def self.authors
+        ["lyndsey-ferguson/@lyndseydf"]
+      end
+
+      def self.is_supported?(platform)
+        platform == :ios
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/test_center/actions/multi_scan.rb

@@ -58,6 +58,11 @@ module Fastlane
             File.absolute_path(relative_filepath)
           end
         end
+        if reportnamer.includes_json?
+          report_files += Dir.glob("#{scan_options[:output_directory]}/**/#{reportnamer.json_fileglob}").map do |relative_filepath|
+            File.absolute_path(relative_filepath)
+          end
+        end
         {
           result: tests_passed,
           total_tests: passing_testcount + failed_tests.size,
@@ -76,6 +81,8 @@ module Fastlane
           batch_count
           testrun_completed_block
           test_without_building
+          output_types
+          output_files
         ]
         config = FastlaneCore::Configuration.create(
           Fastlane::Actions::ScanAction.available_options,
@@ -118,7 +125,7 @@ module Fastlane
       end
 
       def self.scan_options
-        ScanAction.available_options
+        ScanAction.available_options.reject { |config| config.key == :output_types }
       end
 
       def self.available_options
@@ -143,6 +150,13 @@ module Fastlane
               UI.user_error!("Error: Batch counts must be greater than zero") unless count > 0
             end
           ),
+          FastlaneCore::ConfigItem.new(
+            key: :output_types,
+            short_option: "-f",
+            env_name: "SCAN_OUTPUT_TYPES",
+            description: "Comma separated list of the output types (e.g. html, junit, json, json-compilation-database)",
+            default_value: "html,junit"
+          ),
           FastlaneCore::ConfigItem.new(
             key: :testrun_completed_block,
             description: 'A block invoked each time a test run completes',
@@ -286,7 +300,22 @@ module Fastlane
             workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
             scheme: 'AtomicBoy',
             try_count: 3,
-            only_testing: ['AtomicBoyTests']
+            only_testing: ['AtomicBoyTests'],
+            fail_build: false
+          )
+          ",
+          "
+          UI.important(
+            'example: ' \\
+            'multi_scan also works with json.'
+          )
+          multi_scan(
+            workspace: File.absolute_path('../AtomicBoy/AtomicBoy.xcworkspace'),
+            scheme: 'AtomicBoy',
+            try_count: 3,
+            output_types: 'json',
+            output_files: 'report.json',
+            fail_build: false
           )
           "
         ]

data/lib/fastlane/plugin/test_center/helper/correcting_scan_helper.rb

@@ -2,7 +2,7 @@ module TestCenter
   module Helper
     require 'fastlane_core/ui/ui.rb'
     require 'plist'
-
+    require 'json'
     class CorrectingScanHelper
       attr_reader :retry_total_count
 
@@ -26,6 +26,8 @@ module TestCenter
             custom_report_file_name
             fail_build
             testrun_completed_block
+            output_types
+            output_files
           ].include?(option)
         end
         @scan_options[:clean] = false
@@ -92,6 +94,59 @@ module TestCenter
       end
 
       def collate_reports(output_directory, reportnamer)
+        collate_junit_reports(output_directory, reportnamer)
+
+        if reportnamer.includes_html?
+          collate_html_reports(output_directory, reportnamer)
+        end
+
+        if reportnamer.includes_json?
+          collate_json_reports(output_directory, reportnamer)
+        end
+      end
+
+      def passed_test_count_from_summary(summary)
+        /.*Executed (?<test_count>\d+) test, with (?<test_failures>\d+) failure/ =~ summary
+        test_count.to_i - test_failures.to_i
+      end
+
+      def collate_json_reports(output_directory, reportnamer)
+        report_filepaths = Dir.glob("#{output_directory}/#{reportnamer.json_fileglob}").map do |relative_filepath|
+          File.absolute_path(relative_filepath)
+        end
+        if report_filepaths.size > 1
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::CollateJsonReportsAction.available_options,
+            {
+              reports: report_filepaths.sort { |f1, f2| File.mtime(f1) <=> File.mtime(f2) },
+              collated_report: File.absolute_path(File.join(output_directory, reportnamer.json_reportname))
+            }
+          )
+          Fastlane::Actions::CollateJsonReportsAction.run(config)
+        end
+        retried_json_reportfiles = Dir.glob("#{output_directory}/#{reportnamer.json_numbered_fileglob}")
+        FileUtils.rm_f(retried_json_reportfiles)
+      end
+
+      def collate_html_reports(output_directory, reportnamer)
+        report_files = Dir.glob("#{output_directory}/#{reportnamer.html_fileglob}").map do |relative_filepath|
+          File.absolute_path(relative_filepath)
+        end
+        if report_files.size > 1
+          config = FastlaneCore::Configuration.create(
+            Fastlane::Actions::CollateHtmlReportsAction.available_options,
+            {
+              reports: report_files.sort { |f1, f2| File.mtime(f1) <=> File.mtime(f2) },
+              collated_report: File.absolute_path(File.join(output_directory, reportnamer.html_reportname))
+            }
+          )
+          Fastlane::Actions::CollateHtmlReportsAction.run(config)
+        end
+        retried_html_reportfiles = Dir.glob("#{output_directory}/#{reportnamer.html_numbered_fileglob}")
+        FileUtils.rm_f(retried_html_reportfiles)
+      end
+
+      def collate_junit_reports(output_directory, reportnamer)
         report_files = Dir.glob("#{output_directory}/#{reportnamer.junit_fileglob}").map do |relative_filepath|
           File.absolute_path(relative_filepath)
         end
@@ -107,30 +162,13 @@ module TestCenter
         end
         retried_junit_reportfiles = Dir.glob("#{output_directory}/#{reportnamer.junit_numbered_fileglob}")
         FileUtils.rm_f(retried_junit_reportfiles)
-
-        if reportnamer.includes_html?
-          report_files = Dir.glob("#{output_directory}/#{reportnamer.html_fileglob}").map do |relative_filepath|
-            File.absolute_path(relative_filepath)
-          end
-          if report_files.size > 1
-            config = FastlaneCore::Configuration.create(
-              Fastlane::Actions::CollateHtmlReportsAction.available_options,
-              {
-                reports: report_files.sort { |f1, f2| File.mtime(f1) <=> File.mtime(f2) },
-                collated_report: File.absolute_path(File.join(output_directory, reportnamer.html_reportname))
-              }
-            )
-            Fastlane::Actions::CollateHtmlReportsAction.run(config)
-          end
-          retried_html_reportfiles = Dir.glob("#{output_directory}/#{reportnamer.html_numbered_fileglob}")
-          FileUtils.rm_f(retried_html_reportfiles)
-        end
       end
 
       def correcting_scan(scan_run_options, batch, reportnamer)
         scan_options = @scan_options.merge(scan_run_options)
         try_count = 0
         tests_passed = true
+        xcpretty_json_file_output = ENV['XCPRETTY_JSON_FILE_OUTPUT']
         begin
           try_count += 1
           config = FastlaneCore::Configuration.create(
@@ -138,6 +176,12 @@ module TestCenter
             scan_options.merge(reportnamer.scan_options)
           )
           quit_simulators
+          if reportnamer.includes_json?
+            ENV['XCPRETTY_JSON_FILE_OUTPUT'] = File.join(
+              scan_options[:output_directory],
+              reportnamer.json_last_reportname
+            )
+          end
           Fastlane::Actions::ScanAction.run(config)
           @testrun_completed_block && @testrun_completed_block.call(
             testrun_info(batch, try_count, reportnamer, scan_options[:output_directory])
@@ -164,6 +208,8 @@ module TestCenter
             retry
           end
           tests_passed = false
+        ensure
+          ENV['XCPRETTY_JSON_FILE_OUTPUT'] = xcpretty_json_file_output
         end
         tests_passed
       end
@@ -189,6 +235,10 @@ module TestCenter
           html_report_filepath = File.join(output_directory, reportnamer.html_last_reportname)
           info[:html_report_filepath] = html_report_filepath
         end
+        if reportnamer.includes_json?
+          json_report_filepath = File.join(output_directory, reportnamer.json_last_reportname)
+          info[:json_report_filepath] = json_report_filepath
+        end
         info
       end
 

data/lib/fastlane/plugin/test_center/helper/reportname_helper.rb

@@ -39,15 +39,24 @@ module TestCenter
       end
 
       def scan_options
+        options = {}
+
+        types = @output_types.split(',').each(&:chomp)
         files = @output_files.split(',').each(&:chomp)
+        if (json_index = types.find_index('json'))
+          options[:formatter] = 'xcpretty-json-formatter'
+          files.delete_at(json_index)
+          types.delete_at(json_index)
+        end
         files.map! do |filename|
           filename.chomp
           numbered_filename(filename)
         end
-        {
-          output_types: @output_types,
+
+        options.merge(
+          output_types: types.join(','),
           output_files: files.join(',')
-        }
+        )
       end
 
       def junit_last_reportname
@@ -98,6 +107,32 @@ module TestCenter
         "#{File.basename(html_reportname, '.*')}-[1-9]*#{html_filextension}"
       end
 
+      def includes_json?
+        @output_types.split(',').find_index('json') != nil
+      end
+
+      def json_last_reportname
+        json_index = @output_types.split(',').find_index('json')
+        numbered_filename(@output_files.to_s.split(',')[json_index])
+      end
+
+      def json_reportname
+        json_index = @output_types.split(',').find_index('json')
+        @output_files.to_s.split(',')[json_index]
+      end
+
+      def json_filextension
+        File.extname(json_reportname)
+      end
+
+      def json_fileglob
+        "#{File.basename(json_reportname, '.*')}*#{json_filextension}"
+      end
+
+      def json_numbered_fileglob
+        "#{File.basename(json_reportname, '.*')}-[1-9]*#{json_filextension}"
+      end
+
       def increment
         @report_count += 1
       end

data/lib/fastlane/plugin/test_center/version.rb

@@ -1,5 +1,5 @@
 module Fastlane
   module TestCenter
-    VERSION = "3.3.0"
+    VERSION = "3.4.0"
   end
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: fastlane-plugin-test_center
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Lyndsey Ferguson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-07 00:00:00.000000000 Z
+date: 2018-06-24 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: plist
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +192,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: xcpretty-json-formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |2
       This fastlane plugin includes the following actions:
       1) multi_scan: uses scan to run Xcode tests, optionally in batches, a given number of times: only re-testing failing tests.
@@ -196,6 +224,7 @@ files:
 - README.md
 - lib/fastlane/plugin/test_center.rb
 - lib/fastlane/plugin/test_center/actions/collate_html_reports.rb
+- lib/fastlane/plugin/test_center/actions/collate_json_reports.rb
 - lib/fastlane/plugin/test_center/actions/collate_junit_reports.rb
 - lib/fastlane/plugin/test_center/actions/multi_scan.rb
 - lib/fastlane/plugin/test_center/actions/suppress_tests.rb