fastlane 2.226.0 → 2.227.0

data/trainer/lib/trainer/test_parser.rb

@@ -2,18 +2,17 @@ require 'plist'
 
 require 'fastlane_core/print_table'
 
-require_relative 'junit_generator'
 require_relative 'xcresult'
+require_relative 'xcresult/helper'
+require_relative 'junit_generator'
+require_relative 'legacy_xcresult'
+require_relative 'plist_test_summary_parser'
 require_relative 'module'
 
 module Trainer
   class TestParser
     attr_accessor :data
 
-    attr_accessor :file_content
-
-    attr_accessor :raw_json
-
     attr_accessor :number_of_tests
     attr_accessor :number_of_failures
     attr_accessor :number_of_tests_excluding_retries
@@ -102,15 +101,16 @@ module Trainer
       UI.user_error!("File not found at path '#{path}'") unless File.exist?(path)
 
       if File.directory?(path) && path.end_with?(".xcresult")
-        parse_xcresult(path, output_remove_retry_attempts: config[:output_remove_retry_attempts])
+        parser = XCResult::Helper.supports_xcode16_xcresulttool? && !config[:force_legacy_xcresulttool] ? XCResult::Parser : LegacyXCResult::Parser
+        self.data = parser.parse_xcresult(path: path, output_remove_retry_attempts: config[:output_remove_retry_attempts])
       else
-        self.file_content = File.read(path)
-        self.raw_json = Plist.parse_xml(self.file_content)
+        file_content = File.read(path)
+        raw_json = Plist.parse_xml(file_content)
 
-        return if self.raw_json["FormatVersion"].to_s.length.zero? # maybe that's a useless plist file
+        return if raw_json["FormatVersion"].to_s.length.zero? # maybe that's a useless plist file
 
-        ensure_file_valid!
-        parse_content(config[:xcpretty_naming])
+        PlistTestSummaryParser.ensure_file_valid!(raw_json)
+        self.data = PlistTestSummaryParser.parse_content(raw_json, config[:xcpretty_naming])
       end
 
       self.number_of_tests = 0
@@ -131,293 +131,12 @@ module Trainer
 
     # Returns the JUnit report as String
     def to_junit
-      JunitGenerator.new(self.data).generate
+      self.data.kind_of?(Trainer::XCResult::TestPlan) ? self.data.to_xml : JunitGenerator.new(self.data).generate
     end
 
     # @return [Bool] were all tests successful? Is false if at least one test failed
     def tests_successful?
       self.data.collect { |a| a[:number_of_failures_excluding_retries] }.all?(&:zero?)
     end
-
-    private
-
-    def ensure_file_valid!
-      format_version = self.raw_json["FormatVersion"]
-      supported_versions = ["1.1", "1.2"]
-      UI.user_error!("Format version '#{format_version}' is not supported, must be #{supported_versions.join(', ')}") unless supported_versions.include?(format_version)
-    end
-
-    # Converts the raw plist test structure into something that's easier to enumerate
-    def unfold_tests(data)
-      # `data` looks like this
-      # => [{"Subtests"=>
-      #  [{"Subtests"=>
-      #     [{"Subtests"=>
-      #        [{"Duration"=>0.4,
-      #          "TestIdentifier"=>"Unit/testExample()",
-      #          "TestName"=>"testExample()",
-      #          "TestObjectClass"=>"IDESchemeActionTestSummary",
-      #          "TestStatus"=>"Success",
-      #          "TestSummaryGUID"=>"4A24BFED-03E6-4FBE-BC5E-2D80023C06B4"},
-      #         {"FailureSummaries"=>
-      #           [{"FileName"=>"/Users/krausefx/Developer/themoji/Unit/Unit.swift",
-      #             "LineNumber"=>34,
-      #             "Message"=>"XCTAssertTrue failed - ",
-      #             "PerformanceFailure"=>false}],
-      #          "TestIdentifier"=>"Unit/testExample2()",
-
-      tests = []
-      data.each do |current_hash|
-        if current_hash["Subtests"]
-          tests += unfold_tests(current_hash["Subtests"])
-        end
-        if current_hash["TestStatus"]
-          tests << current_hash
-        end
-      end
-      return tests
-    end
-
-    # Returns the test group and test name from the passed summary and test
-    # Pass xcpretty_naming = true to get the test naming aligned with xcpretty
-    def test_group_and_name(testable_summary, test, xcpretty_naming)
-      if xcpretty_naming
-        group = testable_summary["TargetName"] + "." + test["TestIdentifier"].split("/")[0..-2].join(".")
-        name = test["TestName"][0..-3]
-      else
-        group = test["TestIdentifier"].split("/")[0..-2].join(".")
-        name = test["TestName"]
-      end
-      return group, name
-    end
-
-    def execute_cmd(cmd)
-      output = `#{cmd}`
-      raise "Failed to execute - #{cmd}" unless $?.success?
-      return output
-    end
-
-    # Hotfix: From Xcode 16 beta 3 'xcresulttool get --format json' has been deprecated;
-    #         '--legacy' flag required to keep on using the command
-    def generate_cmd_parse_xcresult(path)
-      xcresulttool_cmd = %W(
-        xcrun
-        xcresulttool
-        get
-        --format
-        json
-        --path
-        #{path}
-      )
-
-      # e.g. DEVELOPER_DIR=/Applications/Xcode_16_beta_3.app
-      # xcresulttool version 23021, format version 3.53 (current)
-      match = `xcrun xcresulttool version`.match(/xcresulttool version (?<version>[\d.]+)/)
-      version = match[:version]
-      xcresulttool_cmd << '--legacy' if Gem::Version.new(version) >= Gem::Version.new(23_021)
-
-      xcresulttool_cmd.join(' ')
-    end
-
-    def parse_xcresult(path, output_remove_retry_attempts: false)
-      require 'shellwords'
-      path = Shellwords.escape(path)
-
-      # Executes xcresulttool to get JSON format of the result bundle object
-      # Hotfix: From Xcode 16 beta 3 'xcresulttool get --format json' has been deprecated; '--legacy' flag required to keep on using the command
-      xcresulttool_cmd = generate_cmd_parse_xcresult(path)
-
-      result_bundle_object_raw = execute_cmd(xcresulttool_cmd)
-      result_bundle_object = JSON.parse(result_bundle_object_raw)
-
-      # Parses JSON into ActionsInvocationRecord to find a list of all ids for ActionTestPlanRunSummaries
-      actions_invocation_record = Trainer::XCResult::ActionsInvocationRecord.new(result_bundle_object)
-      test_refs = actions_invocation_record.actions.map do |action|
-        action.action_result.tests_ref
-      end.compact
-      ids = test_refs.map(&:id)
-
-      # Maps ids into ActionTestPlanRunSummaries by executing xcresulttool to get JSON
-      # containing specific information for each test summary,
-      summaries = ids.map do |id|
-        raw = execute_cmd("#{xcresulttool_cmd} --id #{id}")
-        json = JSON.parse(raw)
-        Trainer::XCResult::ActionTestPlanRunSummaries.new(json)
-      end
-
-      # Converts the ActionTestPlanRunSummaries to data for junit generator
-      failures = actions_invocation_record.issues.test_failure_summaries || []
-      summaries_to_data(summaries, failures, output_remove_retry_attempts: output_remove_retry_attempts)
-    end
-
-    def summaries_to_data(summaries, failures, output_remove_retry_attempts: false)
-      # Gets flat list of all ActionTestableSummary
-      all_summaries = summaries.map(&:summaries).flatten
-      testable_summaries = all_summaries.map(&:testable_summaries).flatten
-
-      summaries_to_names = test_summaries_to_configuration_names(all_summaries)
-
-      # Maps ActionTestableSummary to rows for junit generator
-      rows = testable_summaries.map do |testable_summary|
-        all_tests = testable_summary.all_tests.flatten
-
-        # Used by store number of passes and failures by identifier
-        # This is used when Xcode 13 (and up) retries tests
-        # The identifier is duplicated until test succeeds or max count is reached
-        tests_by_identifier = {}
-
-        test_rows = all_tests.map do |test|
-          identifier = "#{test.parent.name}.#{test.name}"
-          test_row = {
-            identifier: identifier,
-            name: test.name,
-            duration: test.duration,
-            status: test.test_status,
-            test_group: test.parent.name,
-
-            # These don't map to anything but keeping empty strings
-            guid: ""
-          }
-
-          info = tests_by_identifier[identifier] || {}
-          info[:failure_count] ||= 0
-          info[:skip_count] ||= 0
-          info[:success_count] ||= 0
-
-          retry_count = info[:retry_count]
-          if retry_count.nil?
-            retry_count = 0
-          else
-            retry_count += 1
-          end
-          info[:retry_count] = retry_count
-
-          # Set failure message if failure found
-          failure = test.find_failure(failures)
-          if failure
-            test_row[:failures] = [{
-              file_name: "",
-              line_number: 0,
-              message: "",
-              performance_failure: {},
-              failure_message: failure.failure_message
-            }]
-
-            info[:failure_count] += 1
-          elsif test.test_status == "Skipped"
-            test_row[:skipped] = true
-            info[:skip_count] += 1
-          else
-            info[:success_count] = 1
-          end
-
-          tests_by_identifier[identifier] = info
-
-          test_row
-        end
-
-        # Remove retry attempts from the count and test rows
-        if output_remove_retry_attempts
-          test_rows = test_rows.reject do |test_row|
-            remove = false
-
-            identifier = test_row[:identifier]
-            info = tests_by_identifier[identifier]
-
-            # Remove if this row is a retry and is a failure
-            if info[:retry_count] > 0
-              remove = !(test_row[:failures] || []).empty?
-            end
-
-            # Remove all failure and retry count if test did eventually pass
-            if remove
-              info[:failure_count] -= 1
-              info[:retry_count] -= 1
-              tests_by_identifier[identifier] = info
-            end
-
-            remove
-          end
-        end
-
-        row = {
-          project_path: testable_summary.project_relative_path,
-          target_name: testable_summary.target_name,
-          test_name: testable_summary.name,
-          configuration_name: summaries_to_names[testable_summary],
-          duration: all_tests.map(&:duration).inject(:+),
-          tests: test_rows
-        }
-
-        row[:number_of_tests] = row[:tests].count
-        row[:number_of_failures] = row[:tests].find_all { |a| (a[:failures] || []).count > 0 }.count
-
-        # Used for seeing if any tests continued to fail after all of the Xcode 13 (and up) retries have finished
-        unique_tests = tests_by_identifier.values || []
-        row[:number_of_tests_excluding_retries] = unique_tests.count
-        row[:number_of_skipped] = unique_tests.map { |a| a[:skip_count] }.inject(:+)
-        row[:number_of_failures_excluding_retries] = unique_tests.find_all { |a| (a[:success_count] + a[:skip_count]) == 0 }.count
-        row[:number_of_retries] = unique_tests.map { |a| a[:retry_count] }.inject(:+)
-
-        row
-      end
-
-      self.data = rows
-    end
-
-    def test_summaries_to_configuration_names(test_summaries)
-      summary_to_name = {}
-      test_summaries.each do |summary|
-        summary.testable_summaries.each do |testable_summary|
-          summary_to_name[testable_summary] = summary.name
-        end
-      end
-      summary_to_name
-    end
-
-    # Convert the Hashes and Arrays in something more useful
-    def parse_content(xcpretty_naming)
-      self.data = self.raw_json["TestableSummaries"].collect do |testable_summary|
-        summary_row = {
-          project_path: testable_summary["ProjectPath"],
-          target_name: testable_summary["TargetName"],
-          test_name: testable_summary["TestName"],
-          duration: testable_summary["Tests"].map { |current_test| current_test["Duration"] }.inject(:+),
-          tests: unfold_tests(testable_summary["Tests"]).collect do |current_test|
-            test_group, test_name = test_group_and_name(testable_summary, current_test, xcpretty_naming)
-            current_row = {
-              identifier: current_test["TestIdentifier"],
-                 test_group: test_group,
-                 name: test_name,
-              object_class: current_test["TestObjectClass"],
-              status: current_test["TestStatus"],
-              guid: current_test["TestSummaryGUID"],
-              duration: current_test["Duration"]
-            }
-            if current_test["FailureSummaries"]
-              current_row[:failures] = current_test["FailureSummaries"].collect do |current_failure|
-                {
-                  file_name: current_failure['FileName'],
-                  line_number: current_failure['LineNumber'],
-                  message: current_failure['Message'],
-                  performance_failure: current_failure['PerformanceFailure'],
-                  failure_message: "#{current_failure['Message']} (#{current_failure['FileName']}:#{current_failure['LineNumber']})"
-                }
-              end
-            end
-            current_row
-          end
-        }
-        summary_row[:number_of_tests] = summary_row[:tests].count
-        summary_row[:number_of_failures] = summary_row[:tests].find_all { |a| (a[:failures] || []).count > 0 }.count
-
-        # Makes sure that plist support matches data output of xcresult
-        summary_row[:number_of_tests_excluding_retries] = summary_row[:number_of_tests]
-        summary_row[:number_of_failures_excluding_retries] = summary_row[:number_of_failures]
-        summary_row[:number_of_retries] = 0
-
-        summary_row
-      end
-    end
   end
 end

data/trainer/lib/trainer/xcresult/helper.rb

@@ -0,0 +1,53 @@
+require 'rexml/document'
+require 'rubygems'
+
+module Trainer
+  module XCResult
+    # Helper class for XML and node operations
+    class Helper
+      # Creates an XML element with the given name and attributes
+      #
+      # @param name [String] The name of the XML element
+      # @param attributes [Hash] A hash of attributes to add to the element
+      # @return [REXML::Element] The created XML element
+      def self.create_xml_element(name, **attributes)
+        # Sanitize invalid XML characters (control chars except tab/CR/LF) to avoid errors when generating XML
+        sanitizer = proc { |text| text.to_s.gsub(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/) { |c| format("\\u%04x", c.ord) } }
+        element = REXML::Element.new(sanitizer.call(name))
+        attributes.compact.each do |key, value|
+          safe_value = sanitizer.call(value.to_s)
+          element.attributes[key.to_s] = safe_value
+        end
+        element
+      end
+
+      # Find children of a node by specified node types
+      #
+      # @param node [Hash, nil] The JSON node to search within
+      # @param node_types [Array<String>] The node types to filter by
+      # @return [Array<Hash>] Array of child nodes matching the specified types
+      def self.find_json_children(node, *node_types)
+        return [] if node.nil? || node['children'].nil?
+
+        node['children'].select { |child| node_types.include?(child['nodeType']) }
+      end
+
+      # Check if the current xcresulttool supports new commands introduced in Xcode 16+
+      #
+      # Since Xcode 16b3, xcresulttool has marked `get <object> --format json` as deprecated/legacy,
+      # and replaced it with `xcrun xcresulttool get test-results tests` instead.
+      #
+      # @return [Boolean] Whether the xcresulttool supports Xcode 16+ commands
+      def self.supports_xcode16_xcresulttool?
+        # e.g. DEVELOPER_DIR=/Applications/Xcode_16_beta_3.app
+        # xcresulttool version 23021, format version 3.53 (current)
+        match = `xcrun xcresulttool version`.match(/xcresulttool version (?<version>[\d.]+)/)
+        version = match[:version]
+
+        Gem::Version.new(version) >= Gem::Version.new(23_021)
+      rescue
+        false
+      end
+    end
+  end
+end

data/trainer/lib/trainer/xcresult/repetition.rb

@@ -0,0 +1,39 @@
+require_relative 'test_case_attributes'
+
+module Trainer
+  module XCResult
+    # Represents retries of a test case, including the original run
+    # e.g. a test case that ran 3 times in total will be represented by 3 Repetition instances in the `xcresulttool` JSON output,
+    # one for the original run and one for the 2 retries.
+    class Repetition
+      include TestCaseAttributes
+
+      attr_reader :name
+      attr_reader :duration
+      attr_reader :result
+      attr_reader :failure_messages
+      attr_reader :source_references
+      attr_reader :attachments
+
+      def initialize(name:, duration:, result:, failure_messages: [], source_references: [], attachments: [])
+        @name = name
+        @duration = duration
+        @result = result
+        @failure_messages = failure_messages
+        @source_references = source_references
+        @attachments = attachments
+      end
+
+      def self.from_json(node:)
+        new(
+          name: node['name'],
+          duration: parse_duration(node['duration']),
+          result: node['result'],
+          failure_messages: extract_failure_messages(node),
+          source_references: extract_source_references(node),
+          attachments: extract_attachments(node)
+        )
+      end
+    end
+  end
+end

data/trainer/lib/trainer/xcresult/test_case.rb

@@ -0,0 +1,221 @@
+require_relative 'helper'
+require_relative 'test_case_attributes'
+require_relative 'repetition'
+
+module Trainer
+  module XCResult
+    # Represents a test case, including its retries (aka repetitions)
+    class TestCase
+      include TestCaseAttributes
+
+      attr_reader :name
+      attr_reader :identifier
+      attr_reader :duration
+      attr_reader :result
+      attr_reader :classname
+      attr_reader :argument
+      # @return [Array<Repetition>] Array of retry attempts for this test case, **including the initial attempt**
+      # This will be `nil` if the test case was not run multiple times, but will contain all repetitions if it was run more than once.
+      attr_reader :retries
+      attr_reader :failure_messages
+      attr_reader :source_references
+      attr_reader :attachments
+      attr_reader :tags
+
+      def initialize(
+        name:, identifier:, duration:, result:, classname:, argument: nil, tags: [], retries: nil,
+        failure_messages: [], source_references: [], attachments: []
+      )
+        @name = name
+        @identifier = identifier
+        @duration = duration
+        @result = result
+        @classname = classname
+        @argument = argument
+        @tags = tags
+        @retries = retries
+        @failure_messages = failure_messages
+        @source_references = source_references
+        @attachments = attachments
+      end
+
+      def self.from_json(node:)
+        # Handle test case arguments
+        argument_nodes = Helper.find_json_children(node, 'Arguments')
+        argument_nodes = [nil] if argument_nodes.empty?
+
+        # Generate test cases for each argument
+        argument_nodes.map do |arg_node|
+          # For repetition nodes, failure messages, source refs, attachments and result attributes,
+          # Search them as children of the argument child node if present, of the test case node otherwise.
+          node_for_attributes = arg_node || node
+
+          retries = Helper.find_json_children(node_for_attributes, 'Repetition', 'Test Case Run')
+                         &.map { |rep_node| Repetition.from_json(node: rep_node) } || []
+
+          failure_messages = if retries.empty?
+                               extract_failure_messages(node_for_attributes)
+                             else
+                               retries.flat_map(&:failure_messages)
+                             end
+
+          source_references = if retries.empty?
+                                extract_source_references(node_for_attributes)
+                              else
+                                retries.flat_map(&:source_references)
+                              end
+
+          attachments = if retries.empty?
+                          extract_attachments(node_for_attributes)
+                        else
+                          retries.flat_map(&:attachments)
+                        end
+
+          new(
+            name: node['name'],
+            identifier: node['nodeIdentifier'],
+            duration: parse_duration(node['duration']),
+            result: node_for_attributes['result'],
+            classname: extract_classname(node),
+            argument: arg_node&.[]('name'), # Only set if there is an argument
+            tags: node['tags'] || [],
+            retries: retries,
+            failure_messages: failure_messages,
+            source_references: source_references,
+            attachments: attachments
+          )
+        end
+      end
+
+      # Generates XML nodes for the test case
+      #
+      # @return [Array<REXML::Element>] An array of XML <testcase> elements
+      #
+      # - If no retries, the array contains a single <testcase> element
+      # - If retries, the array contains one <testcase> element per retry
+      def to_xml_nodes
+        runs = @retries.nil? || @retries.empty? ? [nil] : @retries
+
+        runs.map do |run|
+          Helper.create_xml_element('testcase',
+            name: if @argument.nil?
+                    @name
+                  else
+                    @name.match?(/(\(.*\))/) ? @name.gsub(/(\(.*\))/, "(#{@argument})") : "#{@name} (#{@argument})"
+                  end,
+            classname: @classname,
+            time: (run || self).duration.to_s).tap do |testcase|
+            add_xml_result_elements(testcase, run || self)
+            add_properties_to_xml(testcase, repetition_name: run&.name)
+          end
+        end
+      end
+
+      def retries_count
+        @retries&.count || 0
+      end
+
+      def total_tests_count
+        retries_count > 0 ? retries_count : 1
+      end
+
+      def total_failures_count
+        if retries_count > 0
+          @retries.count(&:failed?)
+        elsif failed?
+          1
+        else
+          0
+        end
+      end
+
+      def self.extract_classname(node)
+        return nil if node['nodeIdentifier'].nil?
+
+        parts = node['nodeIdentifier'].split('/')
+        parts[0...-1].join('.')
+      end
+      private_class_method :extract_classname
+
+      private
+
+      # Adds <properties> element to the XML <testcase> element
+      #
+      # @param testcase [REXML::Element] The XML testcase element to add properties to
+      # @param repetition_name [String, nil] Name of the retry attempt, if this is a retry
+      #
+      # Properties added:
+      # - if argument is present:
+      #   - `testname`: Raw test name (as in such case, <testcase name="…"> would contain a mix of the test name and the argument)
+      #   - `argument`: Test argument value
+      # - `repetitionN`: Name of the retry attempt if present
+      # - `source_referenceN`: Source code references (file/line) for failures
+      # - `attachmentN`: Test attachments like screenshots
+      # - `tagN`: Test tags/categories
+      #
+      # <properties> element is only added to the XML if at least one property exists
+      def add_properties_to_xml(testcase, repetition_name: nil)
+        properties = Helper.create_xml_element('properties')
+
+        # Add argument as property
+        if @argument
+          name_prop = Helper.create_xml_element('property', name: "testname", value: @name)
+          properties.add_element(name_prop)
+          prop = Helper.create_xml_element('property', name: "argument", value: @argument)
+          properties.add_element(prop)
+        end
+
+        # Add repetition as property
+        if repetition_name
+          prop = Helper.create_xml_element('property', name: "repetition", value: repetition_name)
+          properties.add_element(prop)
+        end
+
+        # Add source references as properties
+        (@source_references || []).each_with_index do |ref, index|
+          prop = Helper.create_xml_element('property', name: "source_reference#{index + 1}", value: ref)
+          properties.add_element(prop)
+        end
+
+        # Add attachments as properties
+        (@attachments || []).each_with_index do |attachment, index|
+          prop = Helper.create_xml_element('property', name: "attachment#{index + 1}", value: attachment)
+          properties.add_element(prop)
+        end
+
+        # Add tags as properties
+        (@tags || []).sort.each_with_index do |tag, index|
+          prop = Helper.create_xml_element('property', name: "tag#{index + 1}", value: tag)
+          properties.add_element(prop)
+        end
+
+        # Only add properties to testcase if it has child elements
+        testcase.add_element(properties) if properties.elements.any?
+      end
+
+      # Adds <failure> and <skipped> elements to the XML <testcase> element based on test status
+      #
+      # @param testcase [REXML::Element] The XML testcase element to add result elements to
+      # @param test_obj [Repetition, TestCase] Object representing the test result
+      #   This can be either a Repetition object or the TestCase itself.
+      #   Must respond to the following methods:
+      #   - failed? [Boolean]: Indicates if the test failed
+      #   - skipped? [Boolean]: Indicates if the test was skipped
+      #   - failure_messages [Array<String>, nil]: List of failure messages (optional)
+      #
+      # Adds:
+      # - <failure> elements with messages for failed tests
+      # - <skipped> element for skipped tests
+      # - No elements added for passed tests
+      def add_xml_result_elements(testcase, test_obj)
+        if test_obj.failed?
+          (test_obj.failure_messages&.any? ? test_obj.failure_messages : [nil]).each do |msg|
+            testcase.add_element(Helper.create_xml_element('failure', message: msg))
+          end
+        elsif test_obj.skipped?
+          testcase.add_element(Helper.create_xml_element('skipped', message: test_obj.failure_messages&.first))
+        end
+      end
+    end
+  end
+end