gitlab_quality-test_tooling 0.8.3 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3472eb65666a6691797e9e0f0332a1773bb6babe898245a375bd9976523866e
-  data.tar.gz: 6372e6df38afb56bbe300b39d25a65cb7cdd205765ded99cab3745f4fbe7d812
+  metadata.gz: e9bcb8c6202edb63313039592fdf1a6c953c0b43db4566d9e84105e31d2029d3
+  data.tar.gz: 27dcc6d668da50a3ec76bd13b8898d61b24bbe2835faeee6fc6dbce684945388
 SHA512:
-  metadata.gz: 899483550e08f6f873d137d6a1bc90620ed58eff093827e76e5b4c92de8b7b1f95bba60181d2b726d79bf91b08b627cc93fb3817662dff73db8ba46e81557aad
-  data.tar.gz: 6b21fae0eb76efa44dd868113d22d36c30a9f5d06b5bb1b625dd42129a463a24ae77abf4edf6df05d06800b29261116492cb39aa892490aaa22cfca9ce442aa4
+  metadata.gz: cd3003fced80af3cd8ebafa3495d799eb63756b659b4c6e002c27557ea84beb3ec4bba1bbb08784cf27fe74ab6f4d1c6eb2ed7888fdbf317209683475247310b
+  data.tar.gz: 579e87d66fa4107263220501f4959e3b08b0dbb887224ecb11917a69e5d6774e9cfdd2298d62a20992bc27b7b4be3f561e35574f599a4cbb165230331261e9c5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (0.8.3)
+    gitlab_quality-test_tooling (0.9.0)
       activesupport (>= 6.1, < 7.1)
       gitlab (~> 4.19)
       http (~> 5.0)

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -27,7 +27,7 @@ module GitlabQuality
         FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
         REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
         NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
-        IGNORE_EXCEPTIONS = [
+        IGNORED_FAILURES = [
           'Net::ReadTimeout',
           '403 Forbidden - Your account has been blocked'
         ].freeze
@@ -54,19 +54,19 @@ module GitlabQuality
           TestResults::Builder.new(files).test_results_per_file do |test_results|
             puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
 
-            systemic_exceptions = systemic_exceptions_for_test_results(test_results)
+            systemic_failures = systemic_failures_for_test_results(test_results)
 
             test_results.each do |test|
-              relate_failure_to_issue(test) if should_report?(test, systemic_exceptions)
+              relate_failure_to_issue(test) if should_report?(test, systemic_failures)
             end
 
             test_results.write
           end
         end
 
-        def systemic_exceptions_for_test_results(test_results)
+        def systemic_failures_for_test_results(test_results)
           test_results
-            .flat_map { |test| test.report['exceptions']&.map { |exception| exception['message'] } }
+            .flat_map { |test| test.failures.map { |failure| failure['message'].lines.first.chomp } }
             .compact
             .tally
             .select { |_e, count| count >= SYSTEMIC_EXCEPTIONS_THRESHOLD }
@@ -172,18 +172,16 @@ module GitlabQuality
         end
 
         def failure_issues(test)
-          search_labels = (base_issue_labels + Set.new(%w[test])).to_a
-          gitlab.find_issues(options: { state: 'opened', labels: search_labels }).select do |issue|
-            issue_title = issue.title.strip
-            issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
-          end
+          find_issues(test, (base_issue_labels + Set.new(%w[test])).to_a)
         end
 
         def full_stacktrace(test)
-          if test.failures.first['message_lines'].empty? || test.failures.first['message_lines'].instance_of?(String)
-            test.failures.first['message']
+          first_failure = test.failures.first
+
+          if first_failure['message_lines'].empty?
+            first_failure['message']
           else
-            test.failures.first['message_lines'].join("\n")
+            first_failure['message_lines'].join("\n")
           end
         end
 
@@ -371,7 +369,7 @@ module GitlabQuality
           failure = full_stacktrace(test)
           return if SCREENSHOT_IGNORED_ERRORS.any? { |e| failure.include?(e) }
 
-          relative_url = gitlab.upload_file(file_fullpath: test.failure_screenshot)
+          relative_url = gitlab.upload_file(file_fullpath: test.screenshot_image)
           return unless relative_url
 
           "### Screenshot\n\n#{relative_url.markdown}"
@@ -381,37 +379,44 @@ module GitlabQuality
         #
         # @return [TrueClass|FalseClass] false if the test was skipped or failed because of a transient error that can be ignored.
         # Otherwise returns true.
-        def should_report?(test, systemic_exceptions)
-          return false if test.failures.empty?
+        def should_report?(test, systemic_failure_messages)
+          return false unless test.failures?
 
-          puts "  => Systemic exceptions detected: #{systemic_exceptions}" if systemic_exceptions.any?
-          exceptions_to_ignore = IGNORE_EXCEPTIONS + systemic_exceptions
+          puts "  => Systemic failures detected: #{systemic_failure_messages}" if systemic_failure_messages.any?
+          failure_to_ignore = IGNORED_FAILURES + systemic_failure_messages
 
-          if test.report.key?('exceptions')
-            reason = ignore_failure_reason(test.report['exceptions'], exceptions_to_ignore)
+          reason = ignored_failure_reason(test.failures, failure_to_ignore)
 
-            if reason
-              puts "  => Failure reporting skipped because #{reason}"
+          if reason
+            puts "  => Failure reporting skipped because #{reason}"
 
-              return false
-            end
+            false
+          else
+            true
           end
-
-          true
         end
 
         # Determine any reason to ignore a failure.
         #
-        # @param [Array<Hash>] exceptions the exceptions associated with the failure.
-        # @return [String] the reason to ignore the exceptions, or `nil` if any exceptions should not be ignored.
-        def ignore_failure_reason(exceptions, ignored_exceptions)
-          exception_messages = exceptions
-            .filter_map { |exception| exception['message'] if ignored_exceptions.any? { |e| exception['message'].include?(e) } }
-            .compact
-          return if exception_messages.empty? || exception_messages.size < exceptions.size
+        # @param [Array<Hash>] failures the failures associated with the failure.
+        # @param [Array<String>] failure_to_ignore the failures messages that should be ignored.
+        # @return [String] the reason to ignore the failures, or `nil` if any failures should not be ignored.
+        def ignored_failure_reason(failures, failure_to_ignore)
+          failures_to_ignore = compute_ignored_failures(failures, failure_to_ignore)
+          return if failures_to_ignore.empty? || failures_to_ignore.size < failures.size
+
+          "the errors included: #{failures_to_ignore.map { |e| "`#{e}`" }.join(', ')}"
+        end
 
-          msg = exception_messages.many? ? 'the errors were' : 'the error was'
-          "#{msg} #{exception_messages.join(', ')}"
+        # Determine the failures that should be ignored based on a list of exception messages to ignore.
+        #
+        # @param [Array<Hash>] failures the failures associated with the failure.
+        # @param [Array<String>] failure_to_ignore the failures messages that should be ignored.
+        # @return [Array<String>] the exception messages to ignore, or `nil` if any failures should not be ignored.
+        def compute_ignored_failures(failures, failure_to_ignore)
+          failures
+            .filter_map { |e| failure_to_ignore.find { |m| e['message'].include?(m) } }
+            .compact
         end
       end
     end

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -50,7 +50,7 @@ module GitlabQuality
         def test_file_link(test)
           path_prefix = test.file.start_with?('qa/') ? 'qa/' : ''
 
-          "[`#{path_prefix}#{test.file}`](#{FILE_BASE_URL}#{path_prefix}#{test.file}##{test.line_number})"
+          "[`#{path_prefix}#{test.file}`](#{FILE_BASE_URL}#{path_prefix}#{test.file}#L#{test.line_number})"
         end
 
         def new_issue_labels(_test)
@@ -128,6 +128,13 @@ module GitlabQuality
           labels
         end
 
+        def find_issues(test, labels)
+          gitlab.find_issues(options: { state: 'opened', labels: labels.to_a }).find_all do |issue|
+            issue_title = issue.title.strip
+            issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
+          end
+        end
+
         def pipeline_name_label
           case pipeline
           when 'production'

data/lib/gitlab_quality/test_tooling/report/report_results.rb

@@ -40,7 +40,7 @@ module GitlabQuality
             puts "Reporting tests in #{test_results.path}"
 
             test_results.each do |test|
-              next if test.file.include?('/features/sanity/') || test.skipped
+              next if test.file.include?('/features/sanity/') || test.skipped?
 
               puts "Reporting test: #{test.file} | #{test.name}\n"
 

data/lib/gitlab_quality/test_tooling/report/results_in_issues.rb

@@ -35,7 +35,7 @@ module GitlabQuality
         end
 
         def post_note(issue, test)
-          return false if test.skipped
+          return false if test.skipped?
           return false if test.failures.empty?
 
           note = note_content(test)

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -59,24 +59,17 @@ module GitlabQuality
         def create_slow_issue(test)
           puts "   => Finding existing issues for slow test '#{test.name}' (run time: #{test.run_time} seconds)..."
 
-          issue = find_issue(test)
+          issues = find_issues(test, SEARCH_LABELS)
 
-          puts "   => Existing issue link #{issue['web_url']}" if issue.present?
+          issues.each do |issue|
+            puts "   => Existing issue link #{issue['web_url']}"
+          end
 
-          create_issue(test) unless issue.present?
+          create_issue(test) unless issues.any?
         rescue MultipleIssuesFound => e
           warn(e.message)
         end
 
-        def find_issue(test)
-          search_labels = SEARCH_LABELS
-
-          gitlab.find_issues(options: { state: 'opened', labels: search_labels.to_a }).find do |issue|
-            issue_title = issue.title.strip
-            issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
-          end
-        end
-
         def should_create_slow_issue?(test)
           test.run_time > max_duration_for_test(test)
         end

data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module TestResult
+      class BaseTestResult
+        attr_reader :report
+
+        def initialize(report)
+          @report = report
+        end
+
+        def stage
+          @stage ||= file[%r{(?:api|browser_ui)/(?:(?:\d+_)?(\w+))}, 1]
+        end
+
+        def name
+          raise NotImplementedError
+        end
+
+        def file
+          raise NotImplementedError
+        end
+
+        def skipped?
+          raise NotImplementedError
+        end
+
+        def failures
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_result/j_unit_test_result.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module TestResult
+      class JUnitTestResult < BaseTestResult
+        attr_accessor :testcase # Ignore it for now
+
+        def name
+          report['name']
+        end
+
+        def file
+          report['file'].delete_prefix('./')
+        end
+
+        def skipped?
+          report.search('skipped').any?
+        end
+
+        def failures # rubocop:disable Metrics/AbcSize
+          failures = report.search('failure')
+          return [] if failures.empty?
+
+          failures.map do |exception|
+            trace = exception.content.split("\n").map(&:strip)
+            spec_file_first_index = trace.rindex do |line|
+              line.include?(File.basename(report['file']))
+            end
+
+            exception['message'].gsub!(/(private_token=)[\w-]+/, '********')
+
+            {
+              'message' => "#{exception['type']}: #{exception['message']}",
+              'stacktrace' => trace.slice(0..spec_file_first_index).join("\n")
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_result/json_test_result.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module TestResult
+      class JsonTestResult < BaseTestResult
+        PRIVATE_TOKEN_REGEX = /(private_token=)[\w-]+/
+
+        def name
+          report.fetch('full_description')
+        end
+
+        def file
+          report.fetch('file_path').delete_prefix('./')
+        end
+
+        def status
+          report.fetch('status')
+        end
+
+        def skipped?
+          status == 'pending'
+        end
+
+        def ci_job_url
+          report.fetch('ci_job_url', '')
+        end
+
+        def testcase
+          report.fetch('testcase', '')
+        end
+
+        def testcase=(new_testcase)
+          report['testcase'] = new_testcase
+        end
+
+        def failure_issue
+          report['failure_issue']
+        end
+
+        def failure_issue=(new_failure_issue)
+          report['failure_issue'] = new_failure_issue
+        end
+
+        def quarantine?
+          # The value for 'quarantine' could be nil, a hash, a string,
+          # or true (if the test just has the :quarantine tag)
+          # But any non-nil or false value should means the test is in quarantine
+          !!quarantine
+        end
+
+        def quarantine_type
+          quarantine['type'] if quarantine?
+        end
+
+        def quarantine_issue
+          quarantine['issue'] if quarantine?
+        end
+
+        def screenshot?
+          !!screenshot
+        end
+
+        def screenshot_image
+          screenshot['image'] if screenshot?
+        end
+
+        def product_group
+          report['product_group'].to_s
+        end
+
+        def product_group?
+          product_group != ''
+        end
+
+        def feature_category
+          report['feature_category']
+        end
+
+        def run_time
+          report['run_time'].to_f.round(2)
+        end
+
+        def example_id
+          report['id']
+        end
+
+        def line_number
+          report['line_number']
+        end
+
+        def failures # rubocop:disable Metrics/AbcSize
+          @failures ||=
+            report.fetch('exceptions', []).filter_map do |exception|
+              backtrace = exception['backtrace']
+              next unless backtrace.respond_to?(:rindex)
+
+              spec_file_first_index = backtrace.rindex do |line|
+                line.include?(File.basename(report['file_path']))
+              end
+
+              message = redact_private_token(exception['message'])
+              message_lines = Array(exception['message_lines']).map { |line| redact_private_token(line) }
+
+              {
+                'message' => "#{exception['class']}: #{message}",
+                'message_lines' => message_lines,
+                'stacktrace' => "#{format_message_lines(message_lines)}\n#{backtrace.slice(0..spec_file_first_index).join("\n")}",
+                'correlation_id' => exception['correlation_id']
+              }
+            end
+        end
+
+        def failures?
+          failures.any?
+        end
+
+        private
+
+        def quarantine
+          report.fetch('quarantine', nil)
+        end
+
+        def screenshot
+          report.fetch('screenshot', nil)
+        end
+
+        def format_message_lines(message_lines)
+          message_lines.is_a?(Array) ? message_lines.join("\n") : message_lines
+        end
+
+        def redact_private_token(text)
+          text.gsub(PRIVATE_TOKEN_REGEX, '********')
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_results/j_unit_test_results.rb

@@ -18,7 +18,7 @@ module GitlabQuality
 
         def process
           results.xpath('//testcase').map do |test|
-            TestResult.from_junit(test)
+            GitlabQuality::TestTooling::TestResult::JUnitTestResult.new(test)
           end
         end
       end

data/lib/gitlab_quality/test_tooling/test_results/json_test_results.rb

@@ -20,7 +20,7 @@ module GitlabQuality
 
         def process
           results['examples'].map do |test|
-            TestResult.from_json(test)
+            GitlabQuality::TestTooling::TestResult::JsonTestResult.new(test)
           end
         end
       end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "0.8.3"
+    VERSION = "0.9.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.9.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-06-28 00:00:00.000000000 Z
+date: 2023-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -384,11 +384,13 @@ files:
 - lib/gitlab_quality/test_tooling/system_logs/log_types/rails/graphql_log.rb
 - lib/gitlab_quality/test_tooling/system_logs/shared_fields.rb
 - lib/gitlab_quality/test_tooling/system_logs/system_logs_formatter.rb
+- lib/gitlab_quality/test_tooling/test_result/base_test_result.rb
+- lib/gitlab_quality/test_tooling/test_result/j_unit_test_result.rb
+- lib/gitlab_quality/test_tooling/test_result/json_test_result.rb
 - lib/gitlab_quality/test_tooling/test_results/base_test_results.rb
 - lib/gitlab_quality/test_tooling/test_results/builder.rb
 - lib/gitlab_quality/test_tooling/test_results/j_unit_test_results.rb
 - lib/gitlab_quality/test_tooling/test_results/json_test_results.rb
-- lib/gitlab_quality/test_tooling/test_results/test_result.rb
 - lib/gitlab_quality/test_tooling/version.rb
 - sig/gitlab_quality/test_tooling.rbs
 homepage: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling

data/lib/gitlab_quality/test_tooling/test_results/test_result.rb

@@ -1,200 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_support/core_ext/object/blank'
-
-module GitlabQuality
-  module TestTooling
-    module TestResults
-      class TestResult
-        def self.from_json(report)
-          JsonTestResult.new(report)
-        end
-
-        def self.from_junit(report)
-          JUnitTestResult.new(report)
-        end
-
-        attr_accessor :report, :failures
-
-        def initialize(report)
-          self.report = report
-          self.failures = failures_from_exceptions
-        end
-
-        def stage
-          @stage ||= file[%r{(?:api|browser_ui)/(?:(?:\d+_)?(\w+))}, 1]
-        end
-
-        def name
-          raise NotImplementedError
-        end
-
-        def file
-          raise NotImplementedError
-        end
-
-        def skipped
-          raise NotImplementedError
-        end
-
-        private
-
-        def failures_from_exceptions
-          raise NotImplementedError
-        end
-
-        class JsonTestResult < TestResult
-          def name
-            report['full_description']
-          end
-
-          def file
-            report['file_path'].delete_prefix('./')
-          end
-
-          def status
-            report['status']
-          end
-
-          def ci_job_url
-            report['ci_job_url']
-          end
-
-          def skipped
-            status == 'pending'
-          end
-
-          def testcase
-            report['testcase']
-          end
-
-          def testcase=(new_testcase)
-            report['testcase'] = new_testcase
-          end
-
-          def failure_issue
-            report['failure_issue']
-          end
-
-          def failure_issue=(new_failure_issue)
-            report['failure_issue'] = new_failure_issue
-          end
-
-          def quarantine?
-            # The value for 'quarantine' could be nil, a hash, a string,
-            # or true (if the test just has the :quarantine tag)
-            # But any non-nil or false value should means the test is in quarantine
-            report['quarantine'].present?
-          end
-
-          def quarantine_type
-            report['quarantine']['type'] if quarantine?
-          end
-
-          def quarantine_issue
-            report['quarantine']['issue'] if quarantine?
-          end
-
-          def screenshot?
-            report['screenshot'].present?
-          end
-
-          def failure_screenshot
-            report['screenshot']['image'] if screenshot?
-          end
-
-          def product_group?
-            report['product_group'].present?
-          end
-
-          def product_group
-            report['product_group']
-          end
-
-          def feature_category
-            report['feature_category']
-          end
-
-          def run_time
-            report['run_time'].to_f.round(2)
-          end
-
-          def example_id
-            report['id']
-          end
-
-          def line_number
-            report['line_number']
-          end
-
-          private
-
-          # rubocop:disable Metrics/AbcSize
-          def failures_from_exceptions
-            return [] unless report.key?('exceptions')
-
-            report['exceptions'].map do |exception|
-              spec_file_first_index = exception['backtrace'].rindex do |line|
-                line.include?(File.basename(report['file_path']))
-              end
-
-              exception['message'].gsub!(/(private_token=)[\w-]+/, '********')
-              Array(exception['message_lines']).each { |line| line.gsub!(/(private_token=)([\w-]+)/, '********') }
-
-              {
-                'message' => "#{exception['class']}: #{exception['message']}",
-                'message_lines' => exception['message_lines'],
-                'stacktrace' => "#{format_message_lines(exception['message_lines'])}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}",
-                'correlation_id' => exception['correlation_id']
-              }
-            end
-          end
-
-          def format_message_lines(message_lines)
-            message_lines.is_a?(Array) ? message_lines.join("\n") : message_lines
-          end
-          # rubocop:enable Metrics/AbcSize
-        end
-
-        class JUnitTestResult < TestResult
-          def name
-            report['name']
-          end
-
-          def file
-            report['file'].delete_prefix('./')
-          end
-
-          def skipped
-            report.search('skipped').any?
-          end
-
-          attr_accessor :testcase # Ignore it for now
-
-          private
-
-          # rubocop:disable Metrics/AbcSize
-          def failures_from_exceptions
-            failures = report.search('failure')
-            return [] if failures.empty?
-
-            failures.map do |exception|
-              trace = exception.content.split("\n").map(&:strip)
-              spec_file_first_index = trace.rindex do |line|
-                line.include?(File.basename(report['file']))
-              end
-
-              exception['message'].gsub!(/(private_token=)[\w-]+/, '********')
-
-              {
-                'message' => "#{exception['type']}: #{exception['message']}",
-                'stacktrace' => trace.slice(0..spec_file_first_index).join("\n")
-              }
-            end
-          end
-          # rubocop:enable Metrics/AbcSize
-        end
-      end
-    end
-  end
-end