gitlab-qa 6.7.0 → 6.11.0

data/lib/gitlab/qa/report/gitlab_issue_dry_client.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class GitlabIssueDryClient < GitlabIssueClient
+        def create_issue(title:, description:, labels:)
+          attrs = { description: description, labels: labels }
+
+          puts "The following issue would have been created:"
+          puts "project: #{project}, title: #{title}, attrs: #{attrs}"
+        end
+
+        def edit_issue(iid:, options: {})
+          puts "The #{project}##{iid} issue would have been updated with: #{options}"
+        end
+
+        def create_issue_note(iid:, note:)
+          puts "The following note would have been posted on #{project}##{iid} issue: #{note}"
+        end
+
+        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
+          puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{body}"
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/json_test_results.rb

@@ -6,7 +6,7 @@ module Gitlab
   module QA
     module Report
       class JsonTestResults < BaseTestResults
-        def write(path)
+        def write
           json = results.merge('examples' => testcases.map(&:report))
 
           File.write(path, JSON.pretty_generate(json))
@@ -14,7 +14,7 @@ module Gitlab
 
         private
 
-        def parse(path)
+        def parse
           JSON.parse(File.read(path))
         end
 

data/lib/gitlab/qa/report/junit_test_results.rb

@@ -6,13 +6,13 @@ module Gitlab
   module QA
     module Report
       class JUnitTestResults < BaseTestResults
-        def write(path)
+        def write
           # Ignore it for now
         end
 
         private
 
-        def parse(path)
+        def parse
           Nokogiri::XML.parse(File.read(path))
         end
 

data/lib/gitlab/qa/report/relate_failure_issue.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'active_support/core_ext/enumerable'
+require 'rubygems/text'
+
+module Gitlab
+  module QA
+    module Report
+      # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
+      class RelateFailureIssue < ReportAsIssue
+        DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
+        STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error:.+)(\1)}m.freeze
+        NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
+
+        MultipleIssuesFound = Class.new(StandardError)
+
+        def initialize(max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION, **kwargs)
+          super
+          @max_diff_ratio = max_diff_ratio.to_f
+        end
+
+        private
+
+        attr_reader :max_diff_ratio
+
+        def run!
+          puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          test_results_per_file do |test_results|
+            puts "=> Reporting tests in #{test_results.path}"
+
+            test_results.each do |test|
+              next if test.failures.empty?
+
+              relate_test_to_issue(test)
+            end
+
+            test_results.write
+          end
+        end
+
+        def relate_test_to_issue(test)
+          puts "  => Searching issues for test '#{test.name}'..."
+
+          begin
+            issue = find_or_create_issue(test)
+            return unless issue
+
+            update_labels(issue, test)
+            post_failed_job_note(issue, test)
+            puts "  => Marked #{issue.web_url} as related to #{test.testcase}."
+          rescue MultipleIssuesFound => e
+            warn(e.message)
+          end
+        end
+
+        def find_or_create_issue(test)
+          issue, diff_ratio = find_failure_issue(test)
+
+          if issue
+            puts "  => Found issue #{issue.web_url} for test '#{test.name}' with a diff ratio of #{(diff_ratio * 100).round(2)}%."
+          else
+            issue = create_issue(test)
+            puts "  => Created new issue: #{issue.web_url} for test '#{test.name}'." if issue
+          end
+
+          issue
+        end
+
+        def failure_issues(test)
+          gitlab.find_issues(options: { state: 'opened', labels: 'QA' }).select do |issue|
+            issue_title = issue.title.strip
+            issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
+          end
+        end
+
+        def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
+          ld = Class.new.extend(Gem::Text).method(:levenshtein_distance)
+          first_test_failure_stacktrace = test.failures.first['message_lines'].join("\n")
+
+          # Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
+          failure_issues(test).each_with_object({}) do |issue, memo|
+            relevant_issue_stacktrace = find_issue_stacktrace(issue)
+            next unless relevant_issue_stacktrace
+
+            distance = ld.call(first_test_failure_stacktrace, relevant_issue_stacktrace)
+            diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
+            if diff_ratio <= max_diff_ratio
+              puts "  => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
+              memo[issue] = diff_ratio
+            else
+              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%)."
+            end
+          end
+        end
+
+        def find_issue_stacktrace(issue)
+          issue_stacktrace_match = issue.description.match(STACKTRACE_REGEX)
+
+          if issue_stacktrace_match
+            issue_stacktrace_match[2].gsub(/^#.*$/, '').strip
+          else
+            puts "  => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
+          end
+        end
+
+        def find_failure_issue(test)
+          relevant_issues = find_relevant_failure_issues(test)
+
+          return nil if relevant_issues.empty?
+
+          best_matching_issue, smaller_diff_ratio = relevant_issues.min_by { |_, diff_ratio| diff_ratio }
+
+          unless relevant_issues.values.count(smaller_diff_ratio) == 1 # rubocop:disable Style/IfUnlessModifier
+            raise(MultipleIssuesFound, %(Too many issues found for test '#{test.name}' (`#{test.file}`)!))
+          end
+
+          test.failure_issue ||= best_matching_issue.web_url
+
+          [best_matching_issue, smaller_diff_ratio]
+        end
+
+        def new_issue_description(test)
+          super + [
+            "\n\n### Stack trace",
+            "```\n#{test.failures.first['message_lines'].join("\n")}\n```",
+            "First happened in #{test.ci_job_url}."
+          ].join("\n\n")
+        end
+
+        def deploy_environment_label
+          environment = Runtime::Env.deploy_environment
+
+          case environment
+          when 'production'
+            'found:gitlab.com'
+          when 'canary', 'staging'
+            "found:#{environment}.gitlab.com"
+          when 'preprod'
+            'found:pre.gitlab.com'
+          when 'staging-orchestrated', 'nightly', 'master'
+            "found:#{environment}"
+          else
+            raise "No `found:*` label for the `#{environment}` environment!"
+          end
+        end
+
+        def new_issue_labels(test)
+          NEW_ISSUE_LABELS + up_to_date_labels(test: test)
+        end
+
+        def up_to_date_labels(test:, issue: nil)
+          super << deploy_environment_label
+        end
+
+        def post_failed_job_note(issue, test)
+          gitlab.create_issue_note(iid: issue.iid, note: "/relate #{test.testcase}")
+        end
+
+        def new_issue_title(test)
+          "Failure in #{super}"
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/report_as_issue.rb

@@ -1,13 +1,17 @@
 # frozen_string_literal: true
 
+require 'set'
+
 module Gitlab
   module QA
     module Report
       class ReportAsIssue
-        def initialize(token:, input_files:, project: nil)
-          @gitlab = GitlabIssueClient.new(token: token, project: project)
-          @files = Array(input_files)
+        MAX_TITLE_LENGTH = 255
+
+        def initialize(token:, input_files:, project: nil, dry_run: false, **kwargs)
           @project = project
+          @gitlab = (dry_run ? GitlabIssueDryClient : GitlabIssueClient).new(token: token, project: project)
+          @files = Array(input_files)
         end
 
         def invoke!
@@ -24,6 +28,18 @@ module Gitlab
           raise NotImplementedError
         end
 
+        def new_issue_title(test)
+          "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
+        end
+
+        def new_issue_description(test)
+          "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}"
+        end
+
+        def new_issue_labels(test)
+          []
+        end
+
         def validate_input!
           assert_project!
           assert_input_files!(files)
@@ -41,6 +57,92 @@ module Gitlab
 
           abort "Please provide valid JUnit report files. No files were found matching `#{files.join(',')}`"
         end
+
+        def test_results_per_file
+          Dir.glob(files).each do |path|
+            extension = File.extname(path)
+
+            test_results =
+              case extension
+              when '.json'
+                Report::JsonTestResults.new(path)
+              when '.xml'
+                Report::JUnitTestResults.new(path)
+              else
+                raise "Unknown extension #{extension}"
+              end
+
+            yield test_results
+          end
+        end
+
+        def create_issue(test)
+          gitlab.create_issue(
+            title: title_from_test(test),
+            description: new_issue_description(test),
+            labels: new_issue_labels(test).to_a
+          )
+        end
+
+        def issue_labels(issue)
+          issue&.labels&.to_set || Set.new
+        end
+
+        def update_labels(issue, test)
+          new_labels = up_to_date_labels(test: test, issue: issue)
+
+          return if issue_labels(issue) == new_labels
+
+          gitlab.edit_issue(iid: issue.iid, options: { labels: new_labels.to_a })
+        end
+
+        def up_to_date_labels(test:, issue: nil)
+          labels = issue_labels(issue)
+          labels << "Enterprise Edition" if ee_test?(test)
+          quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
+
+          labels
+        end
+
+        def ee_test?(test)
+          test.file =~ %r{features/ee/(api|browser_ui)}
+        end
+
+        def quarantine_job?
+          Runtime::Env.ci_job_name&.include?('quarantine')
+        end
+
+        def partial_file_path(path)
+          path.match(/((api|browser_ui).*)/i)[1]
+        end
+
+        def title_from_test(test)
+          title = new_issue_title(test)
+
+          return title unless title.length > MAX_TITLE_LENGTH
+
+          "#{title[0...MAX_TITLE_LENGTH - 3]}..."
+        end
+
+        def search_safe(value)
+          value.delete('"')
+        end
+
+        def pipeline
+          # Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
+          #
+          # Tests can be run in several pipelines:
+          #   gitlab-qa, nightly, master, staging, canary, production, preprod, and MRs
+          #
+          # Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
+          #   nightly, staging, canary, production, and preprod
+          #
+          # MR, master, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on master
+          # because the other pipelines will be monitored by the author of the MR that triggered them.
+          # So we assume that we're reporting a master pipeline if the project name is 'gitlab-qa'.
+
+          @pipeline ||= Runtime::Env.pipeline_from_project_name
+        end
       end
     end
   end

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -8,37 +8,23 @@ module Gitlab
     module Report
       # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
       class ResultsInIssues < ReportAsIssue
-        MAX_TITLE_LENGTH = 255
-
         private
 
         def run!
           puts "Reporting test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
-          Dir.glob(files).each do |path|
-            puts "Reporting tests in #{path}"
-            extension = File.extname(path)
-
-            case extension
-            when '.json'
-              test_results = Report::JsonTestResults.new(path)
-            when '.xml'
-              test_results = Report::JUnitTestResults.new(path)
-            else
-              raise "Unknown extension #{extension}"
-            end
+          test_results_per_file do |test_results|
+            puts "Reporting tests in #{test_results.path}"
 
             test_results.each do |test|
               report_test(test)
             end
 
-            test_results.write(path)
+            test_results.write
           end
         end
 
         def report_test(test)
-          return if test.skipped
-
           puts "Reporting test: #{test.file} | #{test.name}"
 
           issue = find_issue(test)
@@ -46,16 +32,23 @@ module Gitlab
           if issue
             puts "Found existing issue: #{issue.web_url}"
           else
+            # Don't create new issues for skipped tests
+            return if test.skipped
+
             issue = create_issue(test)
             puts "Created new issue: #{issue.web_url}"
           end
 
           test.testcase ||= issue.web_url
 
-          update_labels(issue, test)
-          note_status(issue, test)
+          labels_updated = update_labels(issue, test)
+          note_posted = note_status(issue, test)
 
-          puts "Issue updated"
+          if labels_updated || note_posted
+            puts "Issue updated."
+          else
+            puts "Test passed, no update needed."
+          end
         end
 
         def find_issue(test)
@@ -72,12 +65,14 @@ module Gitlab
           issues.first
         end
 
-        def create_issue(test)
-          gitlab.create_issue(
-            title: title_from_test(test),
-            description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}",
-            labels: 'status::automated'
-          )
+        def new_issue_labels(test)
+          %w[status::automated]
+        end
+
+        def up_to_date_labels(test:, issue: nil)
+          labels = super
+          labels.delete_if { |label| label.start_with?("#{pipeline}::") }
+          labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
         end
 
         def iid_from_testcase_url(url)
@@ -85,37 +80,22 @@ module Gitlab
         end
 
         def search_term(test)
-          %("#{test.file}" "#{search_safe(test.name)}")
-        end
-
-        def title_from_test(test)
-          title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
-
-          return title unless title.length > MAX_TITLE_LENGTH
-
-          "#{title[0...MAX_TITLE_LENGTH - 3]}..."
-        end
-
-        def partial_file_path(path)
-          path.match(/((api|browser_ui).*)/i)[1]
-        end
-
-        def search_safe(value)
-          value.delete('"')
+          %("#{partial_file_path(test.file)}" "#{search_safe(test.name)}")
         end
 
         def note_status(issue, test)
-          return if test.failures.empty?
+          return false if test.skipped
+          return false if test.failures.empty?
 
           note = note_content(test)
 
-          gitlab.handle_gitlab_client_exceptions do
-            Gitlab.issue_discussions(project, issue.iid, order_by: 'created_at', sort: 'asc').each do |discussion|
-              return add_note_to_discussion(issue.iid, discussion.id) if new_note_matches_discussion?(note, discussion)
-            end
-
-            Gitlab.create_issue_note(project, issue.iid, note)
+          gitlab.find_issue_discussions(iid: issue.iid).each do |discussion|
+            return gitlab.add_note_to_issue_discussion_as_thread(iid: issue.iid, discussion_id: discussion.id, body: failure_summary) if new_note_matches_discussion?(note, discussion)
           end
+
+          gitlab.create_issue_note(iid: issue.iid, note: note)
+
+          true
         end
 
         def note_content(test)
@@ -143,10 +123,6 @@ module Gitlab
           summary.join(' ')
         end
 
-        def quarantine_job?
-          Runtime::Env.ci_job_name&.include?('quarantine')
-        end
-
         def new_note_matches_discussion?(note, discussion)
           note_error = error_and_stack_trace(note)
           discussion_error = error_and_stack_trace(discussion.notes.first['body'])
@@ -157,47 +133,7 @@ module Gitlab
         end
 
         def error_and_stack_trace(text)
-          result = text.strip[/Error:(.*)/m, 1].to_s
-
-          warn "Could not find `Error:` in text: #{text}" if result.empty?
-
-          result
-        end
-
-        def add_note_to_discussion(issue_iid, discussion_id)
-          gitlab.handle_gitlab_client_exceptions do
-            Gitlab.add_note_to_issue_discussion_as_thread(project, issue_iid, discussion_id, body: failure_summary)
-          end
-        end
-
-        def update_labels(issue, test)
-          labels = issue.labels
-          labels.delete_if { |label| label.start_with?("#{pipeline}::") }
-          labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
-          labels << "Enterprise Edition" if ee_test?(test)
-          quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
-
-          gitlab.edit_issue(iid: issue.iid, options: { labels: labels })
-        end
-
-        def ee_test?(test)
-          test.file =~ %r{features/ee/(api|browser_ui)}
-        end
-
-        def pipeline
-          # Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
-          #
-          # Tests can be run in several pipelines:
-          #   gitlab-qa, nightly, master, staging, canary, production, preprod, and MRs
-          #
-          # Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
-          #   nightly, staging, canary, production, and preprod
-          #
-          # MR, master, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on master
-          # because the other pipelines will be monitored by the author of the MR that triggered them.
-          # So we assume that we're reporting a master pipeline if the project name is 'gitlab-qa'.
-
-          Runtime::Env.pipeline_from_project_name
+          text.strip[/Error:(.*)/m, 1].to_s
         end
       end
     end