gitlab_quality-test_tooling 1.21.1 → 1.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3bd25aa86cb58d8b69f852509893b2d7f28a183474d6b4ac96732d8bd3efca23
-  data.tar.gz: f17357f3f9e1c97089ac675c918682341978a60ac40cace79557722c2ce4cdff
+  metadata.gz: 755ed1e2f1e0a7ac2e1d1264eeed686c76fb23c90d5eb6ebfbcaa64f931045f6
+  data.tar.gz: 5bbc1d0b7b93d8db268e2b0b0081b3878ff4117e97b2a3426023180cdec945ec
 SHA512:
-  metadata.gz: a2d8710d330b9dd79a54ff78de39e15e76b014ce70de89791991723a6a272222edda6a751bb0449ba1a820163713046e617c9d45d4fa6a895f0eb4f8d86889bc
-  data.tar.gz: 2eda2f409021c5d8f8c710a66294266fcda7a8ef46d3580ab680175718d8468c62b4fea6c375cfcb51cd1fe899f9c83918962f767b9c534916cfcb68fab1c676
+  metadata.gz: 3078229a51d390624b6e148444735f38581383319e377db26986d7d1b77d2b53447b440d1fc3eecb8eaf52e92a3cc0c927e4ab44930e34bfb83165e88b011097
+  data.tar.gz: a0ac07071de3a7fe50144f2527faff4a6e5c5530477681b5ebf1137fc1d89aa8991f2aa88ef255bd0ffa84c3a3d8440ab91e3b3b7ef4c28fed9ae3b11c7709e0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (1.21.1)
+    gitlab_quality-test_tooling (1.22.0)
       activesupport (>= 6.1, < 7.2)
       amatch (~> 0.4.1)
       gitlab (~> 4.19)
@@ -328,4 +328,4 @@ DEPENDENCIES
   webmock (= 3.7.0)
 
 BUNDLED WITH
-   2.5.6
+   2.5.4

data/README.md

@@ -156,6 +156,27 @@ Usage: exe/knapsack-report-issues [options]
     -h, --help                       Show the usage
 ```
 
+### `exe/failed-test-issues`
+
+```shell
+Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)
+Usage: exe/failed-test-issues [options]
+    -i, --input-files INPUT_FILES    RSpec report files (JSON or JUnit XML)
+    -p, --project PROJECT            Can be an integer or a group/project string
+    -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+        --max-diff-ratio MAX_DIFF_RATO
+                                     Max stacktrace diff ratio for failure issues detection
+    -r RELATED_ISSUES_FILE,          The file path for the related issues
+        --related-issues-file
+        --base-issue-labels BASE_ISSUE_LABELS
+                                     Labels to add to new failure issues
+        --exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH
+                                     Labels to exclude when searching for existing issues
+        --dry-run                    Perform a dry-run (don't create or update issues)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
 ### `exe/flaky-test-issues`
 
 ```shell

data/exe/failed-test-issues

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files INPUT_FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
+    params[:token] = token
+  end
+
+  opts.on('--max-diff-ratio MAX_DIFF_RATO', Float, 'Max stacktrace diff ratio for failure issues detection') do |max_diff_ratio|
+    params[:max_diff_ratio] = max_diff_ratio
+  end
+
+  opts.on('-r', '--related-issues-file RELATED_ISSUES_FILE', String, 'The file path for the related issues') do |related_issues_file|
+    params[:related_issues_file] = related_issues_file
+  end
+
+  opts.on('--base-issue-labels BASE_ISSUE_LABELS', String,
+    'Labels to add to new failure issues') do |base_issue_labels|
+    params[:base_issue_labels] = base_issue_labels.split(',')
+  end
+
+  opts.on('--exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH', String,
+    'Labels to exclude when searching for existing issues') do |exclude_labels_for_search|
+    params[:exclude_labels_for_search] = exclude_labels_for_search.split(',')
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create or update issues)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)"
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::FailedTestIssue.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb

@@ -13,6 +13,10 @@ module Gitlab
       get("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions", query: options)
     end
 
+    def create_issue_discussion(project, issue_iid, options = {})
+      post("/projects/#{url_encode(project)}/issues/#{issue_iid}/discussions", query: options)
+    end
+
     def add_note_to_issue_discussion_as_thread(project, issue_id, discussion_id, options = {})
       post("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions/#{discussion_id}/notes", query: options)
     end
@@ -93,9 +97,15 @@ module GitlabQuality
           end
         end
 
-        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
+        def create_issue_discussion(iid:, note:)
+          handle_gitlab_client_exceptions do
+            client.create_issue_discussion(project, iid, body: note)
+          end
+        end
+
+        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, note:)
           handle_gitlab_client_exceptions do
-            client.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: body)
+            client.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: note)
           end
         end
 

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb

@@ -23,8 +23,12 @@ module GitlabQuality
           puts "The following note would have been edited on #{project}##{issue_iid} (note #{note_id}) issue: #{note}"
         end
 
-        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
-          puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{body}"
+        def create_issue_discussion(iid:, note:)
+          puts "The following discussion would have been posted on #{project}##{iid} issue: #{note}"
+        end
+
+        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, note:)
+          puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{note}"
         end
 
         def upload_file(file_fullpath:)

data/lib/gitlab_quality/test_tooling/report/failed_test_issue.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require 'amatch'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      # Uses the API to create GitLab issues for any failed test coming from JSON test reports.
+      #
+      # - Takes the JSON test reports like rspec-*.json
+      # - Takes a project where failed test issues should be created
+      # - For every passed test in the report:
+      #   - Find issue by test hash or create a new issue if no issue was found
+      #   - Add a failure report in the "Failure reports" note
+      class FailedTestIssue < ReportAsIssue
+        include Concerns::GroupAndCategoryLabels
+        include Concerns::IssueReports
+        include Amatch
+
+        IDENTITY_LABELS       = ['test', 'automation:bot-authored'].freeze
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'failure::new', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
+        SEARCH_LABELS         = ['test'].freeze
+        FOUND_IN_MR_LABEL     = '~"found:in MR"'
+        FOUND_IN_MASTER_LABEL = '~"found:master"'
+        REPORTS_DISCUSSION_HEADER = '### Failure reports'
+        REPORT_SECTION_HEADER = '#### Failure reports'
+
+        IGNORED_FAILURES = [
+          'Net::ReadTimeout',
+          '403 Forbidden - Your account has been blocked'
+        ].freeze
+        FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
+        ISSUE_STACKTRACE_REGEX = /##### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*/m
+        DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
+
+        MultipleNotesFound = Class.new(StandardError)
+
+        def initialize(
+          token:,
+          input_files:,
+          base_issue_labels: nil,
+          dry_run: false,
+          project: nil,
+          max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION,
+          **_kwargs)
+          super(token: token, input_files: input_files, project: project, dry_run: dry_run)
+
+          @base_issue_labels = Set.new(base_issue_labels)
+          @max_diff_ratio = max_diff_ratio.to_f
+        end
+
+        private
+
+        attr_reader :base_issue_labels, :max_diff_ratio
+
+        def run!
+          puts "Reporting failed tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          TestResults::Builder.new(files).test_results_per_file do |test_results|
+            puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
+
+            process_test_results(test_results)
+          end
+        end
+
+        def process_test_results(test_results)
+          test_results.each do |test|
+            next unless test_is_applicable?(test)
+
+            puts " => Reporting failure for test '#{test.name}'..."
+
+            issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: SEARCH_LABELS)
+            issues << create_issue(test) if issues.empty?
+
+            update_reports(issues, test)
+            collect_issues(test, issues)
+          end
+        end
+
+        def test_is_applicable?(test)
+          test.status == 'failed'
+        end
+
+        def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
+          (base_issue_labels + super).to_a
+        end
+
+        def update_reports(issues, test)
+          issues.each do |issue|
+            puts "   => Adding the failed test to the existing issue: #{issue.web_url}"
+            add_report_to_issue(issue: issue, test: test, related_issues: (issues - [issue]))
+          end
+        end
+
+        def add_report_to_issue(issue:, test:, related_issues:)
+          reports_discussion = find_or_create_reports_discussion(issue: issue)
+          failure_discussion_note = find_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion)
+
+          note_body = [
+            report_body(reports_note: failure_discussion_note, test: test),
+            identity_labels_quick_action,
+            relate_issues_quick_actions(related_issues)
+          ].join("\n")
+
+          if failure_discussion_note
+            gitlab.edit_issue_note(
+              issue_iid: issue.iid,
+              note_id: failure_discussion_note.id,
+              note: note_body
+            )
+          else
+            gitlab.add_note_to_issue_discussion_as_thread(
+              iid: issue.iid,
+              discussion_id: reports_discussion.id,
+              note: note_body
+            )
+          end
+        rescue MultipleNotesFound => e
+          warn(e.message)
+        end
+
+        def find_or_create_reports_discussion(issue:)
+          reports_discussion = existing_reports_discussion(issue: issue)
+          return reports_discussion if reports_discussion
+
+          gitlab.create_issue_discussion(iid: issue.iid, note: REPORTS_DISCUSSION_HEADER)
+        end
+
+        def existing_reports_discussion(issue:)
+          gitlab.find_issue_discussions(iid: issue.iid).find do |discussion|
+            next if discussion.individual_note
+            next unless discussion.notes.first
+
+            discussion.notes.first.body.start_with?(REPORTS_DISCUSSION_HEADER)
+          end
+        end
+
+        def find_failure_discussion_note(issue:, test:, reports_discussion:)
+          return unless reports_discussion
+
+          relevant_notes = find_relevant_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion)
+          return if relevant_notes.empty?
+
+          best_matching_note, smaller_diff_ratio = relevant_notes.min_by { |_, diff_ratio| diff_ratio }
+
+          raise(MultipleNotesFound, %(Too many issues found for test '#{test.name}' (`#{test.file}`)!)) unless relevant_notes.values.count(smaller_diff_ratio) == 1
+
+          # Re-instantiate a `Gitlab::ObjectifiedHash` object after having converted it to a hash in #find_relevant_failure_issues above.
+          best_matching_note = Gitlab::ObjectifiedHash.new(best_matching_note)
+
+          test.failure_issue ||= "#{issue.web_url}#note_#{best_matching_note.id}"
+
+          best_matching_note
+        end
+
+        def find_relevant_failure_discussion_note(issue:, test:, reports_discussion:)
+          return [] unless reports_discussion.notes.size > 1
+
+          clean_test_stacktrace = cleaned_stack_trace_from_test(test: test)
+
+          reports_discussion.notes[1..].each_with_object({}) do |note, memo|
+            clean_note_stacktrace = cleaned_stack_trace_from_note(issue: issue, note: note)
+            diff_ratio = diff_ratio_between_test_and_note_stacktraces(
+              issue: issue,
+              note: note,
+              test_stacktrace: clean_test_stacktrace,
+              note_stacktrace: clean_note_stacktrace)
+
+            memo[note.to_h] = diff_ratio if diff_ratio
+          end
+        end
+
+        def cleaned_stack_trace_from_test(test:)
+          sanitize_stacktrace(stacktrace: full_stacktrace(test: test), regex: FAILURE_STACKTRACE_REGEX) || full_stacktrace(test: test)
+        end
+
+        def cleaned_stack_trace_from_note(issue:, note:)
+          note_stacktrace = sanitize_stacktrace(stacktrace: note.body, regex: ISSUE_STACKTRACE_REGEX)
+          return note_stacktrace if note_stacktrace
+
+          puts "  => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}#note_#{note.id}!"
+        end
+
+        def sanitize_stacktrace(stacktrace:, regex:)
+          stacktrace_match = stacktrace.match(regex)
+
+          if stacktrace_match
+            stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip
+          else
+            puts "  => [DEBUG] Stacktrace doesn't match the regex (#{regex})!"
+          end
+        end
+
+        def full_stacktrace(test:)
+          test.failures.each do |failure|
+            message = failure['message'] || ""
+            message_lines = failure['message_lines'] || []
+
+            next if IGNORED_FAILURES.any? { |e| message.include?(e) }
+
+            return message_lines.empty? ? message : message_lines.join("\n")
+          end
+        end
+
+        def diff_ratio_between_test_and_note_stacktraces(issue:, note:, test_stacktrace:, note_stacktrace:)
+          return if note_stacktrace.nil?
+
+          diff_ratio = compare_stack_traces(test_stacktrace, note_stacktrace)
+
+          if diff_ratio <= max_diff_ratio
+            puts "  => [DEBUG] Note #{issue.web_url}#note_#{note.id} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
+            # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
+            # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
+            # See:
+            # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
+            # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
+            diff_ratio
+          else
+            puts "  => [DEBUG] Found note #{issue.web_url}#note_#{note.id} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
+            puts "  => [DEBUG] Issue stacktrace:\n----------------\n#{note_stacktrace}\n----------------\n"
+            puts "  => [DEBUG] Failure stacktrace:\n----------------\n#{test_stacktrace}\n----------------\n"
+          end
+        end
+
+        def compare_stack_traces(stack_trace_first, stack_trace_second)
+          calculate_diff_ratio(stack_trace_first, stack_trace_second)
+        end
+
+        def calculate_diff_ratio(stack_trace_first, stack_trace_second)
+          distance = Levenshtein.new(stack_trace_first).match(stack_trace_second)
+          distance.zero? ? 0.0 : (distance.to_f / stack_trace_first.size).round(3)
+        end
+
+        def report_body(reports_note:, test:)
+          increment_reports(
+            current_reports_content: reports_note&.body.to_s,
+            test: test,
+            reports_section_header: REPORT_SECTION_HEADER,
+            item_extra_content: found_label,
+            reports_extra_content: "##### Stack trace\n\n```\n#{full_stacktrace(test: test)}\n```"
+          )
+        end
+
+        def found_label
+          if ENV.key?('CI_MERGE_REQUEST_IID')
+            FOUND_IN_MR_LABEL
+          else
+            FOUND_IN_MASTER_LABEL
+          end
+        end
+
+        def identity_labels_quick_action
+          labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
+          %(/label #{labels_list})
+        end
+
+        def relate_issues_quick_actions(issues)
+          issues.map do |issue|
+            "/relate #{issue.web_url}"
+          end.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -4,14 +4,13 @@ module GitlabQuality
   module TestTooling
     module Report
       # Uses the API to create GitLab issues for any passed test coming from JSON test reports.
-      # We expect the test reports to come from rspec-retry, or from a new RSpec process where
-      # we retried failing specs.
+      # We expect the test reports to come from a new RSpec process where we retried failing specs.
       #
-      # - Takes the JSON test reports like rspec-*.json (typically from rspec-retry gem)`
+      # - Takes the JSON test reports like rspec-*.json
       # - Takes a project where flaky test issues should be created
       # - For every passed test in the report:
-      #   - Find issue by test hash
-      #   - Reopen issue if it already exists, but is closed
+      #   - Find issue by test hash or create a new issue if no issue was found
+      #   - Add a flakiness report in the "Flakiness reports" note
       class FlakyTestIssue < ReportAsIssue
         include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -105,13 +105,10 @@ module GitlabQuality
             due_date: new_issue_due_date(test),
             confidential: confidential
           }.compact
-          issue = gitlab.create_issue(**attrs)
 
-          new_link = issue_type == 'test_case' ? issue&.web_url&.sub('/issues/', '/quality/test_cases/') : issue&.web_url
-
-          puts "Created new #{issue_type}: #{new_link}"
-
-          issue
+          gitlab.create_issue(**attrs).tap do |issue|
+            puts "Created new #{issue_type}: #{issue&.web_url}"
+          end
         end
 
         def issue_labels(issue)

data/lib/gitlab_quality/test_tooling/report/results_in_issues.rb

@@ -46,7 +46,7 @@ module GitlabQuality
             gitlab.add_note_to_issue_discussion_as_thread(
               iid: issue.iid,
               discussion_id: discussion.id,
-              body: failure_summary)
+              note: failure_summary)
             return true
           end
 

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "1.21.1"
+    VERSION = "1.22.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 1.21.1
+  version: 1.22.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-03 00:00:00.000000000 Z
+date: 2024-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -308,6 +308,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '4'
+- !ruby/object:Gem::Dependency
+  name: rspec-parameterized
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: table_print
   requirement: !ruby/object:Gem::Requirement
@@ -342,24 +356,11 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
-- !ruby/object:Gem::Dependency
-  name: rspec-parameterized
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
 description: A collection of test-related tools.
 email:
 - quality@gitlab.com
 executables:
+- failed-test-issues
 - flaky-test-issues
 - generate-test-session
 - knapsack-report-issues
@@ -388,6 +389,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/failed-test-issues
 - exe/flaky-test-issues
 - exe/generate-test-session
 - exe/knapsack-report-issues
@@ -421,6 +423,7 @@ files:
 - lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb
 - lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
 - lib/gitlab_quality/test_tooling/report/concerns/utils.rb
+- lib/gitlab_quality/test_tooling/report/failed_test_issue.rb
 - lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb
 - lib/gitlab_quality/test_tooling/report/generate_test_session.rb
 - lib/gitlab_quality/test_tooling/report/issue_logger.rb