gitlab_quality-test_tooling 1.21.1 → 1.23.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3bd25aa86cb58d8b69f852509893b2d7f28a183474d6b4ac96732d8bd3efca23
-  data.tar.gz: f17357f3f9e1c97089ac675c918682341978a60ac40cace79557722c2ce4cdff
+  metadata.gz: c63a7a25d0e3a05f515954353898bb8d0dcb9e7a90634087e738f5cb16174c57
+  data.tar.gz: 386833833dc0fd5cdc6334d3d613a38544fbc885c5ccb396070614f0e889555f
 SHA512:
-  metadata.gz: a2d8710d330b9dd79a54ff78de39e15e76b014ce70de89791991723a6a272222edda6a751bb0449ba1a820163713046e617c9d45d4fa6a895f0eb4f8d86889bc
-  data.tar.gz: 2eda2f409021c5d8f8c710a66294266fcda7a8ef46d3580ab680175718d8468c62b4fea6c375cfcb51cd1fe899f9c83918962f767b9c534916cfcb68fab1c676
+  metadata.gz: bb13a1c06a626b1152115262e0606480e78eb9aaafd6ed5e673cd72d6b1a268ab7659c020dbab743ebb872e71c1332905319d0ed6970cbde8bf0168c208fa2c3
+  data.tar.gz: 40652c63561438f37f5610e5b22f93c6a0404c53d4dc65b17713bbf5469bf769dd13691361f02f5aa8229aab706f7e349db6abf663cf45745aaa040456461db7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (1.21.1)
+    gitlab_quality-test_tooling (1.23.0)
       activesupport (>= 6.1, < 7.2)
       amatch (~> 0.4.1)
       gitlab (~> 4.19)
@@ -328,4 +328,4 @@ DEPENDENCIES
   webmock (= 3.7.0)
 
 BUNDLED WITH
-   2.5.6
+   2.5.4

data/README.md

@@ -156,6 +156,27 @@ Usage: exe/knapsack-report-issues [options]
     -h, --help                       Show the usage
 ```
 
+### `exe/failed-test-issues`
+
+```shell
+Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)
+Usage: exe/failed-test-issues [options]
+    -i, --input-files INPUT_FILES    RSpec report files (JSON or JUnit XML)
+    -p, --project PROJECT            Can be an integer or a group/project string
+    -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+        --max-diff-ratio MAX_DIFF_RATO
+                                     Max stacktrace diff ratio for failure issues detection
+    -r RELATED_ISSUES_FILE,          The file path for the related issues
+        --related-issues-file
+        --base-issue-labels BASE_ISSUE_LABELS
+                                     Labels to add to new failure issues
+        --exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH
+                                     Labels to exclude when searching for existing issues
+        --dry-run                    Perform a dry-run (don't create or update issues)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
 ### `exe/flaky-test-issues`
 
 ```shell

data/exe/failed-test-issues

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files INPUT_FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
+    params[:token] = token
+  end
+
+  opts.on('--max-diff-ratio MAX_DIFF_RATO', Float, 'Max stacktrace diff ratio for failure issues detection') do |max_diff_ratio|
+    params[:max_diff_ratio] = max_diff_ratio
+  end
+
+  opts.on('-r', '--related-issues-file RELATED_ISSUES_FILE', String, 'The file path for the related issues') do |related_issues_file|
+    params[:related_issues_file] = related_issues_file
+  end
+
+  opts.on('--base-issue-labels BASE_ISSUE_LABELS', String,
+    'Labels to add to new failure issues') do |base_issue_labels|
+    params[:base_issue_labels] = base_issue_labels.split(',')
+  end
+
+  opts.on('--exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH', String,
+    'Labels to exclude when searching for existing issues') do |exclude_labels_for_search|
+    params[:exclude_labels_for_search] = exclude_labels_for_search.split(',')
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create or update issues)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)"
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::FailedTestIssue.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb

@@ -13,6 +13,10 @@ module Gitlab
       get("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions", query: options)
     end
 
+    def create_issue_discussion(project, issue_iid, options = {})
+      post("/projects/#{url_encode(project)}/issues/#{issue_iid}/discussions", query: options)
+    end
+
     def add_note_to_issue_discussion_as_thread(project, issue_id, discussion_id, options = {})
       post("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions/#{discussion_id}/notes", query: options)
     end
@@ -93,9 +97,15 @@ module GitlabQuality
           end
         end
 
-        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
+        def create_issue_discussion(iid:, note:)
+          handle_gitlab_client_exceptions do
+            client.create_issue_discussion(project, iid, body: note)
+          end
+        end
+
+        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, note:)
           handle_gitlab_client_exceptions do
-            client.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: body)
+            client.add_note_to_issue_discussion_as_thread(project, iid, discussion_id, body: note)
           end
         end
 

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb

@@ -23,8 +23,12 @@ module GitlabQuality
           puts "The following note would have been edited on #{project}##{issue_iid} (note #{note_id}) issue: #{note}"
         end
 
-        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, body:)
-          puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{body}"
+        def create_issue_discussion(iid:, note:)
+          puts "The following discussion would have been posted on #{project}##{iid} issue: #{note}"
+        end
+
+        def add_note_to_issue_discussion_as_thread(iid:, discussion_id:, note:)
+          puts "The following discussion note would have been posted on #{project}##{iid} (discussion #{discussion_id}) issue: #{note}"
         end
 
         def upload_file(file_fullpath:)

data/lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb

@@ -12,11 +12,57 @@ module GitlabQuality
           REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>\S+)\)/
           LATEST_REPORTS_TO_SHOW = 10
 
+          class ReportsList
+            def initialize(preserved_content:, section_header:, reports:, extra_content:)
+              @preserved_content = preserved_content
+              @section_header = section_header
+              @reports = reports
+              @extra_content = extra_content
+            end
+
+            def self.report_list_item(test, item_extra_content: nil)
+              "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')}) #{item_extra_content}".strip
+            end
+
+            def reports_count
+              reports.size
+            end
+
+            def to_s
+              [
+                preserved_content,
+                "#{section_header} (#{reports_count})",
+                reports_list(reports),
+                extra_content
+              ].reject(&:blank?).compact.join("\n\n")
+            end
+
+            private
+
+            attr_reader :preserved_content, :section_header, :reports, :extra_content
+
+            def reports_list(reports)
+              sorted_reports = reports.sort.reverse
+
+              if sorted_reports.size > LATEST_REPORTS_TO_SHOW
+                [
+                  "Last 10 reports:",
+                  sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
+                  "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
+                  sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
+                  "</details>"
+                ].join("\n\n")
+              else
+                sorted_reports.join("\n")
+              end
+            end
+          end
+
           def initial_reports_section(test)
             <<~REPORTS
             ### Reports (1)
 
-            #{report_list_item(test)}
+            #{ReportsList.report_list_item(test)}
             REPORTS
           end
 
@@ -27,14 +73,14 @@ module GitlabQuality
             item_extra_content: nil,
             reports_extra_content: nil)
             preserved_content = current_reports_content.split(reports_section_header).first&.strip
-            reports = report_lines(current_reports_content) + [report_list_item(test, item_extra_content: item_extra_content)]
-
-            [
-              preserved_content,
-              "#{reports_section_header} (#{reports.size})",
-              reports_list(reports),
-              reports_extra_content
-            ].reject(&:blank?).compact.join("\n\n")
+            reports = report_lines(current_reports_content) + [ReportsList.report_list_item(test, item_extra_content: item_extra_content)]
+
+            ReportsList.new(
+              preserved_content: preserved_content,
+              section_header: reports_section_header,
+              reports: reports,
+              extra_content: reports_extra_content
+            )
           end
 
           def failed_issue_job_url(issue)
@@ -55,40 +101,12 @@ module GitlabQuality
             content.lines.grep(REPORT_ITEM_REGEX).map(&:strip)
           end
 
-          def reports_list(reports)
-            sorted_reports = reports.sort.reverse
-
-            if sorted_reports.size > LATEST_REPORTS_TO_SHOW
-              [
-                "Last 10 reports:",
-                sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
-                "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
-                sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
-                "</details>"
-              ].join("\n\n")
-            else
-              sorted_reports.join("\n")
-            end
-          end
-
-          def report_list_item(test, item_extra_content: nil)
-            "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')}) #{item_extra_content}".strip
-          end
-
           def job_urls_from_description(issue_description, regex)
             issue_description.lines.filter_map do |line|
               match = line.match(regex)
               match[:job_url] if match
             end
           end
-
-          def test_captures_to_report_items(test_captures)
-            test_captures.map do |ci_job_url, _, _|
-              report_list_item(GitlabQuality::TestTooling::TestResult::JsonTestResult.new(
-                'ci_job_url' => ci_job_url
-              ))
-            end
-          end
         end
       end
     end

data/lib/gitlab_quality/test_tooling/report/failed_test_issue.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      # Uses the API to create GitLab issues for any failed test coming from JSON test reports.
+      #
+      # - Takes the JSON test reports like rspec-*.json
+      # - Takes a project where failed test issues should be created
+      # - For every passed test in the report:
+      #   - Find issue by test hash or create a new issue if no issue was found
+      #   - Add a failure report in the "Failure reports" note
+      class FailedTestIssue < ReportAsIssue
+        include Concerns::GroupAndCategoryLabels
+        include Concerns::IssueReports
+
+        IDENTITY_LABELS       = ['test', 'automation:bot-authored'].freeze
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'failure::new', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
+        SEARCH_LABELS         = ['test'].freeze
+        FOUND_IN_MR_LABEL     = '~"found:in MR"'
+        FOUND_IN_MASTER_LABEL = '~"found:master"'
+        REPORTS_DISCUSSION_HEADER = '### Failure reports'
+        REPORT_SECTION_HEADER = '#### Failure reports'
+
+        FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
+        ISSUE_STACKTRACE_REGEX = /##### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*/m
+        DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
+
+        MultipleNotesFound = Class.new(StandardError)
+
+        def initialize(
+          token:,
+          input_files:,
+          base_issue_labels: nil,
+          dry_run: false,
+          project: nil,
+          max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION,
+          **_kwargs)
+          super(token: token, input_files: input_files, project: project, dry_run: dry_run)
+
+          @base_issue_labels = Set.new(base_issue_labels)
+          @max_diff_ratio = max_diff_ratio.to_f
+        end
+
+        private
+
+        attr_reader :base_issue_labels, :max_diff_ratio
+
+        def run!
+          puts "Reporting failed tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          TestResults::Builder.new(files).test_results_per_file do |test_results|
+            puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
+
+            process_test_results(test_results)
+          end
+        end
+
+        def process_test_results(test_results)
+          test_results.each do |test|
+            next unless test_is_applicable?(test)
+
+            puts " => Reporting failure for test '#{test.name}'..."
+
+            issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: SEARCH_LABELS)
+            issues << create_issue(test) if issues.empty?
+
+            update_reports(issues, test)
+            collect_issues(test, issues)
+          end
+        end
+
+        def test_is_applicable?(test)
+          test.status == 'failed'
+        end
+
+        def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
+          (base_issue_labels + super).to_a
+        end
+
+        def update_reports(issues, test)
+          issues.each do |issue|
+            puts "   => Adding the failed test to the existing issue: #{issue.web_url}"
+            add_report_to_issue(issue: issue, test: test, related_issues: (issues - [issue]))
+          end
+        end
+
+        def add_report_to_issue(issue:, test:, related_issues:) # rubocop:disable Metrics/AbcSize:
+          reports_discussion = find_or_create_reports_discussion(issue: issue)
+          current_reports_note = find_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion)
+          new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test)
+
+          note_body = [
+            new_reports_list.to_s,
+            identity_labels_quick_action,
+            relate_issues_quick_actions(related_issues)
+          ].join("\n")
+
+          if current_reports_note
+            gitlab.edit_issue_note(
+              issue_iid: issue.iid,
+              note_id: current_reports_note.id,
+              note: note_body
+            )
+          else
+            gitlab.add_note_to_issue_discussion_as_thread(
+              iid: issue.iid,
+              discussion_id: reports_discussion.id,
+              note: note_body
+            )
+          end
+        rescue MultipleNotesFound => e
+          warn(e.message)
+        end
+
+        def find_or_create_reports_discussion(issue:)
+          reports_discussion = existing_reports_discussion(issue: issue)
+          return reports_discussion if reports_discussion
+
+          gitlab.create_issue_discussion(iid: issue.iid, note: REPORTS_DISCUSSION_HEADER)
+        end
+
+        def existing_reports_discussion(issue:)
+          gitlab.find_issue_discussions(iid: issue.iid).find do |discussion|
+            next if discussion.individual_note
+            next unless discussion.notes.first
+
+            discussion.notes.first.body.start_with?(REPORTS_DISCUSSION_HEADER)
+          end
+        end
+
+        def find_failure_discussion_note(issue:, test:, reports_discussion:)
+          return unless reports_discussion
+
+          relevant_notes = find_relevant_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion)
+          return if relevant_notes.empty?
+
+          best_matching_note, smaller_diff_ratio = relevant_notes.min_by { |_, diff_ratio| diff_ratio }
+
+          raise(MultipleNotesFound, %(Too many issues found for test '#{test.name}' (`#{test.file}`)!)) unless relevant_notes.values.count(smaller_diff_ratio) == 1
+
+          # Re-instantiate a `Gitlab::ObjectifiedHash` object after having converted it to a hash in #find_relevant_failure_issues above.
+          best_matching_note = Gitlab::ObjectifiedHash.new(best_matching_note)
+
+          test.failure_issue ||= "#{issue.web_url}#note_#{best_matching_note.id}"
+
+          best_matching_note
+        end
+
+        def find_relevant_failure_discussion_note(issue:, test:, reports_discussion:)
+          return [] unless reports_discussion.notes.size > 1
+
+          clean_test_stacktrace = cleaned_stack_trace_from_test(test: test)
+
+          # We're skipping the first note of the discussion as this is the "non-collapsible note", aka
+          # the "header note", which doesn't contain any stack trace.
+          reports_discussion.notes[1..].each_with_object({}) do |note, memo|
+            clean_note_stacktrace = cleaned_stack_trace_from_note(issue: issue, note: note)
+            diff_ratio = diff_ratio_between_test_and_note_stacktraces(
+              issue: issue,
+              note: note,
+              test_stacktrace: clean_test_stacktrace,
+              note_stacktrace: clean_note_stacktrace)
+
+            memo[note.to_h] = diff_ratio if diff_ratio
+          end
+        end
+
+        def cleaned_stack_trace_from_test(test:)
+          sanitize_stacktrace(stacktrace: test.full_stacktrace, regex: FAILURE_STACKTRACE_REGEX) || test.full_stacktrace
+        end
+
+        def cleaned_stack_trace_from_note(issue:, note:)
+          note_stacktrace = sanitize_stacktrace(stacktrace: note.body, regex: ISSUE_STACKTRACE_REGEX)
+          return note_stacktrace if note_stacktrace
+
+          puts "  => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}#note_#{note.id}!"
+        end
+
+        def sanitize_stacktrace(stacktrace:, regex:)
+          stacktrace_match = stacktrace.match(regex)
+
+          if stacktrace_match
+            stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip
+          else
+            puts "  => [DEBUG] Stacktrace doesn't match the regex (#{regex})!"
+          end
+        end
+
+        def diff_ratio_between_test_and_note_stacktraces(issue:, note:, test_stacktrace:, note_stacktrace:)
+          return if note_stacktrace.nil?
+
+          stack_trace_comparator = StackTraceComparator.new(test_stacktrace, note_stacktrace)
+
+          if stack_trace_comparator.lower_or_equal_to_diff_ratio?(max_diff_ratio)
+            puts "  => [DEBUG] Note #{issue.web_url}#note_#{note.id} has an acceptable diff ratio of #{stack_trace_comparator.diff_percent}%."
+            # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
+            # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
+            # See:
+            # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
+            # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
+            stack_trace_comparator.diff_ratio
+          else
+            puts "  => [DEBUG] Found note #{issue.web_url}#note_#{note.id} but stacktraces are too different (#{stack_trace_comparator.diff_percent}%).\n"
+            puts "  => [DEBUG] Issue stacktrace:\n----------------\n#{note_stacktrace}\n----------------\n"
+            puts "  => [DEBUG] Failure stacktrace:\n----------------\n#{test_stacktrace}\n----------------\n"
+          end
+        end
+
+        def add_report_for_test(current_reports_content:, test:)
+          increment_reports(
+            current_reports_content: current_reports_content,
+            test: test,
+            reports_section_header: REPORT_SECTION_HEADER,
+            item_extra_content: found_label,
+            reports_extra_content: "##### Stack trace\n\n```\n#{test.full_stacktrace}\n```"
+          )
+        end
+
+        def found_label
+          if ENV.key?('CI_MERGE_REQUEST_IID')
+            FOUND_IN_MR_LABEL
+          else
+            FOUND_IN_MASTER_LABEL
+          end
+        end
+
+        def identity_labels_quick_action
+          labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
+          %(/label #{labels_list})
+        end
+
+        def relate_issues_quick_actions(issues)
+          issues.map do |issue|
+            "/relate #{issue.web_url}"
+          end.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -4,14 +4,13 @@ module GitlabQuality
   module TestTooling
     module Report
       # Uses the API to create GitLab issues for any passed test coming from JSON test reports.
-      # We expect the test reports to come from rspec-retry, or from a new RSpec process where
-      # we retried failing specs.
+      # We expect the test reports to come from a new RSpec process where we retried failing specs.
       #
-      # - Takes the JSON test reports like rspec-*.json (typically from rspec-retry gem)`
+      # - Takes the JSON test reports like rspec-*.json
       # - Takes a project where flaky test issues should be created
       # - For every passed test in the report:
-      #   - Find issue by test hash
-      #   - Reopen issue if it already exists, but is closed
+      #   - Find issue by test hash or create a new issue if no issue was found
+      #   - Add a flakiness report in the "Flakiness reports" note
       class FlakyTestIssue < ReportAsIssue
         include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports
@@ -84,17 +83,20 @@ module GitlabQuality
         end
 
         def add_report_to_issue(issue:, test:, related_issues:)
-          reports_note = existing_reports_note(issue: issue)
+          current_reports_note = existing_reports_note(issue: issue)
+          new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test)
+
           note_body = [
-            report_body(reports_note: reports_note, test: test),
+            new_reports_list.to_s,
+            flakiness_status_labels_quick_action(new_reports_list.reports_count),
             identity_labels_quick_action,
             relate_issues_quick_actions(related_issues)
           ].join("\n")
 
-          if reports_note
+          if current_reports_note
             gitlab.edit_issue_note(
               issue_iid: issue.iid,
-              note_id: reports_note.id,
+              note_id: current_reports_note.id,
               note: note_body
             )
           else
@@ -108,9 +110,9 @@ module GitlabQuality
           end
         end
 
-        def report_body(reports_note:, test:)
+        def add_report_for_test(current_reports_content:, test:)
           increment_reports(
-            current_reports_content: reports_note&.body.to_s,
+            current_reports_content: current_reports_content,
             test: test,
             reports_section_header: REPORT_SECTION_HEADER,
             item_extra_content: found_label,
@@ -126,6 +128,19 @@ module GitlabQuality
           end
         end
 
+        def flakiness_status_labels_quick_action(reports_count)
+          case reports_count
+          when 1000..Float::INFINITY
+            '/label ~"flakiness::1"'
+          when 500..999
+            '/label ~"flakiness::2"'
+          when 10..499
+            '/label ~"flakiness::3"'
+          else
+            '/label ~"flakiness::4"'
+          end
+        end
+
         def identity_labels_quick_action
           labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
           %(/label #{labels_list})

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -2,7 +2,6 @@
 
 require 'nokogiri'
 require 'rubygems/text'
-require 'amatch'
 
 module GitlabQuality
   module TestTooling
@@ -17,7 +16,6 @@ module GitlabQuality
         include TestTooling::Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports
-        include Amatch
 
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
         SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
@@ -26,10 +24,6 @@ module GitlabQuality
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*\n###/m
 
         NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
-        IGNORED_FAILURES = [
-          'Net::ReadTimeout',
-          '403 Forbidden - Your account has been blocked'
-        ].freeze
         SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
 
         MultipleIssuesFound = Class.new(StandardError)
@@ -168,7 +162,9 @@ module GitlabQuality
 
             next if pipeline != pipeline_env_from_job_url(job_url_from_issue)
 
-            compare_stack_traces(cleaned_stack_trace_from_test(test), cleaned_stack_trace_from_issue(issue)) < max_diff_ratio
+            stack_trace_comparator = StackTraceComparator.new(cleaned_stack_trace_from_test(test), cleaned_stack_trace_from_issue(issue))
+
+            stack_trace_comparator.lower_than_diff_ratio?(max_diff_ratio)
           end
         end
 
@@ -196,17 +192,6 @@ module GitlabQuality
           )
         end
 
-        def full_stacktrace(test)
-          test.failures.each do |failure|
-            message = failure['message'] || ""
-            message_lines = failure['message_lines'] || []
-
-            next if IGNORED_FAILURES.any? { |e| message.include?(e) }
-
-            return message_lines.empty? ? message : message_lines.join("\n")
-          end
-        end
-
         def cleaned_stack_trace_from_issue(issue)
           relevant_issue_stacktrace = find_issue_stacktrace(issue)
           return unless relevant_issue_stacktrace
@@ -215,20 +200,11 @@ module GitlabQuality
         end
 
         def cleaned_stack_trace_from_test(test)
-          test_failure_stacktrace = sanitize_stacktrace(full_stacktrace(test),
-            FAILURE_STACKTRACE_REGEX) || full_stacktrace(test)
+          test_failure_stacktrace = sanitize_stacktrace(test.full_stacktrace,
+            FAILURE_STACKTRACE_REGEX) || test.full_stacktrace
           remove_unique_resource_names(test_failure_stacktrace)
         end
 
-        def compare_stack_traces(stack_trace_first, stack_trace_second)
-          calculate_diff_ratio(stack_trace_first, stack_trace_second)
-        end
-
-        def calculate_diff_ratio(stack_trace_first, stack_trace_second)
-          distance = Levenshtein.new(stack_trace_first).match(stack_trace_second)
-          distance.zero? ? 0.0 : (distance.to_f / stack_trace_first.size).round(3)
-        end
-
         def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
           clean_first_test_failure_stacktrace = cleaned_stack_trace_from_test(test)
           # Search with the `search` param returns 500 errors, so we filter by `base_issue_labels` and then filter further in Ruby
@@ -236,17 +212,18 @@ module GitlabQuality
             clean_relevant_issue_stacktrace = cleaned_stack_trace_from_issue(issue)
             next if clean_relevant_issue_stacktrace.nil?
 
-            diff_ratio = compare_stack_traces(clean_first_test_failure_stacktrace, clean_relevant_issue_stacktrace)
-            if diff_ratio <= max_diff_ratio
-              puts "  => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
+            stack_trace_comparator = StackTraceComparator.new(clean_first_test_failure_stacktrace, clean_relevant_issue_stacktrace)
+
+            if stack_trace_comparator.lower_or_equal_to_diff_ratio?(max_diff_ratio)
+              puts "  => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{stack_trace_comparator.diff_percent}%."
               # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
               # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
               # See:
               # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
               # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
-              memo[issue.to_h] = diff_ratio
+              memo[issue.to_h] = stack_trace_comparator.diff_ratio
             else
-              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
+              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{stack_trace_comparator.diff_percent}%).\n"
               puts "  => [DEBUG] Issue stacktrace:\n----------------\n#{clean_relevant_issue_stacktrace}\n----------------\n"
               puts "  => [DEBUG] Failure stacktrace:\n----------------\n#{clean_first_test_failure_stacktrace}\n----------------\n"
             end
@@ -295,7 +272,7 @@ module GitlabQuality
         def new_issue_description(test)
           super + [
             "\n### Stack trace",
-            "```\n#{full_stacktrace(test)}\n```",
+            "```\n#{test.full_stacktrace}\n```",
             screenshot_section(test),
             system_log_errors_section(test),
             initial_reports_section(test)
@@ -345,7 +322,7 @@ module GitlabQuality
           state_event = issue.state == 'closed' ? 'reopen' : nil
 
           issue_attrs = {
-            description: increment_reports(current_reports_content: issue.description, test: test),
+            description: increment_reports(current_reports_content: issue.description, test: test).to_s,
             labels: up_to_date_labels(test: test, issue: issue)
           }
           issue_attrs[:state_event] = state_event if state_event
@@ -357,7 +334,7 @@ module GitlabQuality
         def screenshot_section(test)
           return unless test.screenshot?
 
-          failure = full_stacktrace(test)
+          failure = test.full_stacktrace
           return if SCREENSHOT_IGNORED_ERRORS.any? { |e| failure.include?(e) }
 
           relative_url = gitlab.upload_file(file_fullpath: test.screenshot_image)
@@ -374,7 +351,7 @@ module GitlabQuality
           return false unless test.failures?
 
           puts "  => Systemic failures detected: #{systemic_failure_messages}" if systemic_failure_messages.any?
-          failure_to_ignore = IGNORED_FAILURES + systemic_failure_messages
+          failure_to_ignore = TestResult::BaseTestResult::IGNORED_FAILURES + systemic_failure_messages
 
           reason = ignored_failure_reason(test.failures, failure_to_ignore)
 

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -105,13 +105,10 @@ module GitlabQuality
             due_date: new_issue_due_date(test),
             confidential: confidential
           }.compact
-          issue = gitlab.create_issue(**attrs)
 
-          new_link = issue_type == 'test_case' ? issue&.web_url&.sub('/issues/', '/quality/test_cases/') : issue&.web_url
-
-          puts "Created new #{issue_type}: #{new_link}"
-
-          issue
+          gitlab.create_issue(**attrs).tap do |issue|
+            puts "Created new #{issue_type}: #{issue&.web_url}"
+          end
         end
 
         def issue_labels(issue)

data/lib/gitlab_quality/test_tooling/report/results_in_issues.rb

@@ -46,7 +46,7 @@ module GitlabQuality
             gitlab.add_note_to_issue_discussion_as_thread(
               iid: issue.iid,
               discussion_id: discussion.id,
-              body: failure_summary)
+              note: failure_summary)
             return true
           end
 

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -64,17 +64,19 @@ module GitlabQuality
         end
 
         def add_report_to_issue(issue:, test:, related_issues:)
-          reports_note = existing_reports_note(issue: issue)
+          current_reports_note = existing_reports_note(issue: issue)
+          new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test)
+
           note_body = [
-            report_body(reports_note: reports_note, test: test),
+            new_reports_list.to_s,
             identity_labels_quick_action,
             relate_issues_quick_actions(related_issues)
           ].join("\n")
 
-          if reports_note
+          if current_reports_note
             gitlab.edit_issue_note(
               issue_iid: issue.iid,
-              note_id: reports_note.id,
+              note_id: current_reports_note.id,
               note: note_body
             )
           else
@@ -88,9 +90,9 @@ module GitlabQuality
           end
         end
 
-        def report_body(reports_note:, test:)
+        def add_report_for_test(current_reports_content:, test:)
           increment_reports(
-            current_reports_content: reports_note&.body.to_s,
+            current_reports_content: current_reports_content,
             test: test,
             reports_section_header: REPORT_SECTION_HEADER,
             item_extra_content: "(#{test.run_time} seconds) #{found_label}",

data/lib/gitlab_quality/test_tooling/stack_trace_comparator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'amatch'
+
+module GitlabQuality
+  module TestTooling
+    class StackTraceComparator
+      include Amatch
+
+      def initialize(first_trace, second_trace)
+        @first_trace = first_trace
+        @second_trace = second_trace
+      end
+
+      def diff_ratio
+        @diff_ratio ||= (1 - first_trace.levenshtein_similar(second_trace))
+      end
+
+      def diff_percent
+        (diff_ratio * 100).round(2)
+      end
+
+      def lower_than_diff_ratio?(max_diff_ratio)
+        diff_ratio < max_diff_ratio
+      end
+
+      def lower_or_equal_to_diff_ratio?(max_diff_ratio)
+        diff_ratio <= max_diff_ratio
+      end
+
+      private
+
+      attr_reader :first_trace, :second_trace
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

@@ -4,6 +4,11 @@ module GitlabQuality
   module TestTooling
     module TestResult
       class BaseTestResult
+        IGNORED_FAILURES = [
+          'Net::ReadTimeout',
+          '403 Forbidden - Your account has been blocked'
+        ].freeze
+
         attr_reader :report
 
         def initialize(report)
@@ -41,6 +46,17 @@ module GitlabQuality
         def failures?
           failures.any?
         end
+
+        def full_stacktrace
+          failures.each do |failure|
+            message = failure['message'] || ""
+            message_lines = failure['message_lines'] || []
+
+            next if IGNORED_FAILURES.any? { |e| message.include?(e) }
+
+            return message_lines.empty? ? message : message_lines.join("\n")
+          end
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "1.21.1"
+    VERSION = "1.23.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 1.21.1
+  version: 1.23.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-03 00:00:00.000000000 Z
+date: 2024-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -308,6 +308,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '4'
+- !ruby/object:Gem::Dependency
+  name: rspec-parameterized
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: table_print
   requirement: !ruby/object:Gem::Requirement
@@ -342,24 +356,11 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
-- !ruby/object:Gem::Dependency
-  name: rspec-parameterized
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.0.0
 description: A collection of test-related tools.
 email:
 - quality@gitlab.com
 executables:
+- failed-test-issues
 - flaky-test-issues
 - generate-test-session
 - knapsack-report-issues
@@ -388,6 +389,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/failed-test-issues
 - exe/flaky-test-issues
 - exe/generate-test-session
 - exe/knapsack-report-issues
@@ -421,6 +423,7 @@ files:
 - lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb
 - lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
 - lib/gitlab_quality/test_tooling/report/concerns/utils.rb
+- lib/gitlab_quality/test_tooling/report/failed_test_issue.rb
 - lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb
 - lib/gitlab_quality/test_tooling/report/generate_test_session.rb
 - lib/gitlab_quality/test_tooling/report/issue_logger.rb
@@ -438,6 +441,7 @@ files:
 - lib/gitlab_quality/test_tooling/runtime/logger.rb
 - lib/gitlab_quality/test_tooling/slack/post_to_slack.rb
 - lib/gitlab_quality/test_tooling/slack/post_to_slack_dry.rb
+- lib/gitlab_quality/test_tooling/stack_trace_comparator.rb
 - lib/gitlab_quality/test_tooling/summary_table.rb
 - lib/gitlab_quality/test_tooling/support/http_request.rb
 - lib/gitlab_quality/test_tooling/system_logs/finders/json_log_finder.rb