gitlab_quality-test_tooling 1.14.0 → 1.17.0

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -13,11 +13,19 @@ module GitlabQuality
       #   - Find issue by test hash
       #   - Reopen issue if it already exists, but is closed
       class FlakyTestIssue < ReportAsIssue
+        include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports
 
-        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', 'failure::flaky-test']).freeze
-        FOUND_IN_MR_LABEL     = 'found:in MR'
-        FOUND_IN_MASTER_LABEL = 'found:master'
+        IDENTITY_LABELS       = ['test', 'failure::flaky-test', 'automation:bot-authored'].freeze
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
+        SEARCH_LABELS         = ['test'].freeze
+        FOUND_IN_MR_LABEL     = '~"found:in MR"'
+        FOUND_IN_MASTER_LABEL = '~"found:master"'
+        REPORT_SECTION_HEADER = '### Flakiness reports'
+        REPORTS_DOCUMENTATION = <<~DOC
+          Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html)
+          to learn more about how to reproduce them.
+        DOC
 
         def initialize(
           token:,
@@ -25,18 +33,16 @@ module GitlabQuality
           base_issue_labels: nil,
           confidential: false,
           dry_run: false,
-          merge_request_iid: nil,
           project: nil,
           **_kwargs)
           super(token: token, input_files: input_files, project: project, confidential: confidential, dry_run: dry_run)
 
           @base_issue_labels = Set.new(base_issue_labels)
-          @merge_request_iid = merge_request_iid
         end
 
         private
 
-        attr_reader :base_issue_labels, :merge_request_iid
+        attr_reader :base_issue_labels
 
         def run!
           puts "Reporting flaky tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
@@ -44,62 +50,91 @@ module GitlabQuality
           TestResults::Builder.new(files).test_results_per_file do |test_results|
             puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
 
-            test_results.each do |test|
-              next if test.status != 'passed' # We only want failed tests that passed in the end
-
-              create_flaky_issue(test)
-            end
+            process_test_results(test_results)
           end
         end
 
-        def new_issue_title(test)
-          "Flaky test in #{super}"
-        end
+        def process_test_results(test_results)
+          test_results.each do |test|
+            next unless test_is_applicable?(test)
 
-        def new_issue_labels(_test)
-          found_label =
-            if !merge_request_iid || merge_request_iid.empty?
-              FOUND_IN_MASTER_LABEL
-            else
-              FOUND_IN_MR_LABEL
-            end
+            puts " => Reporting flakiness for test '#{test.name}'..."
 
-          NEW_ISSUE_LABELS + base_issue_labels + [found_label]
+            issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: SEARCH_LABELS)
+            issues << create_issue(test) if issues.empty?
+
+            update_reports(issues, test)
+            collect_issues(test, issues)
+          end
         end
 
-        def new_issue_description(test)
-          super + [
-            "Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html) " \
-            "to learn more about how to reproduce them.",
-            initial_reports_section(test)
-          ].compact.join("\n\n")
+        def test_is_applicable?(test)
+          test.status == 'passed' # We only want failed tests that passed in the end
         end
 
-        def create_flaky_issue(test)
-          puts "   => Finding existing issues for flaky test '#{test.name}' (run time: #{test.run_time} seconds)..."
+        def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
+          (base_issue_labels + super).to_a
+        end
 
-          issues = find_issues_by_hash(test_hash(test))
+        def update_reports(issues, test)
           issues.each do |issue|
-            puts "   => Existing issue link #{issue.web_url}."
+            puts "   => Adding the flaky test to the existing issue: #{issue.web_url}"
+            add_report_to_issue(issue: issue, test: test, related_issues: (issues - [issue]))
+          end
+        end
 
-            puts "   => Adding the flaky test to the existing issue..."
-            add_report_to_issue(issue, test)
+        def add_report_to_issue(issue:, test:, related_issues:)
+          reports_note = existing_reports_note(issue: issue)
+          note_body = [
+            report_body(reports_note: reports_note, test: test),
+            identity_labels_quick_action,
+            relate_issues_quick_actions(related_issues)
+          ].join("\n")
+
+          if reports_note
+            gitlab.edit_issue_note(
+              issue_iid: issue.iid,
+              note_id: reports_note.id,
+              note: note_body
+            )
+          else
+            gitlab.create_issue_note(iid: issue.iid, note: note_body)
+          end
+        end
 
-            if issue.state == 'closed'
-              puts "   => Issue is closed. Reopening it."
-              reopen_issue(issue)
-            end
+        def existing_reports_note(issue:)
+          gitlab.find_issue_notes(iid: issue.iid).find do |note|
+            note.body.start_with?(REPORT_SECTION_HEADER)
           end
+        end
 
-          create_issue(test) unless issues.any?
+        def report_body(reports_note:, test:)
+          increment_reports(
+            current_reports_content: reports_note&.body.to_s,
+            test: test,
+            reports_section_header: REPORT_SECTION_HEADER,
+            item_extra_content: found_label,
+            reports_extra_content: REPORTS_DOCUMENTATION
+          )
+        end
+
+        def found_label
+          if ENV.key?('CI_MERGE_REQUEST_IID')
+            FOUND_IN_MR_LABEL
+          else
+            FOUND_IN_MASTER_LABEL
+          end
         end
 
-        def add_report_to_issue(issue, test)
-          gitlab.edit_issue(iid: issue.iid, options: { description: add_report_to_issue_description(issue, test) })
+        def identity_labels_quick_action
+          labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
+          %(/label #{labels_list})
         end
 
-        def reopen_issue(issue)
-          gitlab.edit_issue(iid: issue.iid, options: { state_event: 'reopen' })
+        def relate_issues_quick_actions(issues)
+          issues.map do |issue|
+            "/relate #{issue.web_url}"
+          end.join("\n")
         end
       end
     end

data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb

@@ -205,12 +205,9 @@ module GitlabQuality
 
         def generate_test_text(testcase, tests_with_same_testcase, passed)
           text = tests_with_same_testcase.map(&:name).uniq.join(', ')
-          encoded_text = ERB::Util.url_encode(text)
 
           if testcase && !passed
-            # Workaround for reducing system notes on testcase issues
-            # The first regex extracts the link to the issues list page from a link to a single issue show page by removing the issue id.
-            "[#{text}](#{testcase.match(%r{[\s\S]+/[^/\d]+})}?state=opened&search=#{encoded_text})"
+            "[#{text}](#{testcase})"
           else
             text
           end

data/lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb

@@ -99,7 +99,6 @@ module GitlabQuality
         end
 
         def update_issue(issue:, spec_run_time:)
-          state_event = issue.state == 'closed' ? 'reopen' : nil
           updated_description = <<~MARKDOWN.chomp
           #{issue.description}
 
@@ -110,8 +109,6 @@ module GitlabQuality
             description: updated_description
           }
 
-          issue_attrs[:state_event] = state_event if state_event
-
           gitlab.edit_issue(iid: issue.iid, options: issue_attrs)
           puts "  => Added a report in #{issue.web_url}!"
         end

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -190,8 +190,8 @@ module GitlabQuality
         end
 
         def failure_issues(test)
-          find_issues_for_test(
-            test,
+          find_issues_by_hash(
+            test_hash(test),
             state: 'opened',
             labels: base_issue_labels + Set.new(%w[test]),
             not_labels: exclude_labels_for_search
@@ -268,7 +268,7 @@ module GitlabQuality
           if stacktrace_match
             stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip
           else
-            puts "  => [DEBUG] Stacktrace doesn't match the regex (#{regex}):\n----------------\n#{stacktrace}\n----------------\n"
+            puts "  => [DEBUG] Stacktrace doesn't match the regex (#{regex})!"
           end
         end
 
@@ -347,7 +347,7 @@ module GitlabQuality
           state_event = issue.state == 'closed' ? 'reopen' : nil
 
           issue_attrs = {
-            description: add_report_to_issue_description(issue, test),
+            description: increment_reports(current_reports_content: issue.description, test: test),
             labels: up_to_date_labels(test: test, issue: issue)
           }
           issue_attrs[:state_event] = state_event if state_event
@@ -356,10 +356,6 @@ module GitlabQuality
           puts "  => Added a report in '#{issue.title}': #{issue.web_url}!"
         end
 
-        def new_issue_title(test)
-          "Failure in #{super}"
-        end
-
         def screenshot_section(test)
           return unless test.screenshot?
 

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -60,7 +60,7 @@ module GitlabQuality
           | Description | `#{test.name}` |
           | Test level | #{test.level} |
           | Hash | `#{test_hash(test)}` |
-          | Duration | #{test.run_time} seconds |
+          | Reference duration | #{test.run_time} seconds |
           | Expected duration | < #{test.max_duration_for_test} seconds |
           #{"| Test case | #{test.testcase} |" if test.testcase}
           DESCRIPTION
@@ -142,8 +142,10 @@ module GitlabQuality
           labels
         end
 
-        def find_issues_by_hash(test_hash)
-          search_options = { search: test_hash }
+        def find_issues_by_hash(test_hash, labels: Set.new, not_labels: Set.new, state: nil)
+          search_options = { search: test_hash, labels: labels.to_a, not: { labels: not_labels.to_a } }
+          search_options[:state] = state if state
+          search_options[:in] = 'description'
           gitlab.find_issues(options: search_options)
         end
 

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -10,16 +10,21 @@ module GitlabQuality
       # - Find issue by title (with test description or test file)
       # - Add test metadata, duration to the issue with group and category labels
       class SlowTestIssue < ReportAsIssue
-        include TestTooling::Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
+        include Concerns::IssueReports
 
-        NEW_ISSUE_LABELS = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'rspec profiling', 'rspec:slow test']).freeze
-        SEARCH_LABELS = %w[test maintenance::performance].freeze
+        IDENTITY_LABELS       = ['test', 'rspec:slow test', 'rspec profiling', 'automation:bot-authored'].freeze
+        NEW_ISSUE_LABELS      = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3']).freeze
+        SEARCH_LABELS         = ['test'].freeze
+        FOUND_IN_MR_LABEL     = '~"found:in MR"'
+        FOUND_IN_MASTER_LABEL = '~"found:master"'
+        REPORT_SECTION_HEADER = '### Slowness reports'
+        REPORTS_DOCUMENTATION = <<~DOC
+          Slow tests were detected, please see the [test speed best practices guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)
+          to improve them. More context available about this issue in the [top slow tests guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#top-slow-tests).
 
-        JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
-        REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
-
-        MultipleIssuesFound = Class.new(StandardError)
+          Add `allowed_to_be_slow: true` to the RSpec test if this is a legit slow test and close the issue.
+        DOC
 
         private
 
@@ -29,99 +34,87 @@ module GitlabQuality
           TestResults::Builder.new(files).test_results_per_file do |test_results|
             puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
 
-            test_results.each do |test|
-              create_slow_issue(test) if test.slow_test?
-            end
+            process_test_results(test_results)
           end
         end
 
-        def new_issue_title(test)
-          "Slow test in #{super}"
-        end
+        def process_test_results(test_results)
+          test_results.each do |test|
+            next unless test.slow_test?
 
-        def new_issue_description(test)
-          super +
-            <<~DESCRIPTION.chomp
-              Slow tests were detected, please see the [test speed best practices guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)
-              to improve them. More context available about this issue in the [top slow tests guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#top-slow-tests).
+            puts " => Reporting slowness for test '#{test.name}'..."
 
-              Add `allowed_to_be_slow: true` to the RSpec test if this is a legit slow test and close the issue.
+            issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: SEARCH_LABELS)
+            issues << create_issue(test) if issues.empty?
 
-              #{reports_section(test)}
-            DESCRIPTION
+            update_reports(issues, test)
+            collect_issues(test, issues)
+          end
         end
 
-        def reports_section(test)
-          <<~REPORTS
-          ### Reports (1)
-
-          #{report_list_item(test)}
-          REPORTS
+        def test_is_applicable?(test)
+          test.slow_test?
         end
 
-        def report_list_item(test)
-          "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
+        def update_reports(issues, test)
+          issues.each do |issue|
+            puts "   => Adding the slow test to the existing issue: #{issue.web_url}"
+            add_report_to_issue(issue: issue, test: test, related_issues: (issues - [issue]))
+          end
         end
 
-        def slow_test_issues(test)
-          find_issues_for_test(
-            test,
-            state: 'opened',
-            labels: SEARCH_LABELS
-          )
+        def add_report_to_issue(issue:, test:, related_issues:)
+          reports_note = existing_reports_note(issue: issue)
+          note_body = [
+            report_body(reports_note: reports_note, test: test),
+            identity_labels_quick_action,
+            relate_issues_quick_actions(related_issues)
+          ].join("\n")
+
+          if reports_note
+            gitlab.edit_issue_note(
+              issue_iid: issue.iid,
+              note_id: reports_note.id,
+              note: note_body
+            )
+          else
+            gitlab.create_issue_note(iid: issue.iid, note: note_body)
+          end
         end
 
-        def create_slow_issue(test)
-          puts "   => Finding existing issues for slow test '#{test.name}' (run time: #{test.run_time} seconds)..."
+        def existing_reports_note(issue:)
+          gitlab.find_issue_notes(iid: issue.iid).find do |note|
+            note.body.start_with?(REPORT_SECTION_HEADER)
+          end
+        end
 
-          issues = slow_test_issues(test)
+        def report_body(reports_note:, test:)
+          increment_reports(
+            current_reports_content: reports_note&.body.to_s,
+            test: test,
+            reports_section_header: REPORT_SECTION_HEADER,
+            item_extra_content: found_label,
+            reports_extra_content: REPORTS_DOCUMENTATION
+          )
+        end
 
-          if issues.blank?
-            issues << create_issue(test)
+        def found_label
+          if ENV.key?('CI_MERGE_REQUEST_IID')
+            FOUND_IN_MR_LABEL
           else
-            issues.each do |issue|
-              puts "   => Existing issue link #{issue['web_url']}"
-
-              update_reports(issue, test)
-            end
+            FOUND_IN_MASTER_LABEL
           end
-
-          collect_issues(test, issues)
-        rescue MultipleIssuesFound => e
-          warn(e.message)
         end
 
-        def update_reports(issue, test)
-          # We reopen closed issues to not lose any history
-          state_event = issue.state == 'closed' ? 'reopen' : nil
-
-          issue_attrs = {
-            description: up_to_date_issue_description(issue.description, test)
-          }
-
-          issue_attrs[:state_event] = state_event if state_event
-
-          gitlab.edit_issue(iid: issue.iid, options: issue_attrs)
-          puts "  => Added a report in '#{issue.title}': #{issue.web_url}!"
+        def identity_labels_quick_action
+          labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
+          %(/label #{labels_list})
         end
 
-        def up_to_date_issue_description(issue_description, test)
-          new_issue_description =
-            if issue_description.include?('### Reports')
-              # We count the number of existing reports.
-              reports_count = issue_description
-                .scan(REPORT_ITEM_REGEX)
-                .size.to_i + 1
-              issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
-            else # For issue with the legacy format, we add the Reports section
-              reports_count = issue_description
-                .scan(JOB_URL_REGEX)
-                .size.to_i + 1
-
-              "#{issue_description}\n\n### Reports (#{reports_count})"
-            end
-
-          "#{new_issue_description}\n#{report_list_item(test)}"
+        def relate_issues_quick_actions(issues)
+          issues.map do |issue|
+            "/relate #{issue.web_url}"
+          end.join("\n")
         end
       end
     end

data/lib/gitlab_quality/test_tooling/runtime/env.rb

@@ -25,7 +25,7 @@ module GitlabQuality
 
         ENV_VARIABLES.each do |env_name, method_name|
           define_method(method_name) do
-            env_var_value_if_defined(env_name) || (instance_variable_get("@#{method_name}") if instance_variable_defined?("@#{method_name}"))
+            env_var_value_if_defined(env_name) || (instance_variable_get(:"@#{method_name}") if instance_variable_defined?(:"@#{method_name}"))
           end
         end
 
@@ -33,6 +33,10 @@ module GitlabQuality
           env_var_value_if_defined('QA_LOG_LEVEL')&.upcase || 'INFO'
         end
 
+        def gitlab_bot_username
+          env_var_value_if_defined('GITLAB_BOT_USERNAME') || 'gitlab-bot'
+        end
+
         def log_path
           env_var_value_if_defined('QA_LOG_PATH') || host_artifacts_dir
         end

data/lib/gitlab_quality/test_tooling/test_meta/processor/add_to_blocking_processor.rb

@@ -14,41 +14,48 @@ module GitlabQuality
             # @param [TestMetaUpdater] context instance of TestMetaUpdater
             def execute(spec, context) # rubocop:disable Metrics/AbcSize
               @context = context
-
+              @existing_mrs = nil
               @file_path = spec["file_path"]
+              testcase = spec["testcase"]
               devops_stage = spec["stage"]
               product_group = spec["product_group"]
               @example_name = spec["name"]
-              @mr_title = format("%{prefix} %{example_name}", prefix: '[PROMOTE TO BLOCKING]', example_name: example_name)
-
-              return unless proceed_with_merge_request?
+              @mr_title = format("%{prefix} %{example_name}", prefix: '[PROMOTE TO BLOCKING]', example_name: example_name).truncate(72, omission: '')
 
               @file_contents = context.get_file_contents(file_path)
 
-              new_content, changed_line_no = add_blocking_metadata
+              new_content, @changed_line_no = add_blocking_metadata
 
-              return if changed_line_no.negative?
+              return unless proceed_with_merge_request?
 
               branch = context.create_branch("blocking-promotion-#{SecureRandom.hex(4)}", example_name, context.ref)
 
               context.commit_changes(branch, <<~COMMIT_MESSAGE, file_path, new_content)
                 Promote end-to-end test to blocking
 
-                Promote to blocking: #{example_name}
+                #{"Promote to blocking: #{example_name}".truncate(72)}
               COMMIT_MESSAGE
 
-              assignee_id, assignee_handle = context.fetch_dri_id(product_group, devops_stage)
+              reviewer_id, assignee_handle = context.fetch_dri_id(product_group, devops_stage)
+
+              gitlab_bot_user_id = context.user_id_for_username(Runtime::Env.gitlab_bot_username)
 
-              merge_request = context.create_merge_request(mr_title, branch, assignee_id) do
+              merge_request = context.create_merge_request(mr_title, branch, gitlab_bot_user_id, [reviewer_id]) do
                 <<~MARKDOWN
                   ## What does this MR do?
 
                   Promotes the test [`#{example_name}`](https://gitlab.com/#{context.project}/-/blob/#{context.ref}/#{file_path}#L#{changed_line_no + 1})
                   to the blocking bucket
 
+                  This test was identified in the reliable e2e test report: #{context.report_issue}
+
+                  [Testcase link](#{testcase})
+
+                  [Spec metrics link](#{context.single_spec_metrics_link(example_name)})
 
                   /label ~"Quality" ~"QA" ~"type::maintenance"
                   /label ~"devops::#{devops_stage}"
+                  #{context.label_from_product_group(product_group)}
 
                   <div align="center">
                   (This MR was automatically generated by [`gitlab_quality-test_tooling`](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling) at #{Time.now.utc})
@@ -62,6 +69,11 @@ module GitlabQuality
                 If you think this MR should not be merged, please close it and add a note of the reason to the blocking report: #{context.report_issue}
               MARKDOWN
 
+              if merge_request
+                context.add_processed_record({ file_path => changed_line_no })
+                Runtime::Logger.info("  Created MR for promotion to blocking: #{merge_request.web_url}")
+              end
+
               merge_request
             end
 
@@ -82,15 +94,20 @@ module GitlabQuality
 
             private
 
-            attr_reader :context, :file_path, :file_contents, :example_name, :mr_title
+            attr_reader :context, :file_path, :file_contents, :example_name, :mr_title, :changed_line_no
 
             # Checks if there is already an MR open
             #
             # @return [Boolean]
-            def proceed_with_merge_request?
-              open_mrs = context.existing_merge_requests(title: mr_title)
-              if open_mrs.any?
-                puts "  An open MR already exists for '#{example_name}':  #{open_mrs.first['web_url']}. Will not proceed with creating MR."
+            def proceed_with_merge_request? # rubocop:disable Metrics/AbcSize
+              if changed_line_no.negative?
+                Runtime::Logger.info("  No lines were changed in #{file_path}. Will not proceed with creating MR.")
+                return false
+              elsif context.record_processed?(file_path, changed_line_no)
+                Runtime::Logger.info("  Record already processed for #{file_path}:#{changed_line_no}. Will not proceed with creating MR.")
+                return false
+              elsif existing_mrs&.any?
+                Runtime::Logger.info("  An open MR already exists for '#{example_name}':  #{existing_mrs.first['web_url']}. Will not proceed with creating MR.")
                 return false
               end
 
@@ -109,8 +126,8 @@ module GitlabQuality
               end
 
               context.update_matched_line(matched_lines.last, file_contents.dup) do |line|
-                if line.include?(',')
-                  line[line.index(',')] = format(BLOCKING_METADATA, suffix: ',')
+                if line.sub(DESCRIPTION_REGEX, '').include?(',')
+                  line[line.index(',', end_of_description_index(line))] = format(BLOCKING_METADATA, suffix: ',')
                 else
                   line[line.rindex(' ')] = format(BLOCKING_METADATA, suffix: ' ')
                 end