gitlab_quality-test_tooling 2.16.0 → 2.25.1

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -3,6 +3,10 @@
 require 'nokogiri'
 require 'rubygems/text'
 
+require_relative 'group_issues/error_pattern_matcher'
+require_relative 'group_issues/error_message_normalizer'
+require_relative 'group_issues/group_results_in_issues'
+
 module GitlabQuality
   module TestTooling
     module Report
@@ -12,6 +16,7 @@ module GitlabQuality
       # - Takes a project where failure issues should be created
       # - Find issue by title (with test description or test file), then further filter by stack trace, then pick the better-matching one
       # - Add the failed job to the issue description, and update labels
+      # - Can group similar failures together when group_similar option is enabled
       class RelateFailureIssue < ReportAsIssue
         include TestTooling::Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
@@ -23,7 +28,7 @@ module GitlabQuality
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*\n###/m
 
-        NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2 automation:bot-authored type::maintenance]).freeze
+        NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2 automation:bot-authored type::maintenance suppress-contributor-links]).freeze
         SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
         FAILURE_ISSUE_GUIDE_URL = "https://handbook.gitlab.com/handbook/engineering/testing/guide-to-e2e-test-failure-issues/"
         FAILURE_ISSUE_HANDBOOK_GUIDE = "**:rotating_light: [End-to-End Test Failure Issue Debugging Guide](#{FAILURE_ISSUE_GUIDE_URL}) :rotating_light:**\n".freeze
@@ -31,13 +36,15 @@ module GitlabQuality
         # there before being released to the public repository
         DIFF_PROJECT_MAPPINGS = {
           'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/security/gitlab',
+          'gitlab-org/quality/test-failure-issues' => 'gitlab-org/security/gitlab',
           'gitlab-org/gitlab' => 'gitlab-org/security/gitlab',
           'gitlab-org/customers-gitlab-com' => 'gitlab-org/customers-gitlab-com'
         }.freeze
 
         # Don't use the E2E test issues project for commit parent
         COMMIT_PROJECT_MAPPINGS = {
-          'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/gitlab'
+          'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/gitlab',
+          'gitlab-org/quality/test-failure-issues' => 'gitlab-org/gitlab'
         }.freeze
 
         # The project contains record of the deployments we use to determine the commit diff
@@ -63,6 +70,8 @@ module GitlabQuality
           base_issue_labels: nil,
           exclude_labels_for_search: nil,
           metrics_files: [],
+          group_similar: false,
+          environment_issues_output_file: nil,
           **kwargs)
           super
           @max_diff_ratio = max_diff_ratio.to_f
@@ -72,21 +81,116 @@ module GitlabQuality
           @issue_type = 'issue'
           @commented_issue_list = Set.new
           @metrics_files = Array(metrics_files)
+          @group_similar = group_similar
+          @environment_issues_output_file = environment_issues_output_file
         end
 
         private
 
-        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client
+        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client, :group_similar,
+          :environment_issues_output_file
 
         def run!
           puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
+          run_with_grouping! if group_similar
+
+          return if similar_issues_grouped?
+
           TestResults::Builder.new(token: token, project: project, file_glob: files).test_results_per_file do |test_results|
             puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
             process_test_results(test_results)
           end
         end
 
+        def similar_issues_grouped?
+          grouper.summary[:grouped_issues].positive?
+        end
+
+        def grouper
+          @grouper ||= GitlabQuality::TestTooling::Report::GroupIssues::GroupResultsInIssues.new(
+            gitlab: gitlab,
+            config: grouper_config
+          )
+        end
+
+        def run_with_grouping!
+          Runtime::Logger.info "=> Grouping similar failures where possible"
+
+          all_test_results = collect_all_test_results
+          return if all_test_results.empty?
+
+          Runtime::Logger.info "=> Processing #{all_test_results.count} failures with GroupResultsInIssues"
+
+          failure_data = convert_test_results_to_failure_data(all_test_results)
+
+          grouper.process_failures(failure_data)
+          grouper.process_issues
+
+          export_environment_issues_for_slack(environment_issues_output_file) if environment_issues_output_file
+        end
+
+        def collect_all_test_results
+          all_test_results = []
+
+          TestResults::Builder.new(token: token, project: project, file_glob: files).test_results_per_file do |test_results|
+            Runtime::Logger.info "=> Collecting #{test_results.count} tests from #{test_results.path}"
+            all_test_results.concat(test_results.select(&:failures?))
+          end
+
+          all_test_results
+        end
+
+        def convert_test_results_to_failure_data(test_results)
+          test_results.map do |test|
+            {
+              description: test.name,
+              full_description: test.name,
+              file_path: test.relative_file,
+              line_number: extract_line_number(test),
+              exception: extract_exception_data(test),
+              ci_job_url: test.ci_job_url,
+              testcase: extract_test_id_or_name(test),
+              product_group: extract_product_group(test),
+              level: extract_level(test)
+            }
+          end
+        end
+
+        def extract_line_number(test)
+          test.respond_to?(:line_number) ? test.line_number : nil
+        end
+
+        def extract_exception_data(test)
+          {
+            'message' => test.failures.first&.dig('message') || test.full_stacktrace || 'Unknown error'
+          }
+        end
+
+        def extract_product_group(test)
+          test.respond_to?(:product_group) ? test.product_group : nil
+        end
+
+        def extract_level(test)
+          test.respond_to?(:level) ? test.level : nil
+        end
+
+        def grouper_config
+          {
+            thresholds: {
+              min_failures_to_group: 2
+            }
+          }
+        end
+
+        def extract_test_id_or_name(test)
+          return test.example_id if test.respond_to?(:example_id)
+          return test.id if test.respond_to?(:id)
+          return test.name if test.respond_to?(:name)
+
+          "#{test.relative_file}:#{test.respond_to?(:line_number) ? test.line_number : 'unknown'}"
+        end
+
         def new_issue_labels(test)
           up_to_date_labels(test: test, new_labels: NEW_ISSUE_LABELS + group_and_category_labels_for_test(test))
         end
@@ -469,10 +573,40 @@ module GitlabQuality
         end
 
         def new_issue_assignee_id(test)
-          return unless test.product_group?
+          assignee_id = try_feature_category_assignment(test)
+          return assignee_id if assignee_id
+
+          try_product_group_assignment(test)
+        end
+
+        def try_feature_category_assignment(test)
+          unless test.respond_to?(:feature_category) && test.feature_category?
+            Runtime::Logger.info("No feature_category found for DRI assignment")
+            return
+          end
+
+          labels_inference = GitlabQuality::TestTooling::LabelsInference.new
+          product_group = labels_inference.product_group_from_feature_category(test.feature_category)
+
+          unless product_group
+            Runtime::Logger.warn("Could not map feature_category '#{test.feature_category}' to product_group")
+            return
+          end
+
+          dri = test_dri(product_group, test.stage, test.section)
+          Runtime::Logger.info("Assigning #{dri} as DRI for the issue (via feature_category).")
+
+          gitlab.find_user_id(username: dri)
+        end
+
+        def try_product_group_assignment(test)
+          unless test.respond_to?(:product_group) && test.product_group?
+            Runtime::Logger.info("No product_group found for DRI assignment")
+            return
+          end
 
           dri = test_dri(test.product_group, test.stage, test.section)
-          puts "  => Assigning #{dri} as DRI for the issue."
+          Runtime::Logger.info("Assigning #{dri} as DRI for the issue (via product_group).")
 
           gitlab.find_user_id(username: dri)
         end
@@ -561,6 +695,43 @@ module GitlabQuality
 
           "|| [Screenshot](#{ci_job_url}/artifacts/file/#{screenshot_path})"
         end
+
+        def export_environment_issues_for_slack(output_file)
+          return unless similar_issues_grouped?
+
+          File.write(output_file, build_environment_issues_data.to_json)
+        rescue StandardError => e
+          puts "Warning: Failed to export environment issues for Slack: #{e.message}"
+        end
+
+        def build_environment_issues_data
+          {
+            grouped_failures: format_grouped_failures,
+            summary: grouper.summary
+          }
+        end
+
+        def format_grouped_failures
+          grouper.grouped_failures.map do |_fingerprint, grouped_failure|
+            {
+              fingerprint: grouped_failure[:fingerprint],
+              pattern_name: grouped_failure[:pattern_name],
+              normalized_message: grouped_failure[:normalized_message],
+              failure_count: grouped_failure[:failures].size,
+              failures: format_individual_failures(grouped_failure[:failures])
+            }
+          end
+        end
+
+        def format_individual_failures(failures)
+          failures.map do |failure|
+            {
+              description: failure[:description],
+              file_path: failure[:file_path],
+              ci_job_url: failure[:ci_job_url]
+            }
+          end
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -104,7 +104,6 @@ module GitlabQuality
             description: new_issue_description(test),
             labels: new_issue_labels(test).to_a,
             issue_type: issue_type,
-            assignee_id: new_issue_assignee_id(test),
             due_date: new_issue_due_date(test),
             confidential: confidential
           }.compact

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -11,7 +11,8 @@ module GitlabQuality
       # - Add test metadata, duration to the issue with group and category labels
       class SlowTestIssue < HealthProblemReporter
         IDENTITY_LABELS       = ['test', 'rspec:slow test', 'test-health:slow', 'rspec profiling', 'automation:bot-authored'].freeze
-        NEW_ISSUE_LABELS      = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
+        NEW_ISSUE_LABELS      = Set.new(
+          ['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'suppress-contributor-links', *IDENTITY_LABELS]).freeze
         REPORT_SECTION_HEADER = '### Slowness reports'
         REPORTS_DOCUMENTATION = <<~DOC
           Slow tests were detected, please see the [test speed best practices guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)

data/lib/gitlab_quality/test_tooling/runtime/env.rb

@@ -15,16 +15,17 @@ module GitlabQuality
           'CI_JOB_ID' => :ci_job_id,
           'CI_JOB_NAME' => :ci_job_name,
           'CI_JOB_URL' => :ci_job_url,
+          'CI_JOB_STATUS' => :ci_job_status,
           'CI_PIPELINE_ID' => :ci_pipeline_id,
           'CI_PIPELINE_NAME' => :ci_pipeline_name,
           'CI_PIPELINE_URL' => :ci_pipeline_url,
           'CI_PROJECT_ID' => :ci_project_id,
           'CI_PROJECT_NAME' => :ci_project_name,
           'CI_PROJECT_PATH' => :ci_project_path,
+          'CI_PIPELINE_CREATED_AT' => :ci_pipeline_created_at,
           'DEPLOY_VERSION' => :deploy_version,
           'GITLAB_QA_ISSUE_URL' => :qa_issue_url,
-          'QA_GITLAB_CI_TOKEN' => :gitlab_ci_token,
-          'SLACK_QA_CHANNEL' => :slack_qa_channel
+          'QA_GITLAB_CI_TOKEN' => :gitlab_ci_token
         }.freeze
 
         ENV_VARIABLES.each do |env_name, method_name|
@@ -61,6 +62,10 @@ module GitlabQuality
           env_var_value_if_defined('GITLAB_GRAPHQL_API_BASE')
         end
 
+        def slack_alerts_channel
+          env_var_value_if_defined('SLACK_ALERTS_CHANNEL') || 'C09HQ5BN07J' # test-tooling-alerts channel ID
+        end
+
         def pipeline_from_project_name
           %w[gitlab gitaly].any? { |str| ci_project_name.to_s.start_with?(str) } ? default_branch : ci_project_name
         end
@@ -111,12 +116,12 @@ module GitlabQuality
         end
 
         def env_var_value_if_defined(variable)
-          return ENV.fetch(variable) if env_var_value_valid?(variable)
+          ENV.fetch(variable) if env_var_value_valid?(variable)
         end
 
         def env_var_name_if_defined(variable)
           # Pass through the variables if they are defined and not empty in the environment
-          return "$#{variable}" if env_var_value_valid?(variable)
+          "$#{variable}" if env_var_value_valid?(variable)
         end
       end
     end

data/lib/gitlab_quality/test_tooling/slack/post_to_slack.rb

@@ -1,15 +1,21 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module GitlabQuality
   module TestTooling
     module Slack
       class PostToSlack
-        def initialize(slack_webhook_url:, channel:, message:, username:, icon_emoji:)
+        MAX_PATTERN_MESSAGE_LENGTH = 150
+        MAX_GROUPED_FAILURES_TO_DISPLAY = 10
+
+        def initialize(slack_webhook_url:, channel:, message:, username:, icon_emoji:, environment_issues_file: nil)
           @slack_webhook_url = slack_webhook_url
           @channel = channel
           @message = message
           @username = username
           @icon_emoji = icon_emoji
+          @environment_issues_file = environment_issues_file
         end
 
         def invoke!
@@ -17,7 +23,7 @@ module GitlabQuality
           params['channel'] = channel
           params['username'] = username
           params['icon_emoji'] = icon_emoji
-          params['text'] = message
+          params['text'] = build_message
 
           Support::HttpRequest.make_http_request(
             method: 'post',
@@ -29,7 +35,101 @@ module GitlabQuality
 
         private
 
-        attr_reader :slack_webhook_url, :channel, :message, :username, :icon_emoji
+        attr_reader :slack_webhook_url, :channel, :message, :username, :icon_emoji, :environment_issues_file
+
+        def build_message
+          messages = []
+          messages << message if message && !message.empty?
+          messages << format_environment_issues if environment_issues_file && File.exist?(environment_issues_file)
+
+          messages.join("\n\n")
+        end
+
+        def format_environment_issues
+          issues_data = JSON.parse(File.read(environment_issues_file))
+          return nil if issues_data.nil? || issues_data['grouped_failures'].empty?
+
+          build_slack_message(issues_data)
+        rescue JSON::ParserError => e
+          ":x: Error parsing environment issues file: #{e.message}"
+        rescue StandardError => e
+          ":x: Error formatting environment issues: #{e.message}"
+        end
+
+        def format_single_environment_issue(failure)
+          pattern_title = pattern_display_name(failure['pattern_name'])
+
+          issue_text = build_issue_header(pattern_title, failure)
+          issue_text + build_job_info(failure)
+        end
+
+        def pattern_display_name(pattern_name)
+          case pattern_name&.downcase
+          when /http_500/
+            "HTTP 500 Internal Server Errors"
+          when /http_400/
+            "HTTP 400 Bad Request Errors"
+          when /http_503/
+            "HTTP 503 Service Unavailable"
+          when /timeout/
+            "Timeout Errors"
+          when /git_rpc|repository/
+            "Git/Repository Errors"
+          else
+            "#{pattern_name&.humanize || 'Unknown'} Errors"
+          end
+        end
+
+        def build_slack_message(issues_data)
+          header = ":warning: *Environment Issues Detected*\n"
+
+          issue_messages = format_issue_messages(issues_data['grouped_failures'])
+          truncation_note = build_truncation_note(issues_data['grouped_failures'].size)
+          summary = build_summary_text(issues_data['summary'])
+
+          header + issue_messages + truncation_note + summary
+        end
+
+        def format_issue_messages(grouped_failures)
+          failures_to_show = grouped_failures.first(MAX_GROUPED_FAILURES_TO_DISPLAY)
+          failures_to_show.map { |failure| format_single_environment_issue(failure) }.join("\n\n")
+        end
+
+        def build_truncation_note(total_failures)
+          return "" unless total_failures > MAX_GROUPED_FAILURES_TO_DISPLAY
+
+          "\n_... and #{total_failures - MAX_GROUPED_FAILURES_TO_DISPLAY} more environment issue(s)_"
+        end
+
+        def build_issue_header(pattern_title, failure)
+          <<~TEXT
+            *#{pattern_title}*
+            • Affected tests: #{failure['failure_count']}
+            • Pattern: `#{truncate_message(failure['normalized_message'])}`
+          TEXT
+        end
+
+        def build_job_info(failure)
+          return "" unless failure['failures']&.any?
+
+          job_urls = failure['failures'].filter_map { |f| f['ci_job_url'] }.uniq
+          job_urls.any? ? "• Jobs affected: #{job_urls.size}\n" : ""
+        end
+
+        def build_summary_text(summary)
+          <<~TEXT
+
+            *Summary:* #{summary['grouped_issues']} environment issue(s) affecting #{summary['total_grouped_failures']} test(s)
+
+            _Note: Future improvements will include direct GitLab issue links and enhanced filtering._
+            _Track progress: https://gitlab.com/groups/gitlab-org/quality/quality-engineering/-/epics/168_
+          TEXT
+        end
+
+        def truncate_message(message)
+          text = message.to_s
+          text.length > MAX_PATTERN_MESSAGE_LENGTH ? "#{text[0..MAX_PATTERN_MESSAGE_LENGTH]}..." : text
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/api_log_finder.rb

@@ -7,7 +7,7 @@ module GitlabQuality
         module Rails
           class ApiLogFinder < JsonLogFinder
             def initialize(base_path, file_path = 'gitlab-rails/api_json.log')
-              super(base_path, file_path)
+              super
             end
 
             def new_log(data)

data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/application_log_finder.rb

@@ -7,7 +7,7 @@ module GitlabQuality
         module Rails
           class ApplicationLogFinder < JsonLogFinder
             def initialize(base_path, file_path = 'gitlab-rails/application_json.log')
-              super(base_path, file_path)
+              super
             end
 
             def new_log(data)

data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/exception_log_finder.rb

@@ -7,7 +7,7 @@ module GitlabQuality
         module Rails
           class ExceptionLogFinder < JsonLogFinder
             def initialize(base_path, file_path = 'gitlab-rails/exceptions_json.log')
-              super(base_path, file_path)
+              super
             end
 
             def new_log(data)

data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/graphql_log_finder.rb

@@ -7,7 +7,7 @@ module GitlabQuality
         module Rails
           class GraphqlLogFinder < JsonLogFinder
             def initialize(base_path, file_path = 'gitlab-rails/graphql_json.log')
-              super(base_path, file_path)
+              super
             end
 
             def new_log(data)

data/lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb

@@ -8,9 +8,7 @@ module GitlabQuality
       class TestMetaUpdater
         include TestTooling::Concerns::FindSetDri
 
-        attr_reader :project, :ref, :report_issue, :processed_commits
-
-        TEST_PLATFORM_MAINTAINERS_SLACK_CHANNEL_ID = 'C0437FV9KBN' # test-platform-maintainers
+        attr_reader :project, :ref, :report_issue, :processed_commits, :token, :specs_file, :dry_run, :processor
 
         def initialize(token:, project:, specs_file:, processor:, ref: 'master', dry_run: false)
           @specs_file = specs_file
@@ -269,12 +267,15 @@ module GitlabQuality
         # Fetch the id for the dri of the product group and stage
         # The first item returned is the id of the assignee and the second item is the handle
         #
-        # @param [String] product_group
+        # @param [Hash] test object
         # @param [String] devops_stage
+        # @param [String] section
         # @return [Array<Integer, String>]
-        def fetch_dri_id(product_group, devops_stage, section)
-          assignee_handle = ENV.fetch('QA_TEST_DRI_HANDLE', nil) || test_dri(product_group, devops_stage, section)
+        def fetch_dri_id(test, devops_stage, section)
+          product_group = determine_product_group(test)
+          return unless product_group
 
+          assignee_handle = ENV.fetch('QA_TEST_DRI_HANDLE', nil) || test_dri(product_group, devops_stage, section)
           [user_id_for_username(assignee_handle), assignee_handle]
         end
 
@@ -291,7 +292,7 @@ module GitlabQuality
         # @param [String] message the message to post
         # @return [HTTP::Response]
         def post_message_on_slack(message)
-          channel = ENV.fetch('SLACK_QA_CHANNEL', nil) || TEST_PLATFORM_MAINTAINERS_SLACK_CHANNEL_ID
+          channel = Runtime::Env.slack_alerts_channel
           slack_options = {
             slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
             channel: channel,
@@ -335,6 +336,16 @@ module GitlabQuality
           label ? %(/label ~"#{label}") : ''
         end
 
+        # Infers the group label from the provided feature category
+        #
+        # @param [String] feature_category feature category
+        # @return [String]
+        def label_from_feature_category(feature_category)
+          labels = labels_inference.infer_labels_from_feature_category(feature_category)
+          group_label = labels.find { |label| label.start_with?('group::') }
+          group_label ? %(/label ~"#{group_label}") : ''
+        end
+
         # Returns the link to the Grafana dashboard for single spec metrics
         #
         # @param [String] example_name the full example name
@@ -344,10 +355,6 @@ module GitlabQuality
           base_url + CGI.escape(example_name)
         end
 
-        private
-
-        attr_reader :token, :specs_file, :dry_run, :processor
-
         # Returns any test description string within single or double quotes
         #
         # @param [String] line the line to check for any quoted string
@@ -380,6 +387,27 @@ module GitlabQuality
         def labels_inference
           @labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
         end
+
+        private
+
+        def determine_product_group(test)
+          return map_feature_category_to_product_group(test) if has_feature_category?(test)
+          return test.product_group if has_product_group?(test)
+
+          nil
+        end
+
+        def has_feature_category?(test)
+          test.respond_to?(:feature_category) && test.feature_category?
+        end
+
+        def has_product_group?(test)
+          test.respond_to?(:product_group) && test.product_group?
+        end
+
+        def map_feature_category_to_product_group(test)
+          labels_inference.product_group_from_feature_category(test.feature_category)
+        end
       end
     end
   end