gitlab_quality-test_tooling 2.16.0 → 2.25.1

data/lib/gitlab_quality/test_tooling/report/group_issues/error_pattern_matcher.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class ErrorPatternMatcher
+          ENVIRONMENT_ERROR_PATTERNS = [
+            { name: "http_500_api_fabrication", pattern: /Fabrication of .+ using the API failed \(500\)/i },
+            { name: "http_500_internal_server", pattern: /(500 Internal Server Error|request returned \(500\)|Expected \(200\), request returned \(500\))/i },
+            { name: "http_400_backend_failing", pattern: /failed \(400\) with.+connections to all backends failing/i },
+            { name: "http_503_service_unavailable", pattern: /Unexpected status code 503/i },
+            { name: "pipeline_creation_timeout", pattern: /Wait for pipeline to be created failed after \d+ seconds/i },
+            { name: "event_timeout", pattern: /(Timed out waiting for event|EventNotFoundError: Timed out waiting)/i },
+            { name: "git_rpc_failure", pattern: /error: RPC failed; HTTP 500/i },
+            { name: "repository_fabricate_error", pattern: /Repository fabricate/i }
+          ].freeze
+
+          def match(error_message)
+            return nil if error_message.nil? || error_message.empty?
+
+            ENVIRONMENT_ERROR_PATTERNS.find { |pattern_def| error_message.match?(pattern_def[:pattern]) }
+          end
+
+          def environment_error?(error_message)
+            !match(error_message).nil?
+          end
+
+          def pattern_name(error_message)
+            match(error_message)&.dig(:name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/failure_processor.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class FailureProcessor
+          DEFAULT_MIN_FAILURES = 2
+
+          def initialize(options = {})
+            @options = options
+            @pattern_matcher = options[:pattern_matcher] || ErrorPatternMatcher.new
+            @normalizer = options[:normalizer] || ErrorMessageNormalizer.new
+            @config = options[:config] || {}
+          end
+
+          def process_failures(failures, &)
+            Runtime::Logger.info "Processing #{failures.size} failures for grouping..."
+            grouped_failures = {}
+
+            failures.each do |failure|
+              process_single_failure(failure, grouped_failures)
+            end
+
+            Runtime::Logger.info "Found #{grouped_failures.size} groups before filtering"
+            grouped_failures.each_value(&)
+          end
+
+          def filter_groups_by_threshold(grouped_failures)
+            min_failures = @config.dig(:thresholds, :min_failures_to_group) || DEFAULT_MIN_FAILURES
+
+            grouped_failures.select! do |_fingerprint, grouped_failure|
+              grouped_failure[:failures].size >= min_failures
+            end
+
+            Runtime::Logger.info "Found #{grouped_failures.size} groups after filtering"
+          end
+
+          private
+
+          def process_single_failure(failure, grouped_failures)
+            error_message = failure.dig(:exception, 'message') || failure.dig(:exceptions, 0, 'message')
+            Runtime::Logger.info "Processing failure: #{failure[:description]}"
+            Runtime::Logger.info "Error message: #{error_message[0..100]}..." if error_message
+
+            return unless error_message && @pattern_matcher.environment_error?(error_message)
+
+            Runtime::Logger.info "Identified as environment error"
+            group_environment_failure(failure, error_message, grouped_failures)
+          end
+
+          def group_environment_failure(failure, error_message, grouped_failures)
+            normalized_message = @normalizer.normalize(error_message)
+            fingerprint = @normalizer.create_fingerprint(normalized_message)
+            pattern_name = @pattern_matcher.pattern_name(error_message)
+
+            grouped_failures[fingerprint] ||= build_grouped_failure(fingerprint, pattern_name, normalized_message)
+            grouped_failures[fingerprint][:failures] << failure
+          end
+
+          def build_grouped_failure(fingerprint, pattern_name, normalized_message)
+            {
+              fingerprint: fingerprint,
+              pattern_name: pattern_name,
+              normalized_message: normalized_message,
+              failures: []
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/group_results_in_issues.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+require 'time'
+require_relative 'error_pattern_matcher'
+require_relative 'error_message_normalizer'
+require_relative 'incident_checker'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class GroupResultsInIssues
+          attr_reader :grouped_failures
+
+          def initialize(options = {})
+            @options = options
+            @failure_processor = FailureProcessor.new(options)
+            @issue_manager = IssueManager.new(options)
+            @grouped_failures = {}
+          end
+
+          def process_failures(failures)
+            @failure_processor.process_failures(failures) do |grouped_failure|
+              fingerprint = grouped_failure[:fingerprint]
+              @grouped_failures[fingerprint] = grouped_failure
+            end
+
+            @failure_processor.filter_groups_by_threshold(@grouped_failures)
+          end
+
+          def process_issues
+            @grouped_failures.each_value do |grouped_failure|
+              @issue_manager.create_or_update_issue(grouped_failure)
+            end
+          end
+
+          def summary
+            {
+              grouped_issues: @grouped_failures.size,
+              total_grouped_failures: @grouped_failures.values.sum { |group| group[:failures].size }
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/incident_checker.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IncidentChecker
+          GITLAB_PRODUCTION_PROJECT_ID = '7444821' # gitlab-com/gl-infra/production
+
+          def self.get_active_incidents(token: nil, gitlab_url: 'https://gitlab.com')
+            return [] unless token
+
+            begin
+              client = GitlabClient::IssuesClient.new(token: token, endpoint: "#{gitlab_url}/api/v4", project: GITLAB_PRODUCTION_PROJECT_ID)
+
+              issues = client.find_issues(options: {
+                labels: 'Incident::Active',
+                state: 'opened',
+                per_page: 10,
+                order_by: 'created_at',
+                sort: 'desc'
+              })
+
+              issues.map do |issue|
+                {
+                  title: issue.title,
+                  url: issue.web_url
+                }
+              end
+
+            rescue Gitlab::Error::Error => e
+              Runtime::Logger.error "GitLab API error fetching incidents: #{e.message}"
+              []
+            rescue StandardError => e
+              Runtime::Logger.error "Warning: Could not fetch active incidents: #{e.message}"
+              []
+            end
+          end
+
+          def self.format_incidents_for_issue(incidents)
+            return "" if incidents.empty?
+
+            incident_list = incidents.map { |inc| "- [#{inc[:title]}](#{inc[:url]})" }.join("\n")
+
+            <<~MARKDOWN
+
+        ### Related GitLab Incidents
+        The following active incidents may be related to these test failures:
+
+        #{incident_list}
+
+        Check the [GitLab Production Issues](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/?label_name%5B%5D=Incident%3A%3AActive) for updates.
+            MARKDOWN
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_base.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueBase
+          def initialize(client, options = {})
+            @client = client
+            @options = options
+            @gitlab_url = ENV.fetch('CI_SERVER_URL', 'https://gitlab.com')
+            @project_id = options[:target_project] || ENV['RESULTS_ISSUE_PROJECT'] || ENV.fetch('CI_PROJECT_ID', nil)
+            @token = options[:token]
+          end
+
+          private
+
+          def handle_gitlab_api_error(operation, context = nil)
+            yield
+          rescue Gitlab::Error::Error => e
+            log_gitlab_error(operation, context, e)
+            nil
+          rescue StandardError => e
+            log_standard_error(operation, context, e)
+            nil
+          end
+
+          def log_gitlab_error(operation, context, error)
+            context_info = context ? " #{context}" : ""
+            Runtime::Logger.error "GitLab API error #{operation}#{context_info}: #{error.message}"
+          end
+
+          def log_standard_error(operation, context, error)
+            context_info = context ? " #{context}" : ""
+            Runtime::Logger.error "Error #{operation}#{context_info}: #{error.message}"
+          end
+
+          def display_description_preview(description, title = "Description preview:")
+            Runtime::Logger.info title
+            lines = description.split("\n")
+            Runtime::Logger.info lines.first(15).join("\n")
+            Runtime::Logger.info "..." if lines.length > 15
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_creator.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueCreator < IssueBase
+          GROUPED_ISSUE_LABELS = Set.new(%w[test failure::test-environment automation:bot-authored type::maintenance]).freeze
+
+          def initialize(client, options = {})
+            super
+            @formatter = IssueFormatter.new
+          end
+
+          def create_new_issue(grouped_failure)
+            title = @formatter.generate_issue_title(grouped_failure)
+            description = @formatter.generate_issue_description(grouped_failure, @options)
+            labels = GROUPED_ISSUE_LABELS
+
+            Runtime::Logger.info "Creating new grouped issue: #{title} (#{grouped_failure[:failures].size} failures)"
+
+            create_issue(
+              title: title,
+              description: description,
+              labels: labels,
+              failures: grouped_failure[:failures]
+            )
+          end
+
+          private
+
+          def create_issue(title:, description:, labels:, failures:)
+            Runtime::Logger.info "Creating issue: #{title} with #{failures.size} failures"
+
+            handle_gitlab_api_error("creating issue") do
+              issue = @client.create_issue(title: title, description: description, labels: labels.join(','))
+              Runtime::Logger.info "Issue created successfully: #{issue.web_url}" if issue&.web_url
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_finder.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueFinder
+          DEFAULT_MAX_AGE_HOURS = 24
+          ISSUES_PER_PAGE = 50
+
+          def initialize(client, options = {})
+            @client = client
+            @options = options
+            @token = options[:token]
+            @project_id = options[:target_project] || ENV['RESULTS_ISSUE_PROJECT'] || ENV.fetch('CI_PROJECT_ID', nil)
+          end
+
+          def find_existing_issue(grouped_failure)
+            find_related_issue_by_fingerprint(
+              fingerprint: grouped_failure[:fingerprint],
+              max_age_hours: DEFAULT_MAX_AGE_HOURS
+            )
+          end
+
+          private
+
+          def find_related_issue_by_fingerprint(fingerprint:, max_age_hours:)
+            Runtime::Logger.info "Searching for existing issue with fingerprint: #{fingerprint}"
+
+            begin
+              issues = fetch_recent_open_issues(max_age_hours)
+              matching_issue = find_issue_with_fingerprint(issues, fingerprint)
+              if matching_issue
+                Runtime::Logger.info "Found existing issue: ##{matching_issue.iid} - #{matching_issue.title}"
+                convert_to_struct(matching_issue)
+              end
+            rescue Gitlab::Error::Error => e
+              Runtime::Logger.error "GitLab API error searching for issues: #{e.message}"
+              nil
+            rescue StandardError => e
+              Runtime::Logger.error "Error searching for existing issues: #{e.message}"
+              nil
+            end
+          end
+
+          def fetch_recent_open_issues(max_age_hours)
+            cutoff_time = Time.now - (max_age_hours * 3600)
+
+            @client.find_issues(options: {
+              state: 'opened',
+              created_after: cutoff_time.utc.iso8601,
+              per_page: ISSUES_PER_PAGE,
+              order_by: 'created_at',
+              sort: 'desc'
+            })
+          end
+
+          def find_issue_with_fingerprint(issues, fingerprint)
+            fingerprint_tag = "grouped-failure-fingerprint:#{fingerprint}"
+            issues.find { |issue| issue.description&.include?(fingerprint_tag) }
+          end
+
+          def convert_to_struct(obj)
+            return obj unless obj.is_a?(Hash)
+
+            struct_class = Struct.new(*obj.keys.map(&:to_sym))
+            struct = struct_class.new
+
+            obj.each do |key, value|
+              struct[key.to_sym] = value.is_a?(Hash) ? convert_to_struct(value) : value
+            end
+
+            struct
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_formatter.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueFormatter
+          def generate_issue_title(grouped_failure)
+            case grouped_failure[:pattern_name]
+            when /http_500/ then "Environment Issue: HTTP 500 Internal Server Errors"
+            when /http_400/ then "Environment Issue: Backend Connection Failures"
+            when /http_503/ then "Environment Issue: Service Unavailable (503)"
+            when /timeout/ then "Environment Issue: Timeout Failures"
+            when /git_rpc|repository/ then "Environment Issue: Repository/Git Operation Failures"
+            else "Environment Issue: Multiple Similar Failures"
+            end
+          end
+
+          def generate_issue_description(grouped_failure, options = {})
+            active_incidents = IncidentChecker.get_active_incidents(token: options[:token])
+            incident_section = IncidentChecker.format_incidents_for_issue(active_incidents)
+
+            <<~MARKDOWN
+            ## Environment Issue: #{grouped_failure[:pattern_name]}
+
+            Multiple tests have failed with similar error patterns, indicating an environment-related issue affecting multiple test cases.
+
+            ### Error Pattern
+            ```
+            #{grouped_failure[:normalized_message]}
+            ```
+
+            ### Affected Tests (#{grouped_failure[:failures].size} failures)
+            #{format_affected_tests(grouped_failure[:failures])}
+
+            ### Pipeline Information
+            #{format_pipeline_info(grouped_failure[:failures].first)}
+
+            ### Recommended Actions
+            #{generate_recommended_actions(grouped_failure)}
+
+            #{incident_section}
+            ---
+            <!-- grouped-failure-fingerprint:#{grouped_failure[:fingerprint]} -->
+            MARKDOWN
+          end
+
+          def format_affected_tests(failures)
+            failures.map do |failure|
+              job_name = failure[:ci_job_url] || failure.dig(:ci_job, :name) || 'unknown_job'
+              spec_file = failure[:file_path] || failure[:file] || 'unknown_spec'
+              line_number = failure[:line_number] || failure[:line]
+              test_name = failure[:description] || failure[:test_name] || 'Unknown test'
+
+              spec_with_line = line_number.to_s.empty? ? spec_file : "#{spec_file}:#{line_number}"
+              "- **#{test_name}** (Job: `#{job_name}`, Spec: `#{spec_with_line}`)"
+            end.join("\n")
+          end
+
+          def format_pipeline_info(failure)
+            pipeline_url = ENV['CI_PIPELINE_URL'] || "Pipeline #{ENV.fetch('CI_PIPELINE_ID', 'unknown')}"
+            job_url = failure[:ci_job_url] || 'Unknown job'
+
+            "- **Pipeline**: #{pipeline_url}\n- **Job**: #{job_url}"
+          end
+
+          def generate_recommended_actions(grouped_failure)
+            case grouped_failure[:pattern_name]
+            when /http_500/
+              "1. Check GitLab instance status and logs\n2. Verify database connectivity\n3. Review application server health"
+            when /timeout/
+              "1. Check network connectivity\n2. Review timeout configurations\n3. Monitor system resources"
+            when /git_rpc|repository/
+              "1. Verify Git repository accessibility\n2. Check Gitaly service status\n3. Review storage capacity"
+            else
+              "1. Check if there are ongoing incidents affecting the GitLab instance\n2. Verify API endpoints are responding correctly\n3. Review system logs for related errors"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_manager.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueManager
+          DEFAULT_MAX_AGE_HOURS = 24
+          ISSUES_PER_PAGE = 50
+          GROUPED_ISSUE_LABELS = Set.new(%w[test failure::test-environment automation:bot-authored type::maintenance]).freeze
+
+          def initialize(options = {})
+            @options = options
+            @client = options[:gitlab]
+            @issue_finder = IssueFinder.new(@client, @options)
+            @issue_updater = IssueUpdater.new(@client, @options)
+            @issue_creator = IssueCreator.new(@client, @options)
+          end
+
+          def create_or_update_issue(grouped_failure)
+            existing_issue = @issue_finder.find_existing_issue(grouped_failure)
+
+            if existing_issue
+              @issue_updater.update_existing_issue(existing_issue, grouped_failure)
+            else
+              @issue_creator.create_new_issue(grouped_failure)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_updater.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueUpdater < IssueBase
+          def update_existing_issue(issue, grouped_failure)
+            log_issue_update(issue, grouped_failure)
+            append_failures_to_issue(issue, grouped_failure[:failures])
+            add_update_comment(issue, grouped_failure[:failures].size)
+          end
+
+          private
+
+          def append_failures_to_issue(issue, failures)
+            current_issue = @client.find_issues(iid: issue.iid).first
+            return unless current_issue
+
+            existing_description = current_issue.description
+            affected_tests_match = existing_description.match(/### Affected Tests \((\d+) failures?\)/)
+            return unless affected_tests_match
+
+            updated_description = build_updated_description(existing_description, affected_tests_match, failures)
+            update_issue_description(issue, updated_description, failures.size)
+          end
+
+          def update_issue_description(issue, updated_description, failure_count)
+            handle_gitlab_api_error("updating issue", "##{issue.web_url}") do
+              @client.edit_issue(iid: issue.iid, options: { description: updated_description })
+              Runtime::Logger.info "Successfully appended #{failure_count} failures to issue #{issue.web_url}"
+              true
+            end
+          end
+
+          def build_updated_description(existing_description, affected_tests_match, failures)
+            current_count = affected_tests_match[1].to_i
+            new_count = current_count + failures.size
+
+            updated_description = existing_description.gsub(
+              /### Affected Tests \(\d+ failures?\)/,
+              "### Affected Tests (#{new_count} failures)"
+            )
+            insert_new_failures(updated_description, failures)
+          end
+
+          def insert_new_failures(description, failures)
+            formatter = IssueFormatter.new
+            new_failure_entries = formatter.format_affected_tests(failures)
+
+            test_section_end = description.index('### Pipeline Information')
+            return description unless test_section_end
+
+            insertion_point = description.rindex("\n", test_section_end - 1)
+            return description unless insertion_point
+
+            "#{description[0..insertion_point]}#{new_failure_entries}\n#{description[insertion_point + 1..]}"
+          end
+
+          def add_update_comment(issue, failure_count)
+            pipeline_url = ENV['CI_PIPELINE_URL'] || "Pipeline #{ENV.fetch('CI_PIPELINE_ID', nil)}"
+            comment = build_update_comment(pipeline_url, failure_count)
+            add_comment_to_issue(issue, comment)
+          end
+
+          def build_update_comment(pipeline_url, failure_count)
+            "🔄 **New failures added from #{pipeline_url}**\n\n" \
+              "Added #{failure_count} additional test failures with the same error pattern."
+          end
+
+          def add_comment_to_issue(issue, comment)
+            handle_gitlab_api_error("adding comment to issue", issue.web_url) do
+              @client.create_issue_note(iid: issue.iid, note: comment)
+              Runtime::Logger.info "Comment added successfully to issue #{issue.web_url}"
+              true
+            end
+          end
+
+          def log_issue_update(issue, grouped_failure)
+            pipeline_id = ENV.fetch('CI_PIPELINE_ID', nil)
+            Runtime::Logger.info "Updating existing issue ##{issue.iid} with #{grouped_failure[:failures].size} new failures from pipeline #{pipeline_id}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/health_problem_reporter.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'date'
+require 'time'
+
 module GitlabQuality
   module TestTooling
     module Report
@@ -15,7 +18,6 @@ module GitlabQuality
       class HealthProblemReporter < ReportAsIssue
         include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports
-        include TestMetricsExporter::Support::GcsTools
 
         BASE_SEARCH_LABELS    = ['test'].freeze
         FOUND_IN_MR_LABEL     = '~"found:in MR"'
@@ -151,7 +153,7 @@ module GitlabQuality
         def push_test_to_gcs(tests_data, test_results_filename)
           Runtime::Logger.info "will push the test data to GCS"
 
-          gcs_client(project_id: gcs_project_id, credentials: gcs_credentials, dry_run: dry_run).put_object(
+          GcsTools.gcs_client(project_id: gcs_project_id, credentials: gcs_credentials, dry_run: dry_run).put_object(
             gcs_bucket,
             gcs_metrics_file_name(test_results_filename),
             tests_data.to_json,
@@ -163,7 +165,7 @@ module GitlabQuality
         end
 
         def gcs_metrics_file_name(test_results_filename)
-          today = Time.now.strftime('%Y-%m-%d')
+          today = Time.now.to_date.iso8601
 
           "#{today}-#{test_results_filename}"
         end
@@ -253,6 +255,7 @@ module GitlabQuality
             issue_url: issues.first&.web_url,
             job_id: Runtime::Env.ci_job_id,
             job_web_url: test.ci_job_url,
+            job_status: Runtime::Env.ci_job_status,
             pipeline_id: Runtime::Env.ci_pipeline_id,
             pipeline_ref: Runtime::Env.ci_commit_ref_name,
             pipeline_web_url: Runtime::Env.ci_pipeline_url,

data/lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb

@@ -18,7 +18,7 @@ module GitlabQuality
 
         NEW_ISSUE_LABELS = Set.new([
           'test', 'automation:bot-authored', 'type::maintenance', 'maintenance::performance',
-          'priority::3', 'severity::3', 'knapsack_report'
+          'priority::3', 'severity::3', 'knapsack_report', 'suppress-contributor-links'
         ]).freeze
         SEARCH_LABELS = %w[test maintenance::performance knapsack_report].freeze
         JOB_TIMEOUT_EPIC_URL = 'https://gitlab.com/groups/gitlab-org/quality/engineering-productivity/-/epics/19'

data/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb

@@ -78,13 +78,9 @@ module GitlabQuality
         end
 
         def slow_test_rows(slow_test)
-          rows = []
-
-          slow_test.each do |test|
-            rows << slow_test_table_row(test)
+          slow_test.map do |test|
+            slow_test_table_row(test)
           end
-
-          rows
         end
 
         def build_note(slow_test)