gitlab_quality-test_tooling 2.16.0 → 2.20.2

data/lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb

@@ -43,14 +43,7 @@ module GitlabQuality
           return unless ENV['CI_SLACK_WEBHOOK_URL']
 
           pipeline = Runtime::Env.pipeline_from_project_name
-          channel = case pipeline
-                    when "canary"
-                      "e2e-run-production"
-                    when "staging", "staging-canary"
-                      "e2e-run-staging"
-                    else
-                      "e2e-run-#{pipeline}"
-                    end
+          channel = Runtime::Env.slack_alerts_channel
 
           slack_options = {
             slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
@@ -58,7 +51,7 @@ module GitlabQuality
             username: "GitLab Quality Test Tooling",
             icon_emoji: ':ci_failing:',
             message: <<~MSG
-              An unexpected error occurred while reporting test results in issues.
+              Env: #{pipeline}. An unexpected error occurred while reporting test results in issues.
               The error occurred in job: #{Runtime::Env.ci_job_url}
               `#{error.class.name} #{error.message}`
             MSG

data/lib/gitlab_quality/test_tooling/gitlab_client/group_labels_client.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module GitlabClient
+      class GroupLabelsClient < GitlabClient
+        def initialize(token:, group:, endpoint: nil, **_kwargs)
+          @token = token
+          @group = group
+          @endpoint = endpoint
+        end
+
+        def group_labels(options: {})
+          client.group_labels(group, options)
+        end
+
+        def create_group_label(name:, color: '#428BCA', description: nil)
+          client.create_group_label(group, name, color, description: description)
+        end
+
+        private
+
+        attr_reader :group, :token, :endpoint
+
+        def client
+          @client ||= Gitlab.client(
+            endpoint: endpoint || ENV['GITLAB_API_BASE'] || Runtime::Env.gitlab_api_base,
+            private_token: token
+          )
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb

@@ -68,7 +68,7 @@ module GitlabQuality
 
         def find_issue_notes(iid:)
           handle_gitlab_client_exceptions do
-            client.issue_notes(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
+            client.issue_notes(project, iid, order_by: 'created_at', sort: 'asc', activity_filter: 'only_comments').auto_paginate
           end
         end
 

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb

@@ -4,8 +4,8 @@ module GitlabQuality
   module TestTooling
     module GitlabClient
       class IssuesDryClient < IssuesClient
-        def create_issue(title:, description:, labels:, issue_type: 'issue', _assignee_id: nil, _due_date: nil, confidential: false)
-          attrs = { description: description, labels: labels, confidential: confidential }
+        def create_issue(title:, description:, labels:, assignee_id: 'unknown', issue_type: 'issue', confidential: false)
+          attrs = { description: description, labels: labels, confidential: confidential, assignee_id: assignee_id }
 
           puts "The following #{issue_type} would have been created:"
           puts "project: #{project}, title: #{title}, attrs: #{attrs}"

data/lib/gitlab_quality/test_tooling/report/failed_test_issue.rb

@@ -12,7 +12,7 @@ module GitlabQuality
       #   - Add a failure report in the "Failure reports" note
       class FailedTestIssue < HealthProblemReporter
         IDENTITY_LABELS       = ['test', 'test-health:failures', 'automation:bot-authored'].freeze
-        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'failure::new', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'failure::new', 'priority::3', 'severity::3', 'suppress-contributor-links', *IDENTITY_LABELS]).freeze
         REPORT_SECTION_HEADER = '#### Failure reports'
 
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -13,10 +13,10 @@ module GitlabQuality
       #   - Add a flakiness report in the "Flakiness reports" note
       class FlakyTestIssue < HealthProblemReporter
         IDENTITY_LABELS       = ['test', 'failure::flaky-test', 'test-health:pass-after-retry', 'automation:bot-authored'].freeze
-        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', 'suppress-contributor-links', *IDENTITY_LABELS]).freeze
         REPORT_SECTION_HEADER = '### Flakiness reports'
         REPORTS_DOCUMENTATION = <<~DOC
-          Flaky tests were detected. Please refer to the [Flaky tests reproducibility instructions](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#how-to-reproduce-a-flaky-test-locally)
+          Flaky tests were detected. Please refer to the [Flaky tests reproducibility instructions](https://docs.gitlab.com/development/testing_guide/unhealthy_tests/#how-to-reproduce-a-flaky-test-locally)
           to learn more about how to reproduce them.
         DOC
 

data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb

@@ -31,9 +31,9 @@ module GitlabQuality
           tests = tests.select { |test| pipeline_stages.include? test.report["stage"] } unless pipeline_stages.empty?
 
           issue = gitlab.create_issue(
-            title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
+            title: "#{Time.now.to_date.iso8601} Test session report | #{Runtime::Env.qa_run_type}",
             description: generate_description(tests),
-            labels: ['automation:bot-authored', 'E2E', 'triage report', pipeline_name_label],
+            labels: ['automation:bot-authored', 'E2E', 'triage report', pipeline_name_label, 'suppress-contributor-links'],
             confidential: confidential
           )
 

data/lib/gitlab_quality/test_tooling/report/group_issues/error_message_normalizer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'openssl'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class ErrorMessageNormalizer
+          NORMALIZATION_PATTERNS = [
+            { pattern: /\d{4}-\d{2}-\d{2}T?[ ]?\d{2}:\d{2}:\d{2}(\.\d+)?Z?/, replacement: "<TIMESTAMP>" },
+            { pattern: /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i, replacement: "<UUID>" },
+            { pattern: /Correlation Id: [\w]+/, replacement: "Correlation Id: <UUID>" },
+            { pattern: /Fabrication of QA::Resource::[A-Za-z:]+/, replacement: "Fabrication of QA::Resource::<RESOURCE>" },
+            { pattern: /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?\b/, replacement: "<IP>" },
+            { pattern: /user\d+/, replacement: "<USER>" },
+            { pattern: /group\d+/, replacement: "<GROUP>" },
+            { pattern: /project\d+/, replacement: "<PROJECT>" },
+            { pattern: %r{https?://[^/\s]+/[^\s]*}, replacement: "<URL>" },
+            { pattern: %r{/tmp/[^\s]+}, replacement: "<TMPFILE>" },
+            { pattern: %r{/var/[^\s]+}, replacement: "<VARFILE>" },
+            { pattern: /token=[^\s&]+/, replacement: "token=<TOKEN>" },
+            { pattern: /after \d+ seconds/, replacement: "after <N> seconds" },
+            { pattern: /waited \d+ seconds/, replacement: "waited <N> seconds" },
+            { pattern: /\d+ attempts?/, replacement: "<N> attempts" },
+            { pattern: /\s+/, replacement: " " }
+          ].freeze
+
+          def normalize(message)
+            return "" if message.nil? || message.empty?
+
+            result = message.dup.strip
+
+            NORMALIZATION_PATTERNS.each do |pattern_rule|
+              result.gsub!(pattern_rule[:pattern], pattern_rule[:replacement])
+            end
+
+            result.strip
+          end
+
+          def create_fingerprint(normalized_message)
+            OpenSSL::Digest::SHA256.hexdigest(normalized_message.downcase)[0..15]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/error_pattern_matcher.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class ErrorPatternMatcher
+          ENVIRONMENT_ERROR_PATTERNS = [
+            { name: "http_500_api_fabrication", pattern: /Fabrication of .+ using the API failed \(500\)/i },
+            { name: "http_500_internal_server", pattern: /(500 Internal Server Error|request returned \(500\)|Expected \(200\), request returned \(500\))/i },
+            { name: "http_400_backend_failing", pattern: /failed \(400\) with.+connections to all backends failing/i },
+            { name: "http_503_service_unavailable", pattern: /Unexpected status code 503/i },
+            { name: "pipeline_creation_timeout", pattern: /Wait for pipeline to be created failed after \d+ seconds/i },
+            { name: "event_timeout", pattern: /(Timed out waiting for event|EventNotFoundError: Timed out waiting)/i },
+            { name: "git_rpc_failure", pattern: /error: RPC failed; HTTP 500/i },
+            { name: "repository_fabricate_error", pattern: /Repository fabricate/i }
+          ].freeze
+
+          def match(error_message)
+            return nil if error_message.nil? || error_message.empty?
+
+            ENVIRONMENT_ERROR_PATTERNS.find { |pattern_def| error_message.match?(pattern_def[:pattern]) }
+          end
+
+          def environment_error?(error_message)
+            !match(error_message).nil?
+          end
+
+          def pattern_name(error_message)
+            match(error_message)&.dig(:name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/failure_processor.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class FailureProcessor
+          DEFAULT_MIN_FAILURES = 2
+
+          def initialize(options = {})
+            @options = options
+            @pattern_matcher = options[:pattern_matcher] || ErrorPatternMatcher.new
+            @normalizer = options[:normalizer] || ErrorMessageNormalizer.new
+            @config = options[:config] || {}
+          end
+
+          def process_failures(failures, &)
+            Runtime::Logger.info "Processing #{failures.size} failures for grouping..."
+            grouped_failures = {}
+
+            failures.each do |failure|
+              process_single_failure(failure, grouped_failures)
+            end
+
+            Runtime::Logger.info "Found #{grouped_failures.size} groups before filtering"
+            grouped_failures.each_value(&)
+          end
+
+          def filter_groups_by_threshold(grouped_failures)
+            min_failures = @config.dig(:thresholds, :min_failures_to_group) || DEFAULT_MIN_FAILURES
+
+            grouped_failures.select! do |_fingerprint, grouped_failure|
+              grouped_failure[:failures].size >= min_failures
+            end
+
+            Runtime::Logger.info "Found #{grouped_failures.size} groups after filtering"
+          end
+
+          private
+
+          def process_single_failure(failure, grouped_failures)
+            error_message = failure.dig(:exception, 'message') || failure.dig(:exceptions, 0, 'message')
+            Runtime::Logger.info "Processing failure: #{failure[:description]}"
+            Runtime::Logger.info "Error message: #{error_message[0..100]}..." if error_message
+
+            return unless error_message && @pattern_matcher.environment_error?(error_message)
+
+            Runtime::Logger.info "Identified as environment error"
+            group_environment_failure(failure, error_message, grouped_failures)
+          end
+
+          def group_environment_failure(failure, error_message, grouped_failures)
+            normalized_message = @normalizer.normalize(error_message)
+            fingerprint = @normalizer.create_fingerprint(normalized_message)
+            pattern_name = @pattern_matcher.pattern_name(error_message)
+
+            grouped_failures[fingerprint] ||= build_grouped_failure(fingerprint, pattern_name, normalized_message)
+            grouped_failures[fingerprint][:failures] << failure
+          end
+
+          def build_grouped_failure(fingerprint, pattern_name, normalized_message)
+            {
+              fingerprint: fingerprint,
+              pattern_name: pattern_name,
+              normalized_message: normalized_message,
+              failures: []
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/group_results_in_issues.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+require 'time'
+require_relative 'error_pattern_matcher'
+require_relative 'error_message_normalizer'
+require_relative 'incident_checker'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class GroupResultsInIssues
+          attr_reader :grouped_failures
+
+          def initialize(options = {})
+            @options = options
+            @failure_processor = FailureProcessor.new(options)
+            @issue_manager = IssueManager.new(options)
+            @grouped_failures = {}
+          end
+
+          def process_failures(failures)
+            @failure_processor.process_failures(failures) do |grouped_failure|
+              fingerprint = grouped_failure[:fingerprint]
+              @grouped_failures[fingerprint] = grouped_failure
+            end
+
+            @failure_processor.filter_groups_by_threshold(@grouped_failures)
+          end
+
+          def process_issues
+            @grouped_failures.each_value do |grouped_failure|
+              @issue_manager.create_or_update_issue(grouped_failure)
+            end
+          end
+
+          def summary
+            {
+              grouped_issues: @grouped_failures.size,
+              total_grouped_failures: @grouped_failures.values.sum { |group| group[:failures].size }
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/incident_checker.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IncidentChecker
+          GITLAB_PRODUCTION_PROJECT_ID = '7444821' # gitlab-com/gl-infra/production
+
+          def self.get_active_incidents(token: nil, gitlab_url: 'https://gitlab.com')
+            return [] unless token
+
+            begin
+              client = GitlabClient::IssuesClient.new(token: token, endpoint: "#{gitlab_url}/api/v4", project: GITLAB_PRODUCTION_PROJECT_ID)
+
+              issues = client.find_issues(options: {
+                labels: 'Incident::Active',
+                state: 'opened',
+                per_page: 10,
+                order_by: 'created_at',
+                sort: 'desc'
+              })
+
+              issues.map do |issue|
+                {
+                  title: issue.title,
+                  url: issue.web_url
+                }
+              end
+
+            rescue Gitlab::Error::Error => e
+              Runtime::Logger.error "GitLab API error fetching incidents: #{e.message}"
+              []
+            rescue StandardError => e
+              Runtime::Logger.error "Warning: Could not fetch active incidents: #{e.message}"
+              []
+            end
+          end
+
+          def self.format_incidents_for_issue(incidents)
+            return "" if incidents.empty?
+
+            incident_list = incidents.map { |inc| "- [#{inc[:title]}](#{inc[:url]})" }.join("\n")
+
+            <<~MARKDOWN
+
+        ### Related GitLab Incidents
+        The following active incidents may be related to these test failures:
+
+        #{incident_list}
+
+        Check the [GitLab Production Issues](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/?label_name%5B%5D=Incident%3A%3AActive) for updates.
+            MARKDOWN
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_base.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueBase
+          def initialize(client, options = {})
+            @client = client
+            @options = options
+            @gitlab_url = ENV.fetch('CI_SERVER_URL', 'https://gitlab.com')
+            @project_id = options[:target_project] || ENV['RESULTS_ISSUE_PROJECT'] || ENV.fetch('CI_PROJECT_ID', nil)
+            @token = options[:token]
+          end
+
+          private
+
+          def handle_gitlab_api_error(operation, context = nil)
+            yield
+          rescue Gitlab::Error::Error => e
+            log_gitlab_error(operation, context, e)
+            nil
+          rescue StandardError => e
+            log_standard_error(operation, context, e)
+            nil
+          end
+
+          def log_gitlab_error(operation, context, error)
+            context_info = context ? " #{context}" : ""
+            Runtime::Logger.error "GitLab API error #{operation}#{context_info}: #{error.message}"
+          end
+
+          def log_standard_error(operation, context, error)
+            context_info = context ? " #{context}" : ""
+            Runtime::Logger.error "Error #{operation}#{context_info}: #{error.message}"
+          end
+
+          def display_description_preview(description, title = "Description preview:")
+            Runtime::Logger.info title
+            lines = description.split("\n")
+            Runtime::Logger.info lines.first(15).join("\n")
+            Runtime::Logger.info "..." if lines.length > 15
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_creator.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueCreator < IssueBase
+          GROUPED_ISSUE_LABELS = Set.new(%w[test failure::test-environment automation:bot-authored type::maintenance]).freeze
+
+          def initialize(client, options = {})
+            super
+            @formatter = IssueFormatter.new
+          end
+
+          def create_new_issue(grouped_failure)
+            title = @formatter.generate_issue_title(grouped_failure)
+            description = @formatter.generate_issue_description(grouped_failure, @options)
+            labels = GROUPED_ISSUE_LABELS
+
+            Runtime::Logger.info "Creating new grouped issue: #{title} (#{grouped_failure[:failures].size} failures)"
+
+            create_issue(
+              title: title,
+              description: description,
+              labels: labels,
+              failures: grouped_failure[:failures]
+            )
+          end
+
+          private
+
+          def create_issue(title:, description:, labels:, failures:)
+            Runtime::Logger.info "Creating issue: #{title} with #{failures.size} failures"
+
+            handle_gitlab_api_error("creating issue") do
+              issue = @client.create_issue(title: title, description: description, labels: labels.join(','))
+              Runtime::Logger.info "Issue created successfully: #{issue.web_url}" if issue&.web_url
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_finder.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueFinder
+          DEFAULT_MAX_AGE_HOURS = 24
+          ISSUES_PER_PAGE = 50
+
+          def initialize(client, options = {})
+            @client = client
+            @options = options
+            @token = options[:token]
+            @project_id = options[:target_project] || ENV['RESULTS_ISSUE_PROJECT'] || ENV.fetch('CI_PROJECT_ID', nil)
+          end
+
+          def find_existing_issue(grouped_failure)
+            find_related_issue_by_fingerprint(
+              fingerprint: grouped_failure[:fingerprint],
+              max_age_hours: DEFAULT_MAX_AGE_HOURS
+            )
+          end
+
+          private
+
+          def find_related_issue_by_fingerprint(fingerprint:, max_age_hours:)
+            Runtime::Logger.info "Searching for existing issue with fingerprint: #{fingerprint}"
+
+            begin
+              issues = fetch_recent_open_issues(max_age_hours)
+              matching_issue = find_issue_with_fingerprint(issues, fingerprint)
+              if matching_issue
+                Runtime::Logger.info "Found existing issue: ##{matching_issue.iid} - #{matching_issue.title}"
+                convert_to_struct(matching_issue)
+              end
+            rescue Gitlab::Error::Error => e
+              Runtime::Logger.error "GitLab API error searching for issues: #{e.message}"
+              nil
+            rescue StandardError => e
+              Runtime::Logger.error "Error searching for existing issues: #{e.message}"
+              nil
+            end
+          end
+
+          def fetch_recent_open_issues(max_age_hours)
+            cutoff_time = Time.now - (max_age_hours * 3600)
+
+            @client.find_issues(options: {
+              state: 'opened',
+              created_after: cutoff_time.utc.iso8601,
+              per_page: ISSUES_PER_PAGE,
+              order_by: 'created_at',
+              sort: 'desc'
+            })
+          end
+
+          def find_issue_with_fingerprint(issues, fingerprint)
+            fingerprint_tag = "grouped-failure-fingerprint:#{fingerprint}"
+            issues.find { |issue| issue.description&.include?(fingerprint_tag) }
+          end
+
+          def convert_to_struct(obj)
+            return obj unless obj.is_a?(Hash)
+
+            struct_class = Struct.new(*obj.keys.map(&:to_sym))
+            struct = struct_class.new
+
+            obj.each do |key, value|
+              struct[key.to_sym] = value.is_a?(Hash) ? convert_to_struct(value) : value
+            end
+
+            struct
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_formatter.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueFormatter
+          def generate_issue_title(grouped_failure)
+            case grouped_failure[:pattern_name]
+            when /http_500/ then "Environment Issue: HTTP 500 Internal Server Errors"
+            when /http_400/ then "Environment Issue: Backend Connection Failures"
+            when /http_503/ then "Environment Issue: Service Unavailable (503)"
+            when /timeout/ then "Environment Issue: Timeout Failures"
+            when /git_rpc|repository/ then "Environment Issue: Repository/Git Operation Failures"
+            else "Environment Issue: Multiple Similar Failures"
+            end
+          end
+
+          def generate_issue_description(grouped_failure, options = {})
+            active_incidents = IncidentChecker.get_active_incidents(token: options[:token])
+            incident_section = IncidentChecker.format_incidents_for_issue(active_incidents)
+
+            <<~MARKDOWN
+            ## Environment Issue: #{grouped_failure[:pattern_name]}
+
+            Multiple tests have failed with similar error patterns, indicating an environment-related issue affecting multiple test cases.
+
+            ### Error Pattern
+            ```
+            #{grouped_failure[:normalized_message]}
+            ```
+
+            ### Affected Tests (#{grouped_failure[:failures].size} failures)
+            #{format_affected_tests(grouped_failure[:failures])}
+
+            ### Pipeline Information
+            #{format_pipeline_info(grouped_failure[:failures].first)}
+
+            ### Recommended Actions
+            #{generate_recommended_actions(grouped_failure)}
+
+            #{incident_section}
+            ---
+            <!-- grouped-failure-fingerprint:#{grouped_failure[:fingerprint]} -->
+            MARKDOWN
+          end
+
+          def format_affected_tests(failures)
+            failures.map do |failure|
+              job_name = failure[:ci_job_url] || failure.dig(:ci_job, :name) || 'unknown_job'
+              spec_file = failure[:file_path] || failure[:file] || 'unknown_spec'
+              line_number = failure[:line_number] || failure[:line]
+              test_name = failure[:description] || failure[:test_name] || 'Unknown test'
+
+              spec_with_line = line_number.to_s.empty? ? spec_file : "#{spec_file}:#{line_number}"
+              "- **#{test_name}** (Job: `#{job_name}`, Spec: `#{spec_with_line}`)"
+            end.join("\n")
+          end
+
+          def format_pipeline_info(failure)
+            pipeline_url = ENV['CI_PIPELINE_URL'] || "Pipeline #{ENV.fetch('CI_PIPELINE_ID', 'unknown')}"
+            job_url = failure[:ci_job_url] || 'Unknown job'
+
+            "- **Pipeline**: #{pipeline_url}\n- **Job**: #{job_url}"
+          end
+
+          def generate_recommended_actions(grouped_failure)
+            case grouped_failure[:pattern_name]
+            when /http_500/
+              "1. Check GitLab instance status and logs\n2. Verify database connectivity\n3. Review application server health"
+            when /timeout/
+              "1. Check network connectivity\n2. Review timeout configurations\n3. Monitor system resources"
+            when /git_rpc|repository/
+              "1. Verify Git repository accessibility\n2. Check Gitaly service status\n3. Review storage capacity"
+            else
+              "1. Check if there are ongoing incidents affecting the GitLab instance\n2. Verify API endpoints are responding correctly\n3. Review system logs for related errors"
+            end
+          end
+        end
+      end
+    end
+  end
+end