gitlab_quality-test_tooling 2.16.0 → 2.21.0

data/lib/gitlab_quality/test_tooling/gcs_tools.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'fog/google'
+
+module GitlabQuality
+  module TestTooling
+    module GcsTools
+      class GCSMockClient
+        def initialize(*_args, **_kwargs); end
+
+        def put_object(gcs_bucket, filename, data, **_kwargs)
+          Runtime::Logger.info "The #{filename} file would have been pushed to the #{gcs_bucket} bucket, with this content:"
+          Runtime::Logger.info data
+        end
+      end
+
+      class << self
+        # GCS Client
+        #
+        # @param project_id [String]
+        # @param credentials [String]
+        # @return [Fog::Storage::Google]
+        def gcs_client(project_id:, credentials:, dry_run: false)
+          if dry_run
+            GCSMockClient.new
+          else
+            Fog::Storage::Google.new(
+              google_project: project_id || raise("Missing Google project_id"),
+              **gcs_creds(credentials)
+            )
+          end
+        end
+
+        private
+
+        # GCS Credentials
+        #
+        # @param credentials [String]
+        # @return [Hash]
+        def gcs_creds(credentials)
+          json_key = credentials || raise('Missing Google credentials')
+          return { google_json_key_location: json_key } if File.exist?(json_key)
+
+          { google_json_key_string: json_key }
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb

@@ -43,14 +43,7 @@ module GitlabQuality
           return unless ENV['CI_SLACK_WEBHOOK_URL']
 
           pipeline = Runtime::Env.pipeline_from_project_name
-          channel = case pipeline
-                    when "canary"
-                      "e2e-run-production"
-                    when "staging", "staging-canary"
-                      "e2e-run-staging"
-                    else
-                      "e2e-run-#{pipeline}"
-                    end
+          channel = Runtime::Env.slack_alerts_channel
 
           slack_options = {
             slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
@@ -58,7 +51,7 @@ module GitlabQuality
             username: "GitLab Quality Test Tooling",
             icon_emoji: ':ci_failing:',
             message: <<~MSG
-              An unexpected error occurred while reporting test results in issues.
+              Env: #{pipeline}. An unexpected error occurred while reporting test results in issues.
               The error occurred in job: #{Runtime::Env.ci_job_url}
               `#{error.class.name} #{error.message}`
             MSG

data/lib/gitlab_quality/test_tooling/gitlab_client/group_labels_client.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module GitlabClient
+      class GroupLabelsClient < GitlabClient
+        def initialize(token:, group:, endpoint: nil, **_kwargs)
+          @token = token
+          @group = group
+          @endpoint = endpoint
+        end
+
+        def group_labels(options: {})
+          client.group_labels(group, options)
+        end
+
+        def create_group_label(name:, color: '#428BCA', description: nil)
+          client.create_group_label(group, name, color, description: description)
+        end
+
+        private
+
+        attr_reader :group, :token, :endpoint
+
+        def client
+          @client ||= Gitlab.client(
+            endpoint: endpoint || ENV['GITLAB_API_BASE'] || Runtime::Env.gitlab_api_base,
+            private_token: token
+          )
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb

@@ -68,7 +68,7 @@ module GitlabQuality
 
         def find_issue_notes(iid:)
           handle_gitlab_client_exceptions do
-            client.issue_notes(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
+            client.issue_notes(project, iid, order_by: 'created_at', sort: 'asc', activity_filter: 'only_comments').auto_paginate
           end
         end
 

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb

@@ -4,8 +4,8 @@ module GitlabQuality
   module TestTooling
     module GitlabClient
       class IssuesDryClient < IssuesClient
-        def create_issue(title:, description:, labels:, issue_type: 'issue', _assignee_id: nil, _due_date: nil, confidential: false)
-          attrs = { description: description, labels: labels, confidential: confidential }
+        def create_issue(title:, description:, labels:, assignee_id: 'unknown', issue_type: 'issue', confidential: false)
+          attrs = { description: description, labels: labels, confidential: confidential, assignee_id: assignee_id }
 
           puts "The following #{issue_type} would have been created:"
           puts "project: #{project}, title: #{title}, attrs: #{attrs}"

data/lib/gitlab_quality/test_tooling/report/failed_test_issue.rb

@@ -12,7 +12,7 @@ module GitlabQuality
       #   - Add a failure report in the "Failure reports" note
       class FailedTestIssue < HealthProblemReporter
         IDENTITY_LABELS       = ['test', 'test-health:failures', 'automation:bot-authored'].freeze
-        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'failure::new', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'failure::new', 'priority::3', 'severity::3', 'suppress-contributor-links', *IDENTITY_LABELS]).freeze
         REPORT_SECTION_HEADER = '#### Failure reports'
 
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -13,10 +13,10 @@ module GitlabQuality
       #   - Add a flakiness report in the "Flakiness reports" note
       class FlakyTestIssue < HealthProblemReporter
         IDENTITY_LABELS       = ['test', 'failure::flaky-test', 'test-health:pass-after-retry', 'automation:bot-authored'].freeze
-        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', 'suppress-contributor-links', *IDENTITY_LABELS]).freeze
         REPORT_SECTION_HEADER = '### Flakiness reports'
         REPORTS_DOCUMENTATION = <<~DOC
-          Flaky tests were detected. Please refer to the [Flaky tests reproducibility instructions](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#how-to-reproduce-a-flaky-test-locally)
+          Flaky tests were detected. Please refer to the [Flaky tests reproducibility instructions](https://docs.gitlab.com/development/testing_guide/unhealthy_tests/#how-to-reproduce-a-flaky-test-locally)
           to learn more about how to reproduce them.
         DOC
 

data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb

@@ -31,9 +31,9 @@ module GitlabQuality
           tests = tests.select { |test| pipeline_stages.include? test.report["stage"] } unless pipeline_stages.empty?
 
           issue = gitlab.create_issue(
-            title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
+            title: "#{Time.now.to_date.iso8601} Test session report | #{Runtime::Env.qa_run_type}",
             description: generate_description(tests),
-            labels: ['automation:bot-authored', 'E2E', 'triage report', pipeline_name_label],
+            labels: ['automation:bot-authored', 'E2E', 'triage report', pipeline_name_label, 'suppress-contributor-links'],
             confidential: confidential
           )
 

data/lib/gitlab_quality/test_tooling/report/group_issues/error_message_normalizer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'openssl'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class ErrorMessageNormalizer
+          NORMALIZATION_PATTERNS = [
+            { pattern: /\d{4}-\d{2}-\d{2}T?[ ]?\d{2}:\d{2}:\d{2}(\.\d+)?Z?/, replacement: "<TIMESTAMP>" },
+            { pattern: /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i, replacement: "<UUID>" },
+            { pattern: /Correlation Id: [\w]+/, replacement: "Correlation Id: <UUID>" },
+            { pattern: /Fabrication of QA::Resource::[A-Za-z:]+/, replacement: "Fabrication of QA::Resource::<RESOURCE>" },
+            { pattern: /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?\b/, replacement: "<IP>" },
+            { pattern: /user\d+/, replacement: "<USER>" },
+            { pattern: /group\d+/, replacement: "<GROUP>" },
+            { pattern: /project\d+/, replacement: "<PROJECT>" },
+            { pattern: %r{https?://[^/\s]+/[^\s]*}, replacement: "<URL>" },
+            { pattern: %r{/tmp/[^\s]+}, replacement: "<TMPFILE>" },
+            { pattern: %r{/var/[^\s]+}, replacement: "<VARFILE>" },
+            { pattern: /token=[^\s&]+/, replacement: "token=<TOKEN>" },
+            { pattern: /after \d+ seconds/, replacement: "after <N> seconds" },
+            { pattern: /waited \d+ seconds/, replacement: "waited <N> seconds" },
+            { pattern: /\d+ attempts?/, replacement: "<N> attempts" },
+            { pattern: /\s+/, replacement: " " }
+          ].freeze
+
+          def normalize(message)
+            return "" if message.nil? || message.empty?
+
+            result = message.dup.strip
+
+            NORMALIZATION_PATTERNS.each do |pattern_rule|
+              result.gsub!(pattern_rule[:pattern], pattern_rule[:replacement])
+            end
+
+            result.strip
+          end
+
+          def create_fingerprint(normalized_message)
+            OpenSSL::Digest::SHA256.hexdigest(normalized_message.downcase)[0..15]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/error_pattern_matcher.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class ErrorPatternMatcher
+          ENVIRONMENT_ERROR_PATTERNS = [
+            { name: "http_500_api_fabrication", pattern: /Fabrication of .+ using the API failed \(500\)/i },
+            { name: "http_500_internal_server", pattern: /(500 Internal Server Error|request returned \(500\)|Expected \(200\), request returned \(500\))/i },
+            { name: "http_400_backend_failing", pattern: /failed \(400\) with.+connections to all backends failing/i },
+            { name: "http_503_service_unavailable", pattern: /Unexpected status code 503/i },
+            { name: "pipeline_creation_timeout", pattern: /Wait for pipeline to be created failed after \d+ seconds/i },
+            { name: "event_timeout", pattern: /(Timed out waiting for event|EventNotFoundError: Timed out waiting)/i },
+            { name: "git_rpc_failure", pattern: /error: RPC failed; HTTP 500/i },
+            { name: "repository_fabricate_error", pattern: /Repository fabricate/i }
+          ].freeze
+
+          def match(error_message)
+            return nil if error_message.nil? || error_message.empty?
+
+            ENVIRONMENT_ERROR_PATTERNS.find { |pattern_def| error_message.match?(pattern_def[:pattern]) }
+          end
+
+          def environment_error?(error_message)
+            !match(error_message).nil?
+          end
+
+          def pattern_name(error_message)
+            match(error_message)&.dig(:name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/failure_processor.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class FailureProcessor
+          DEFAULT_MIN_FAILURES = 2
+
+          def initialize(options = {})
+            @options = options
+            @pattern_matcher = options[:pattern_matcher] || ErrorPatternMatcher.new
+            @normalizer = options[:normalizer] || ErrorMessageNormalizer.new
+            @config = options[:config] || {}
+          end
+
+          def process_failures(failures, &)
+            Runtime::Logger.info "Processing #{failures.size} failures for grouping..."
+            grouped_failures = {}
+
+            failures.each do |failure|
+              process_single_failure(failure, grouped_failures)
+            end
+
+            Runtime::Logger.info "Found #{grouped_failures.size} groups before filtering"
+            grouped_failures.each_value(&)
+          end
+
+          def filter_groups_by_threshold(grouped_failures)
+            min_failures = @config.dig(:thresholds, :min_failures_to_group) || DEFAULT_MIN_FAILURES
+
+            grouped_failures.select! do |_fingerprint, grouped_failure|
+              grouped_failure[:failures].size >= min_failures
+            end
+
+            Runtime::Logger.info "Found #{grouped_failures.size} groups after filtering"
+          end
+
+          private
+
+          def process_single_failure(failure, grouped_failures)
+            error_message = failure.dig(:exception, 'message') || failure.dig(:exceptions, 0, 'message')
+            Runtime::Logger.info "Processing failure: #{failure[:description]}"
+            Runtime::Logger.info "Error message: #{error_message[0..100]}..." if error_message
+
+            return unless error_message && @pattern_matcher.environment_error?(error_message)
+
+            Runtime::Logger.info "Identified as environment error"
+            group_environment_failure(failure, error_message, grouped_failures)
+          end
+
+          def group_environment_failure(failure, error_message, grouped_failures)
+            normalized_message = @normalizer.normalize(error_message)
+            fingerprint = @normalizer.create_fingerprint(normalized_message)
+            pattern_name = @pattern_matcher.pattern_name(error_message)
+
+            grouped_failures[fingerprint] ||= build_grouped_failure(fingerprint, pattern_name, normalized_message)
+            grouped_failures[fingerprint][:failures] << failure
+          end
+
+          def build_grouped_failure(fingerprint, pattern_name, normalized_message)
+            {
+              fingerprint: fingerprint,
+              pattern_name: pattern_name,
+              normalized_message: normalized_message,
+              failures: []
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/group_results_in_issues.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+require 'time'
+require_relative 'error_pattern_matcher'
+require_relative 'error_message_normalizer'
+require_relative 'incident_checker'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class GroupResultsInIssues
+          attr_reader :grouped_failures
+
+          def initialize(options = {})
+            @options = options
+            @failure_processor = FailureProcessor.new(options)
+            @issue_manager = IssueManager.new(options)
+            @grouped_failures = {}
+          end
+
+          def process_failures(failures)
+            @failure_processor.process_failures(failures) do |grouped_failure|
+              fingerprint = grouped_failure[:fingerprint]
+              @grouped_failures[fingerprint] = grouped_failure
+            end
+
+            @failure_processor.filter_groups_by_threshold(@grouped_failures)
+          end
+
+          def process_issues
+            @grouped_failures.each_value do |grouped_failure|
+              @issue_manager.create_or_update_issue(grouped_failure)
+            end
+          end
+
+          def summary
+            {
+              grouped_issues: @grouped_failures.size,
+              total_grouped_failures: @grouped_failures.values.sum { |group| group[:failures].size }
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/incident_checker.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IncidentChecker
+          GITLAB_PRODUCTION_PROJECT_ID = '7444821' # gitlab-com/gl-infra/production
+
+          def self.get_active_incidents(token: nil, gitlab_url: 'https://gitlab.com')
+            return [] unless token
+
+            begin
+              client = GitlabClient::IssuesClient.new(token: token, endpoint: "#{gitlab_url}/api/v4", project: GITLAB_PRODUCTION_PROJECT_ID)
+
+              issues = client.find_issues(options: {
+                labels: 'Incident::Active',
+                state: 'opened',
+                per_page: 10,
+                order_by: 'created_at',
+                sort: 'desc'
+              })
+
+              issues.map do |issue|
+                {
+                  title: issue.title,
+                  url: issue.web_url
+                }
+              end
+
+            rescue Gitlab::Error::Error => e
+              Runtime::Logger.error "GitLab API error fetching incidents: #{e.message}"
+              []
+            rescue StandardError => e
+              Runtime::Logger.error "Warning: Could not fetch active incidents: #{e.message}"
+              []
+            end
+          end
+
+          def self.format_incidents_for_issue(incidents)
+            return "" if incidents.empty?
+
+            incident_list = incidents.map { |inc| "- [#{inc[:title]}](#{inc[:url]})" }.join("\n")
+
+            <<~MARKDOWN
+
+        ### Related GitLab Incidents
+        The following active incidents may be related to these test failures:
+
+        #{incident_list}
+
+        Check the [GitLab Production Issues](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/?label_name%5B%5D=Incident%3A%3AActive) for updates.
+            MARKDOWN
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_base.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueBase
+          def initialize(client, options = {})
+            @client = client
+            @options = options
+            @gitlab_url = ENV.fetch('CI_SERVER_URL', 'https://gitlab.com')
+            @project_id = options[:target_project] || ENV['RESULTS_ISSUE_PROJECT'] || ENV.fetch('CI_PROJECT_ID', nil)
+            @token = options[:token]
+          end
+
+          private
+
+          def handle_gitlab_api_error(operation, context = nil)
+            yield
+          rescue Gitlab::Error::Error => e
+            log_gitlab_error(operation, context, e)
+            nil
+          rescue StandardError => e
+            log_standard_error(operation, context, e)
+            nil
+          end
+
+          def log_gitlab_error(operation, context, error)
+            context_info = context ? " #{context}" : ""
+            Runtime::Logger.error "GitLab API error #{operation}#{context_info}: #{error.message}"
+          end
+
+          def log_standard_error(operation, context, error)
+            context_info = context ? " #{context}" : ""
+            Runtime::Logger.error "Error #{operation}#{context_info}: #{error.message}"
+          end
+
+          def display_description_preview(description, title = "Description preview:")
+            Runtime::Logger.info title
+            lines = description.split("\n")
+            Runtime::Logger.info lines.first(15).join("\n")
+            Runtime::Logger.info "..." if lines.length > 15
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_creator.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueCreator < IssueBase
+          GROUPED_ISSUE_LABELS = Set.new(%w[test failure::test-environment automation:bot-authored type::maintenance]).freeze
+
+          def initialize(client, options = {})
+            super
+            @formatter = IssueFormatter.new
+          end
+
+          def create_new_issue(grouped_failure)
+            title = @formatter.generate_issue_title(grouped_failure)
+            description = @formatter.generate_issue_description(grouped_failure, @options)
+            labels = GROUPED_ISSUE_LABELS
+
+            Runtime::Logger.info "Creating new grouped issue: #{title} (#{grouped_failure[:failures].size} failures)"
+
+            create_issue(
+              title: title,
+              description: description,
+              labels: labels,
+              failures: grouped_failure[:failures]
+            )
+          end
+
+          private
+
+          def create_issue(title:, description:, labels:, failures:)
+            Runtime::Logger.info "Creating issue: #{title} with #{failures.size} failures"
+
+            handle_gitlab_api_error("creating issue") do
+              issue = @client.create_issue(title: title, description: description, labels: labels.join(','))
+              Runtime::Logger.info "Issue created successfully: #{issue.web_url}" if issue&.web_url
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/issue_finder.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class IssueFinder
+          DEFAULT_MAX_AGE_HOURS = 24
+          ISSUES_PER_PAGE = 50
+
+          def initialize(client, options = {})
+            @client = client
+            @options = options
+            @token = options[:token]
+            @project_id = options[:target_project] || ENV['RESULTS_ISSUE_PROJECT'] || ENV.fetch('CI_PROJECT_ID', nil)
+          end
+
+          def find_existing_issue(grouped_failure)
+            find_related_issue_by_fingerprint(
+              fingerprint: grouped_failure[:fingerprint],
+              max_age_hours: DEFAULT_MAX_AGE_HOURS
+            )
+          end
+
+          private
+
+          def find_related_issue_by_fingerprint(fingerprint:, max_age_hours:)
+            Runtime::Logger.info "Searching for existing issue with fingerprint: #{fingerprint}"
+
+            begin
+              issues = fetch_recent_open_issues(max_age_hours)
+              matching_issue = find_issue_with_fingerprint(issues, fingerprint)
+              if matching_issue
+                Runtime::Logger.info "Found existing issue: ##{matching_issue.iid} - #{matching_issue.title}"
+                convert_to_struct(matching_issue)
+              end
+            rescue Gitlab::Error::Error => e
+              Runtime::Logger.error "GitLab API error searching for issues: #{e.message}"
+              nil
+            rescue StandardError => e
+              Runtime::Logger.error "Error searching for existing issues: #{e.message}"
+              nil
+            end
+          end
+
+          def fetch_recent_open_issues(max_age_hours)
+            cutoff_time = Time.now - (max_age_hours * 3600)
+
+            @client.find_issues(options: {
+              state: 'opened',
+              created_after: cutoff_time.utc.iso8601,
+              per_page: ISSUES_PER_PAGE,
+              order_by: 'created_at',
+              sort: 'desc'
+            })
+          end
+
+          def find_issue_with_fingerprint(issues, fingerprint)
+            fingerprint_tag = "grouped-failure-fingerprint:#{fingerprint}"
+            issues.find { |issue| issue.description&.include?(fingerprint_tag) }
+          end
+
+          def convert_to_struct(obj)
+            return obj unless obj.is_a?(Hash)
+
+            struct_class = Struct.new(*obj.keys.map(&:to_sym))
+            struct = struct_class.new
+
+            obj.each do |key, value|
+              struct[key.to_sym] = value.is_a?(Hash) ? convert_to_struct(value) : value
+            end
+
+            struct
+          end
+        end
+      end
+    end
+  end
+end