gitlab_quality-test_tooling 2.10.0 → 2.18.0

data/lib/gitlab_quality/test_tooling/feature_readiness/operational_readiness_check.rb

@@ -14,7 +14,7 @@ module GitlabQuality
 
         def initialize(token:, project: nil, group: nil, limit_to_minutes: nil, search_labels: [], issue_is_blocking: false, dry_run: false)
           @token = token
-          @project = project
+          @project = "#{group}/#{project}"
           @group = group
           @limit_to_minutes = limit_to_minutes
           @search_labels = search_labels
@@ -26,7 +26,7 @@ module GitlabQuality
           created_after = utc_time_minus_mins(limit_to_minutes)
 
           epics = work_items_client.paginated_call(:group_work_items,
-            labels: search_labels.concat(BASE_LABELS_FOR_SEARCH), state: 'opened', created_after: created_after, extras: [:work_item_fields])
+            labels: search_labels.concat(BASE_LABELS_FOR_SEARCH).uniq, state: 'opened', created_after: created_after, extras: [:work_item_fields])
 
           epics.each do |epic|
             process_epic(epic)
@@ -58,11 +58,11 @@ module GitlabQuality
         end
 
         def process_epic(epic) # rubocop:disable Metrics/AbcSize
-          epic = fetch_work_item(epic[:iid], work_items_client)
+          epic = fetch_work_item(epic[:iid], work_items_client, [:notes, :linked_items, :labels, :hierarchy])
 
           return if has_a_child_epic?(epic)
 
-          pre_check_comment = add_operational_readiness_precheck_comment(epic, work_items_client)
+          pre_check_comment = add_operational_readiness_precheck_comment(epic, work_items_client, label_client)
 
           return unless note_has_emoji?(pre_check_comment, 'white_check_mark') && !has_operational_readiness_issue_linked?(linked_issue_iids(epic), issue_client)
 

data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb

@@ -72,6 +72,12 @@ module GitlabQuality
           end
         end
 
+        def related_merge_requests(iid:)
+          handle_gitlab_client_exceptions do
+            client.related_merge_requests(project, iid).auto_paginate
+          end
+        end
+
         def find_issue_discussions(iid:)
           handle_gitlab_client_exceptions do
             client.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
@@ -169,7 +175,7 @@ module GitlabQuality
         def find_commit_parent(project, sha)
           handle_gitlab_client_exceptions do
             # In a merged results commit, the first parent is the one from
-            # the main branch, and the second parent is from the branch
+            # the default branch, and the second parent is from the branch
             # itself (more likely to have caused the issue)
             client.commit(project, sha).parent_ids.last
           end

data/lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_client.rb

@@ -1,5 +1,16 @@
 # frozen_string_literal: true
 
+require 'gitlab'
+
+module Gitlab
+  # Monkey patch the Gitlab client to allow passing query options
+  class Client
+    def merge_request_diffs(project, merge_request_iid, options = {})
+      get("/projects/#{url_encode(project)}/merge_requests/#{merge_request_iid}/diffs", query: options).auto_paginate
+    end
+  end
+end
+
 module GitlabQuality
   module TestTooling
     module GitlabClient
@@ -10,6 +21,12 @@ module GitlabQuality
           end
         end
 
+        def merge_request_diffs(merge_request_iid:)
+          handle_gitlab_client_exceptions do
+            client.merge_request_diffs(project, merge_request_iid, per_page: 100)
+          end
+        end
+
         def create_merge_request(title:, source_branch:, target_branch:, description:, labels:, assignee_id: nil, reviewer_ids: [])
           attrs = {
             source_branch: source_branch,
@@ -33,6 +50,10 @@ module GitlabQuality
           merge_request
         end
 
+        def merge_request(id:, options: {})
+          client.merge_request(project, id, options)
+        end
+
         def find(iid: nil, options: {}, &select)
           select ||= :itself
 

data/lib/gitlab_quality/test_tooling/gitlab_client/merge_requests_dry_client.rb

@@ -4,16 +4,6 @@ module GitlabQuality
   module TestTooling
     module GitlabClient
       class MergeRequestsDryClient < MergeRequestsClient
-        def find_merge_request_changes(merge_request_iid:)
-          puts "Finding changes for merge_request_id #{merge_request_iid}"
-          puts "project: #{project}"
-        end
-
-        def merge_request_changed_files(merge_request_iid:)
-          puts "Changed files for #{merge_request_iid}"
-          []
-        end
-
         def find_note(body:, merge_request_iid:)
           puts "Find note for #{merge_request_iid} with body: #{body} for mr_iid: #{merge_request_iid}"
         end

data/lib/gitlab_quality/test_tooling/gitlab_client/work_items_client.rb

@@ -3,15 +3,14 @@
 module GitlabQuality
   module TestTooling
     module GitlabClient
-      # The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
       class WorkItemsClient < GitlabGraphqlClient
-        def work_item(workitem_iid: nil)
+        def work_item(workitem_iid:, widgets: [:notes, :linked_items, :labels, :hierarchy])
           query = <<~GQL
             query {
               namespace(fullPath: "#{group}") {
                 workItem(iid: "#{workitem_iid}") {
                   #{work_item_fields}
-                  #{work_item_widgets}
+                  #{work_item_widgets(widgets)}
                 }
               }
             }
@@ -19,7 +18,7 @@ module GitlabQuality
           post(query)[:workItem]
         end
 
-        def group_work_items(labels: [], cursor: '', state: 'opened', created_after: nil, extras: [])
+        def group_work_items(labels: [], cursor: '', state: 'opened', created_after: nil, extras: [:work_item_fields])
           query = <<~GQL
             query {
               group(fullPath: "#{group}") {
@@ -73,6 +72,18 @@ module GitlabQuality
           post(query)
         end
 
+        def update_note(note_id:, body:)
+          query = <<~GQL
+            mutation UpdateNote {
+                updateNote(input: { body: "#{body}", id: "#{note_id}" }) {
+                    clientMutationId
+                    errors
+                }
+            }
+          GQL
+          post(query)
+        end
+
         def create_linked_items(work_item_id:, item_ids:, link_type:)
           query = <<~GQL
             mutation WorkItemAddLinkedItems {
@@ -160,50 +171,76 @@ module GitlabQuality
           GQL
         end
 
-        def work_item_widgets
+        def work_item_widget_notes
           <<~GQL
-            widgets(onlyTypes: [LINKED_ITEMS, NOTES, LABELS, HIERARCHY]) {
-              ... on WorkItemWidgetNotes {
-                discussions(filter: ONLY_COMMENTS) {
-                  nodes {
-                          notes {
-                            nodes {
-                            #{note_fields}
-                            }
+            ... on WorkItemWidgetNotes {
+              discussions(filter: ONLY_COMMENTS) {
+                nodes {
+                        notes {
+                          nodes {
+                          #{note_fields}
                           }
                         }
-                }
-              }
-              ... on WorkItemWidgetLinkedItems {
-                  linkedItems {
-                      nodes {
-                          linkType
-                          workItem {
-                              #{work_item_fields}
-                          }
                       }
-                  }
               }
+            }
+          GQL
+        end
 
-              ... on WorkItemWidgetLabels{
-                labels{
-                   nodes{
-                       title
-                   }
+        def work_item_widget_linked_items
+          <<~GQL
+            ... on WorkItemWidgetLinkedItems {
+                linkedItems {
+                    nodes {
+                        linkType
+                        workItem {
+                            #{work_item_fields}
+                        }
+                    }
                 }
+            }
+          GQL
+        end
+
+        def work_item_widget_labels
+          <<~GQL
+            ... on WorkItemWidgetLabels{
+              labels{
+                 nodes{
+                     title
+                 }
               }
+            }
+          GQL
+        end
 
-              ... on WorkItemWidgetHierarchy {
-                children {
-                    nodes{
-                     #{work_item_fields}
-                    }
-                }
+        def work_item_widget_hierarchy
+          <<~GQL
+            ... on WorkItemWidgetHierarchy {
+              children {
+                  nodes{
+                   #{work_item_fields}
+                  }
               }
             }
           GQL
         end
 
+        def work_item_widgets(widgets = [])
+          <<~GQL
+            widgets(onlyTypes: [#{types_for_widgets(widgets)}]) {
+              #{work_item_widget_notes if widgets.include?(:notes)}
+              #{work_item_widget_linked_items if widgets.include?(:linked_items)}
+              #{work_item_widget_labels if widgets.include?(:labels)}
+              #{work_item_widget_hierarchy if widgets.include?(:hierarchy)}
+            }
+          GQL
+        end
+
+        def types_for_widgets(widgets = [])
+          widgets.map(&:upcase).join(', ')
+        end
+
         # https://docs.gitlab.com/api/graphql/reference/#note
         def note_fields
           <<~GQL

data/lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb

@@ -36,7 +36,7 @@ module GitlabQuality
           end
 
           def new_issue_labels(_test)
-            %w[Quality status::automated]
+            %w[E2E status::automated]
           end
 
           def up_to_date_labels(test:, issue: nil, new_labels: Set.new)

data/lib/gitlab_quality/test_tooling/report/concerns/utils.rb

@@ -43,9 +43,9 @@ module GitlabQuality
             # Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
             #   nightly, staging, canary, production, and preprod
             #
-            # MR, master/main, and gitlab tests run in gitlab-qa, but we only want to report tests run on
-            # master/main because the other pipelines will be monitored by the author of the MR that triggered them.
-            # So we assume that we're reporting a master/main pipeline if the project name is 'gitlab'.
+            # MR, master, and gitlab tests run in gitlab-qa, but we only want to report tests run on
+            # master because the other pipelines will be monitored by the author of the MR that triggered them.
+            # So we assume that we're reporting a master pipeline if the project name is 'gitlab'.
 
             @pipeline ||= Runtime::Env.pipeline_from_project_name
           end

data/lib/gitlab_quality/test_tooling/report/feature_readiness/report_on_epic.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'pp'
+require 'stringio'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module FeatureReadiness
+        class ReportOnEpic
+          FEATURE_READINESS_REPORT_COMMENT_ID = '<!-- FEATURE READINESS REPORT COMMENT -->'
+
+          class << self
+            include GitlabQuality::TestTooling::FeatureReadiness::Concerns::WorkItemConcern
+
+            def report(analyzed_epic, work_item_client)
+              must_haves_report_rows = generate_report_rows(analyzed_epic, :must_haves)
+              should_haves_report_rows = generate_report_rows(analyzed_epic, :should_haves)
+
+              existing_note = existing_note_containing_text(FEATURE_READINESS_REPORT_COMMENT_ID, analyzed_epic[:epic_iid], work_item_client)
+
+              if existing_note
+                work_item_client.update_note(note_id: existing_note[:id],
+                  body: comment({ must_haves: must_haves_report_rows, should_haves: should_haves_report_rows }, analyzed_epic).tr('"', "'"))
+              else
+                work_item_client.create_discussion(id: analyzed_epic[:epic_id],
+                  note: comment({ must_haves: must_haves_report_rows, should_haves: should_haves_report_rows }, analyzed_epic).tr('"', "'"))
+              end
+            end
+
+            private
+
+            def generate_report_rows(epic, type)
+              status_checks = check_statuses(epic)
+              create_rows(epic, type, status_checks)
+            end
+
+            def create_rows(epic, type, status_checks)
+              if type == :must_haves
+                [
+                  create_documentation_row(epic, status_checks),
+                  create_feature_flag_row(epic, status_checks),
+                  create_unit_tests_coverage_row(status_checks)
+
+                ]
+              else
+                [
+                  create_feature_tests_row(epic, status_checks),
+                  create_e2e_tests_row(epic, status_checks)
+                ]
+              end
+            end
+
+            def create_documentation_row(epic, status_checks)
+              ["Documentation added?", status_icon(status_checks[:has_docs]),
+                prepend_text('Added in:', format_links(epic[:doc_mrs]))]
+            end
+
+            def create_feature_flag_row(epic, status_checks)
+              ["Feature Flag added?", status_icon(status_checks[:feature_flag_added]),
+                prepend_text('Added in:', format_links(epic[:feature_flag_mrs]))]
+            end
+
+            def create_feature_tests_row(epic, status_checks)
+              ["Feature tests added?", status_icon(status_checks[:has_feature_specs]),
+                format_links(epic[:feature_spec_mrs])]
+            end
+
+            def create_e2e_tests_row(epic, status_checks)
+              ["End-to-end tests added?", status_icon(status_checks[:has_e2e_specs]),
+                format_links(epic[:e2e_spec_mrs])]
+            end
+
+            def create_unit_tests_coverage_row(status_checks)
+              ["Unit tests coverage complete?", status_icon(status_checks[:has_complete_unit_tests]),
+                prepend_text('Coverage missing for:', format_links(status_checks[:missing_specs]))]
+            end
+
+            def prepend_text(prepend_text, text)
+              return "#{prepend_text} #{text}" unless text.empty?
+
+              text
+            end
+
+            def check_statuses(epic)
+              {
+                has_docs: epic[:doc_mrs].any?,
+                feature_flag_added: epic[:feature_flag_mrs].any?,
+                has_feature_specs: epic[:feature_spec_mrs].any?,
+                has_e2e_specs: epic[:e2e_spec_mrs].any?,
+                missing_specs: missing_spec_mrs(epic),
+                has_complete_unit_tests: missing_spec_mrs(epic).empty?
+              }
+            end
+
+            def comment(rows, epic)
+              # Generate markdown table
+              must_haves_table_rows = rows[:must_haves].map do |description, status, links|
+                "| #{description} | #{status} | #{links} |"
+              end.join("\n")
+
+              should_haves_table_rows = rows[:should_haves].map do |description, status, links|
+                "| #{description} | #{status} | #{links} |"
+              end.join("\n")
+
+              <<~COMMENT
+                #{FEATURE_READINESS_REPORT_COMMENT_ID}
+
+                # :vertical_traffic_light: Feature Readiness Evaluation Report
+
+                ### :octagonal_sign: Must haves
+
+                | Evaluation | Result | Notes |
+                |------------|--------|-------|
+                #{must_haves_table_rows}
+
+                ### :warning: Should haves
+
+                | Evaluation | Result | Notes |
+                |------------|--------|-------|
+                #{should_haves_table_rows}
+
+                #{data(epic)}
+
+                ---
+
+                _Please note that this automation is under testing. Please add any feedback on [this issue](https://gitlab.com/gitlab-org/quality/quality-engineering/team-tasks/-/issues/3587)._
+
+              COMMENT
+            end
+
+            def status_icon(condition)
+              condition ? ':white_check_mark:' : ':x:'
+            end
+
+            def format_links(data)
+              return '' if data.empty?
+
+              data.map do |item|
+                item.map { |key, url| "[#{key}](#{url})" }.first
+              end.join(", ")
+            end
+
+            def missing_spec_mrs(epic)
+              epic[:issues].flat_map do |issue|
+                issue[:merge_requests].flat_map do |mr|
+                  next [] unless mr[:files_with_missing_specs]&.any?
+
+                  mr[:files_with_missing_specs].map do |file|
+                    { file => mr[:merge_request_web_url] }
+                  end
+                end.compact
+              end
+            end
+
+            def data(epic)
+              output = StringIO.new
+              PP.pp(epic, output)
+              <<~DATA
+                <details><summary>Expand for data</summary>
+
+                ```ruby
+                #{output.string}
+                ```
+
+                </details>
+              DATA
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -16,7 +16,7 @@ module GitlabQuality
         NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
         REPORT_SECTION_HEADER = '### Flakiness reports'
         REPORTS_DOCUMENTATION = <<~DOC
-          Flaky tests were detected. Please refer to the [Flaky tests reproducibility instructions](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#how-to-reproduce-a-flaky-test-locally)
+          Flaky tests were detected. Please refer to the [Flaky tests reproducibility instructions](https://docs.gitlab.com/development/testing_guide/unhealthy_tests/#how-to-reproduce-a-flaky-test-locally)
           to learn more about how to reproduce them.
         DOC
 

data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb

@@ -33,7 +33,7 @@ module GitlabQuality
           issue = gitlab.create_issue(
             title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
             description: generate_description(tests),
-            labels: ['automation:bot-authored', 'Quality', 'QA', 'triage report', pipeline_name_label],
+            labels: ['automation:bot-authored', 'E2E', 'triage report', pipeline_name_label],
             confidential: confidential
           )
 

data/lib/gitlab_quality/test_tooling/report/group_issues/error_message_normalizer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'openssl'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class ErrorMessageNormalizer
+          NORMALIZATION_PATTERNS = [
+            { pattern: /\d{4}-\d{2}-\d{2}T?[ ]?\d{2}:\d{2}:\d{2}(\.\d+)?Z?/, replacement: "<TIMESTAMP>" },
+            { pattern: /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i, replacement: "<UUID>" },
+            { pattern: /Correlation Id: [\w]+/, replacement: "Correlation Id: <UUID>" },
+            { pattern: /Fabrication of QA::Resource::[A-Za-z:]+/, replacement: "Fabrication of QA::Resource::<RESOURCE>" },
+            { pattern: /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?\b/, replacement: "<IP>" },
+            { pattern: /user\d+/, replacement: "<USER>" },
+            { pattern: /group\d+/, replacement: "<GROUP>" },
+            { pattern: /project\d+/, replacement: "<PROJECT>" },
+            { pattern: %r{https?://[^/\s]+/[^\s]*}, replacement: "<URL>" },
+            { pattern: %r{/tmp/[^\s]+}, replacement: "<TMPFILE>" },
+            { pattern: %r{/var/[^\s]+}, replacement: "<VARFILE>" },
+            { pattern: /token=[^\s&]+/, replacement: "token=<TOKEN>" },
+            { pattern: /after \d+ seconds/, replacement: "after <N> seconds" },
+            { pattern: /waited \d+ seconds/, replacement: "waited <N> seconds" },
+            { pattern: /\d+ attempts?/, replacement: "<N> attempts" },
+            { pattern: /\s+/, replacement: " " }
+          ].freeze
+
+          def normalize(message)
+            return "" if message.nil? || message.empty?
+
+            result = message.dup.strip
+
+            NORMALIZATION_PATTERNS.each do |pattern_rule|
+              result.gsub!(pattern_rule[:pattern], pattern_rule[:replacement])
+            end
+
+            result.strip
+          end
+
+          def create_fingerprint(normalized_message)
+            OpenSSL::Digest::SHA256.hexdigest(normalized_message.downcase)[0..15]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/error_pattern_matcher.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class ErrorPatternMatcher
+          ENVIRONMENT_ERROR_PATTERNS = [
+            { name: "http_500_api_fabrication", pattern: /Fabrication of .+ using the API failed \(500\)/i },
+            { name: "http_500_internal_server", pattern: /(500 Internal Server Error|request returned \(500\)|Expected \(200\), request returned \(500\))/i },
+            { name: "http_400_backend_failing", pattern: /failed \(400\) with.+connections to all backends failing/i },
+            { name: "http_503_service_unavailable", pattern: /Unexpected status code 503/i },
+            { name: "pipeline_creation_timeout", pattern: /Wait for pipeline to be created failed after \d+ seconds/i },
+            { name: "event_timeout", pattern: /(Timed out waiting for event|EventNotFoundError: Timed out waiting)/i },
+            { name: "git_rpc_failure", pattern: /error: RPC failed; HTTP 500/i },
+            { name: "repository_fabricate_error", pattern: /Repository fabricate/i }
+          ].freeze
+
+          def match(error_message)
+            return nil if error_message.nil? || error_message.empty?
+
+            ENVIRONMENT_ERROR_PATTERNS.find { |pattern_def| error_message.match?(pattern_def[:pattern]) }
+          end
+
+          def environment_error?(error_message)
+            !match(error_message).nil?
+          end
+
+          def pattern_name(error_message)
+            match(error_message)&.dig(:name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/group_issues/failure_processor.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module GroupIssues
+        class FailureProcessor
+          DEFAULT_MIN_FAILURES = 2
+
+          def initialize(options = {})
+            @options = options
+            @pattern_matcher = options[:pattern_matcher] || ErrorPatternMatcher.new
+            @normalizer = options[:normalizer] || ErrorMessageNormalizer.new
+            @config = options[:config] || {}
+          end
+
+          def process_failures(failures, &)
+            Runtime::Logger.info "Processing #{failures.size} failures for grouping..."
+            grouped_failures = {}
+
+            failures.each do |failure|
+              process_single_failure(failure, grouped_failures)
+            end
+
+            Runtime::Logger.info "Found #{grouped_failures.size} groups before filtering"
+            grouped_failures.each_value(&)
+          end
+
+          def filter_groups_by_threshold(grouped_failures)
+            min_failures = @config.dig(:thresholds, :min_failures_to_group) || DEFAULT_MIN_FAILURES
+
+            grouped_failures.select! do |_fingerprint, grouped_failure|
+              grouped_failure[:failures].size >= min_failures
+            end
+
+            Runtime::Logger.info "Found #{grouped_failures.size} groups after filtering"
+          end
+
+          private
+
+          def process_single_failure(failure, grouped_failures)
+            error_message = failure.dig(:exception, 'message') || failure.dig(:exceptions, 0, 'message')
+            Runtime::Logger.info "Processing failure: #{failure[:description]}"
+            Runtime::Logger.info "Error message: #{error_message[0..100]}..." if error_message
+
+            return unless error_message && @pattern_matcher.environment_error?(error_message)
+
+            Runtime::Logger.info "Identified as environment error"
+            group_environment_failure(failure, error_message, grouped_failures)
+          end
+
+          def group_environment_failure(failure, error_message, grouped_failures)
+            normalized_message = @normalizer.normalize(error_message)
+            fingerprint = @normalizer.create_fingerprint(normalized_message)
+            pattern_name = @pattern_matcher.pattern_name(error_message)
+
+            grouped_failures[fingerprint] ||= build_grouped_failure(fingerprint, pattern_name, normalized_message)
+            grouped_failures[fingerprint][:failures] << failure
+          end
+
+          def build_grouped_failure(fingerprint, pattern_name, normalized_message)
+            {
+              fingerprint: fingerprint,
+              pattern_name: pattern_name,
+              normalized_message: normalized_message,
+              failures: []
+            }
+          end
+        end
+      end
+    end
+  end
+end