gitlab_quality-test_tooling 2.10.0 → 2.18.0

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -3,6 +3,10 @@
 require 'nokogiri'
 require 'rubygems/text'
 
+require_relative 'group_issues/error_pattern_matcher'
+require_relative 'group_issues/error_message_normalizer'
+require_relative 'group_issues/group_results_in_issues'
+
 module GitlabQuality
   module TestTooling
     module Report
@@ -12,6 +16,7 @@ module GitlabQuality
       # - Takes a project where failure issues should be created
       # - Find issue by title (with test description or test file), then further filter by stack trace, then pick the better-matching one
       # - Add the failed job to the issue description, and update labels
+      # - Can group similar failures together when group_similar option is enabled
       class RelateFailureIssue < ReportAsIssue
         include TestTooling::Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
@@ -23,16 +28,23 @@ module GitlabQuality
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*\n###/m
 
-        NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2 automation:bot-authored]).freeze
+        NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2 automation:bot-authored type::maintenance]).freeze
         SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
-
+        FAILURE_ISSUE_GUIDE_URL = "https://handbook.gitlab.com/handbook/engineering/testing/guide-to-e2e-test-failure-issues/"
+        FAILURE_ISSUE_HANDBOOK_GUIDE = "**:rotating_light: [End-to-End Test Failure Issue Debugging Guide](#{FAILURE_ISSUE_GUIDE_URL}) :rotating_light:**\n".freeze
         # Map commits to security fork (gitlab-org/security/gitlab) for gitlab-org/gitlab since security patches exist
         # there before being released to the public repository
         DIFF_PROJECT_MAPPINGS = {
+          'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/security/gitlab',
           'gitlab-org/gitlab' => 'gitlab-org/security/gitlab',
           'gitlab-org/customers-gitlab-com' => 'gitlab-org/customers-gitlab-com'
         }.freeze
 
+        # Don't use the E2E test issues project for commit parent
+        COMMIT_PROJECT_MAPPINGS = {
+          'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/gitlab'
+        }.freeze
+
         # The project contains record of the deployments we use to determine the commit diff
         OPS_RELEASES_METADATA_PROJECT = 'gitlab-org/release/metadata'
 
@@ -56,6 +68,7 @@ module GitlabQuality
           base_issue_labels: nil,
           exclude_labels_for_search: nil,
           metrics_files: [],
+          group_similar: false,
           **kwargs)
           super
           @max_diff_ratio = max_diff_ratio.to_f
@@ -65,19 +78,106 @@ module GitlabQuality
           @issue_type = 'issue'
           @commented_issue_list = Set.new
           @metrics_files = Array(metrics_files)
+          @group_similar = group_similar
         end
 
         private
 
-        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client
+        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client, :group_similar
 
         def run!
           puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
+          if group_similar
+            run_with_grouping!
+          else
+            TestResults::Builder.new(token: token, project: project, file_glob: files).test_results_per_file do |test_results|
+              puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
+              process_test_results(test_results)
+            end
+          end
+        end
+
+        def grouper
+          @grouper ||= GitlabQuality::TestTooling::Report::GroupIssues::GroupResultsInIssues.new(
+            gitlab: gitlab,
+            config: grouper_config
+          )
+        end
+
+        def run_with_grouping!
+          Runtime::Logger.info "=> Grouping similar failures where possible"
+
+          all_test_results = collect_all_test_results
+          return if all_test_results.empty?
+
+          Runtime::Logger.info "=> Processing #{all_test_results.count} failures with GroupResultsInIssues"
+
+          failure_data = convert_test_results_to_failure_data(all_test_results)
+
+          grouper.process_failures(failure_data)
+          grouper.process_issues
+        end
+
+        def collect_all_test_results
+          all_test_results = []
+
           TestResults::Builder.new(token: token, project: project, file_glob: files).test_results_per_file do |test_results|
-            puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
-            process_test_results(test_results)
+            Runtime::Logger.info "=> Collecting #{test_results.count} tests from #{test_results.path}"
+            all_test_results.concat(test_results.select(&:failures?))
           end
+
+          all_test_results
+        end
+
+        def convert_test_results_to_failure_data(test_results)
+          test_results.map do |test|
+            {
+              description: test.name,
+              full_description: test.name,
+              file_path: test.relative_file,
+              line_number: extract_line_number(test),
+              exception: extract_exception_data(test),
+              ci_job_url: test.ci_job_url,
+              testcase: extract_test_id_or_name(test),
+              product_group: extract_product_group(test),
+              level: extract_level(test)
+            }
+          end
+        end
+
+        def extract_line_number(test)
+          test.respond_to?(:line_number) ? test.line_number : nil
+        end
+
+        def extract_exception_data(test)
+          {
+            'message' => test.failures.first&.dig('message') || test.full_stacktrace || 'Unknown error'
+          }
+        end
+
+        def extract_product_group(test)
+          test.respond_to?(:product_group) ? test.product_group : nil
+        end
+
+        def extract_level(test)
+          test.respond_to?(:level) ? test.level : nil
+        end
+
+        def grouper_config
+          {
+            thresholds: {
+              min_failures_to_group: 2
+            }
+          }
+        end
+
+        def extract_test_id_or_name(test)
+          return test.example_id if test.respond_to?(:example_id)
+          return test.id if test.respond_to?(:id)
+          return test.name if test.respond_to?(:name)
+
+          "#{test.relative_file}:#{test.respond_to?(:line_number) ? test.line_number : 'unknown'}"
         end
 
         def new_issue_labels(test)
@@ -179,7 +279,9 @@ module GitlabQuality
           end
 
           created_issue = super
-          test.failure_issue ||= created_issue.web_url
+
+          # On a dry run, created_issue may not be populated
+          test.failure_issue ||= created_issue&.web_url
 
           created_issue
         end
@@ -300,7 +402,7 @@ module GitlabQuality
         end
 
         def new_issue_description(test)
-          super + [
+          FAILURE_ISSUE_HANDBOOK_GUIDE + super + [
             "\n#{commit_diff_section}",
             "### Stack trace",
             "```\n#{test.full_stacktrace}\n```",
@@ -318,10 +420,10 @@ module GitlabQuality
           initialize_gitlab_ops_client
 
           if Runtime::Env.ci_pipeline_url.include?('ops.gitlab.net')
-            pipeline = ops_gitlab_client.find_pipeline(project, Runtime::Env.ci_pipeline_id.to_i)
+            pipeline = ops_gitlab_client.find_pipeline(Runtime::Env.ci_project_path, Runtime::Env.ci_pipeline_id.to_i)
             generate_ops_gitlab_diff(pipeline)
           else
-            pipeline = gitlab.find_pipeline(project, Runtime::Env.ci_pipeline_id.to_i)
+            pipeline = gitlab.find_pipeline(Runtime::Env.ci_project_path, Runtime::Env.ci_pipeline_id.to_i)
             generate_gitlab_diff(pipeline)
           end
         end
@@ -342,6 +444,8 @@ module GitlabQuality
         end
 
         def fetch_deployment_info(pipeline)
+          return 'No pipeline name set.' unless Runtime::Env.ci_pipeline_name
+
           pipeline_deploy_version = Runtime::Env.ci_pipeline_name.match(/(\d+\.\d+\.\d+)(?:-|$)/)&.captures&.first
           deployments = fetch_deployments(ops_gitlab_client, pipeline)
           found_deployment = find_matching_deployment(pipeline_deploy_version, deployments)
@@ -427,7 +531,8 @@ module GitlabQuality
 
         def generate_gitlab_diff(pipeline)
           pipeline_sha = pipeline.sha
-          parent_sha = gitlab.find_commit_parent(project, pipeline_sha)
+          commit_project = COMMIT_PROJECT_MAPPINGS.fetch(project, project)
+          parent_sha = gitlab.find_commit_parent(commit_project, pipeline_sha)
           diff_project = if DIFF_PROJECT_MAPPINGS.key?(project)
                            DIFF_PROJECT_MAPPINGS[project]
                          else
@@ -457,10 +562,40 @@ module GitlabQuality
         end
 
         def new_issue_assignee_id(test)
-          return unless test.product_group?
+          assignee_id = try_feature_category_assignment(test)
+          return assignee_id if assignee_id
+
+          try_product_group_assignment(test)
+        end
+
+        def try_feature_category_assignment(test)
+          unless test.respond_to?(:feature_category) && test.feature_category?
+            Runtime::Logger.info("No feature_category found for DRI assignment")
+            return
+          end
+
+          labels_inference = GitlabQuality::TestTooling::LabelsInference.new
+          product_group = labels_inference.product_group_from_feature_category(test.feature_category)
+
+          unless product_group
+            Runtime::Logger.warn("Could not map feature_category '#{test.feature_category}' to product_group")
+            return
+          end
+
+          dri = test_dri(product_group, test.stage, test.section)
+          Runtime::Logger.info("Assigning #{dri} as DRI for the issue (via feature_category).")
+
+          gitlab.find_user_id(username: dri)
+        end
+
+        def try_product_group_assignment(test)
+          unless test.respond_to?(:product_group) && test.product_group?
+            Runtime::Logger.info("No product_group found for DRI assignment")
+            return
+          end
 
           dri = test_dri(test.product_group, test.stage, test.section)
-          puts "  => Assigning #{dri} as DRI for the issue."
+          Runtime::Logger.info("Assigning #{dri} as DRI for the issue (via product_group).")
 
           gitlab.find_user_id(username: dri)
         end
@@ -542,6 +677,8 @@ module GitlabQuality
         end
 
         def screenshot_artifact_url(test)
+          return "" unless test.screenshot_image
+
           ci_job_url = test.ci_job_url
           screenshot_path = test.screenshot_image[%r{qa/.*$}]
 

data/lib/gitlab_quality/test_tooling/runtime/env.rb

@@ -46,7 +46,7 @@ module GitlabQuality
         end
 
         def default_branch
-          env_var_value_if_defined('QA_DEFAULT_BRANCH') || 'main'
+          'master'
         end
 
         def ci_api_v4_url

data/lib/gitlab_quality/test_tooling/test_meta/processor/add_to_blocking_processor.rb

@@ -97,7 +97,7 @@ module GitlabQuality
 
                   This MR was created based on data from reliable e2e test report: #{context.report_issue}
 
-                  /label ~"Quality" ~"QA" ~"type::maintenance"
+                  /label ~"E2E" ~"type::maintenance"
                   /label ~"devops::#{devops_stage}"
                   #{context.label_from_product_group(product_group)}
 

data/lib/gitlab_quality/test_tooling/test_meta/processor/add_to_quarantine_processor.rb

@@ -137,7 +137,7 @@ module GitlabQuality
                 - [ ] To ensure a faster turnaround, ask in the `#quality_maintainers` Slack channel for someone to review and merge the merge request, rather than assigning it directly.
 
                 <!-- Base labels. -->
-                /label ~"Quality" ~"QA" ~"type::maintenance" ~"maintenance::pipelines"
+                /label ~"E2E" ~"type::maintenance" ~"maintenance::pipelines"
 
                 <!--
                 Choose the stage that appears in the test path, e.g. ~"devops::create" for

data/lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb

@@ -8,7 +8,7 @@ module GitlabQuality
       class TestMetaUpdater
         include TestTooling::Concerns::FindSetDri
 
-        attr_reader :project, :ref, :report_issue, :processed_commits
+        attr_reader :project, :ref, :report_issue, :processed_commits, :token, :specs_file, :dry_run, :processor
 
         TEST_PLATFORM_MAINTAINERS_SLACK_CHANNEL_ID = 'C0437FV9KBN' # test-platform-maintainers
 
@@ -269,12 +269,15 @@ module GitlabQuality
         # Fetch the id for the dri of the product group and stage
         # The first item returned is the id of the assignee and the second item is the handle
         #
-        # @param [String] product_group
+        # @param [Hash] test object
         # @param [String] devops_stage
+        # @param [String] section
         # @return [Array<Integer, String>]
-        def fetch_dri_id(product_group, devops_stage, section)
-          assignee_handle = ENV.fetch('QA_TEST_DRI_HANDLE', nil) || test_dri(product_group, devops_stage, section)
+        def fetch_dri_id(test, devops_stage, section)
+          product_group = determine_product_group(test)
+          return unless product_group
 
+          assignee_handle = ENV.fetch('QA_TEST_DRI_HANDLE', nil) || test_dri(product_group, devops_stage, section)
           [user_id_for_username(assignee_handle), assignee_handle]
         end
 
@@ -335,6 +338,16 @@ module GitlabQuality
           label ? %(/label ~"#{label}") : ''
         end
 
+        # Infers the group label from the provided feature category
+        #
+        # @param [String] feature_category feature category
+        # @return [String]
+        def label_from_feature_category(feature_category)
+          labels = labels_inference.infer_labels_from_feature_category(feature_category)
+          group_label = labels.find { |label| label.start_with?('group::') }
+          group_label ? %(/label ~"#{group_label}") : ''
+        end
+
         # Returns the link to the Grafana dashboard for single spec metrics
         #
         # @param [String] example_name the full example name
@@ -344,10 +357,6 @@ module GitlabQuality
           base_url + CGI.escape(example_name)
         end
 
-        private
-
-        attr_reader :token, :specs_file, :dry_run, :processor
-
         # Returns any test description string within single or double quotes
         #
         # @param [String] line the line to check for any quoted string
@@ -380,6 +389,27 @@ module GitlabQuality
         def labels_inference
           @labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
         end
+
+        private
+
+        def determine_product_group(test)
+          return map_feature_category_to_product_group(test) if has_feature_category?(test)
+          return test.product_group if has_product_group?(test)
+
+          nil
+        end
+
+        def has_feature_category?(test)
+          test.respond_to?(:feature_category) && test.feature_category?
+        end
+
+        def has_product_group?(test)
+          test.respond_to?(:product_group) && test.product_group?
+        end
+
+        def map_feature_category_to_product_group(test)
+          labels_inference.product_group_from_feature_category(test.feature_category)
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

@@ -14,7 +14,8 @@ module GitlabQuality
           "Error reference number: 502",
           "(502): `GitLab is not responding`",
           "<head><title>502 Bad Gateway</title></head>",
-          "14:connections to all backends failing"
+          "14:connections to all backends failing",
+          "gitlab_canary=true cookie was set in browser but 'Next' badge was not shown on UI"
         ].freeze
 
         SHARED_EXAMPLES_CALLERS = %w[include_examples it_behaves_like].freeze
@@ -72,6 +73,10 @@ module GitlabQuality
           product_group != ''
         end
 
+        def feature_category?
+          feature_category && !feature_category.empty?
+        end
+
         def failure_issue
           report['failure_issue']
         end
@@ -114,7 +119,7 @@ module GitlabQuality
         end
 
         def file_base_url
-          @file_base_url ||= "https://gitlab.com/#{project}/-/blob/#{ref}/"
+          @file_base_url ||= "https://gitlab.com/#{project == 'gitlab-org/quality/e2e-test-issues' ? 'gitlab-org/gitlab' : project}/-/blob/#{ref}/"
         end
 
         def test_file_link
@@ -143,15 +148,23 @@ module GitlabQuality
         def files_client
           @files_client ||= GitlabClient::RepositoryFilesClient.new(
             token: token,
-            project: project,
+            project: mapped_project,
             file_path: file,
-            ref: ref)
+            ref: ref.nil? || ref.empty? ? 'master' : ref)
         end
 
         private
 
         attr_reader :token, :project, :ref
 
+        def mapped_project
+          if project == 'gitlab-org/quality/e2e-test-issues'
+            'gitlab-org/gitlab'
+          else
+            project
+          end
+        end
+
         def screenshot
           report.fetch('screenshot', nil)
         end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "2.10.0"
+    VERSION = "2.18.0"
   end
 end

data/lib/gitlab_quality/test_tooling.rb

@@ -7,6 +7,8 @@ module GitlabQuality
   module TestTooling
     Error = Class.new(StandardError)
     loader = Zeitwerk::Loader.new
+    loader.push_dir(__dir__.to_s, namespace: GitlabQuality)
+    loader.ignore("#{__dir__}/test_tooling.rb")
     loader.push_dir("#{__dir__}/test_tooling", namespace: GitlabQuality::TestTooling)
     loader.ignore("#{__dir__}/test_tooling/version.rb")
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 2.10.0
+  version: 2.18.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-04-23 00:00:00.000000000 Z
+date: 2025-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -263,7 +263,7 @@ dependencies:
         version: '4.19'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -273,7 +273,7 @@ dependencies:
         version: '4.19'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement
@@ -380,16 +380,22 @@ dependencies:
   name: rspec-parameterized
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: table_print
   requirement: !ruby/object:Gem::Requirement
@@ -431,7 +437,8 @@ executables:
 - detect-infrastructure-failures
 - existing-test-health-issue
 - failed-test-issues
-- feature-readiness-check
+- feature-readiness-checklist
+- feature-readiness-evaluation
 - flaky-test-issues
 - generate-test-session
 - knapsack-report-issues
@@ -463,7 +470,8 @@ files:
 - exe/detect-infrastructure-failures
 - exe/existing-test-health-issue
 - exe/failed-test-issues
-- exe/feature-readiness-check
+- exe/feature-readiness-checklist
+- exe/feature-readiness-evaluation
 - exe/flaky-test-issues
 - exe/generate-test-session
 - exe/knapsack-report-issues
@@ -479,8 +487,12 @@ files:
 - lib/gitlab_quality/test_tooling.rb
 - lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb
 - lib/gitlab_quality/test_tooling/failed_jobs_table.rb
+- lib/gitlab_quality/test_tooling/feature_readiness/analyzed_items/analyzed_epic.rb
+- lib/gitlab_quality/test_tooling/feature_readiness/analyzed_items/analyzed_issue.rb
+- lib/gitlab_quality/test_tooling/feature_readiness/analyzed_items/analyzed_merge_request.rb
 - lib/gitlab_quality/test_tooling/feature_readiness/concerns/issue_concern.rb
 - lib/gitlab_quality/test_tooling/feature_readiness/concerns/work_item_concern.rb
+- lib/gitlab_quality/test_tooling/feature_readiness/evaluation.rb
 - lib/gitlab_quality/test_tooling/feature_readiness/operational_readiness_check.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/branches_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/branches_dry_client.rb
@@ -507,8 +519,20 @@ files:
 - lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
 - lib/gitlab_quality/test_tooling/report/concerns/utils.rb
 - lib/gitlab_quality/test_tooling/report/failed_test_issue.rb
+- lib/gitlab_quality/test_tooling/report/feature_readiness/report_on_epic.rb
 - lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb
 - lib/gitlab_quality/test_tooling/report/generate_test_session.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/error_message_normalizer.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/error_pattern_matcher.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/failure_processor.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/group_results_in_issues.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/incident_checker.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/issue_base.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/issue_creator.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/issue_finder.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/issue_formatter.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/issue_manager.rb
+- lib/gitlab_quality/test_tooling/report/group_issues/issue_updater.rb
 - lib/gitlab_quality/test_tooling/report/health_problem_reporter.rb
 - lib/gitlab_quality/test_tooling/report/issue_logger.rb
 - lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb