gitlab_quality-test_tooling 2.0.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a5b20eb9951ec812418e3b1f5774674419d29961fabf4710f32e71d7a7f6c73
-  data.tar.gz: 7ec895c4672772ef7801cf1fa864c87d1f76b59f2834434658f13e7c64cc0615
+  metadata.gz: 7437a045ff423ca3c6585be2aa13b11d97fd94abb5d8ffd95a14d11f48970b68
+  data.tar.gz: fd91b826d48c661a7b05f196857ec95c7fc59d478e8718ffa9ca5d534f38ffe5
 SHA512:
-  metadata.gz: f013d08976015caf80341b6aa86ecbb1b7198adeac56911dc89a9e3958a78a8001c065cb7a777ed8248109fd72d5b2109a577692c494ad785a3f55e0296c2be3
-  data.tar.gz: 66be21506f82e4cbf566216f546308df9e1607aaa709792f35c9053c117da2a2845abb376fe88ff743790a0d5006a6d71e2ee2b9365f109b50772367a6c2bf8a
+  metadata.gz: 4de31f3a187af7c92df5b015c631e5fa1278fa6a3d62f8111294ce07c245236b9e30e78a6760859eee4d024ee0de62dd544201e2d3e4beec73ef7be00f443def
+  data.tar.gz: 1d44821f07f2d61ac2d1b31b0837149bcf015070ffa1c4e6791d4685c3be322ab0991a28bf1551130eaf274045d58855a405b680948673f78aa208c9a34cab77

data/.tool-versions

@@ -1 +1,2 @@
 ruby 3.2.5
+lefthook 1.7.14

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (2.0.0)
+    gitlab_quality-test_tooling (2.2.0)
       activesupport (>= 7.0, < 7.2)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)
@@ -302,6 +302,8 @@ GEM
       binding_of_caller
       rspec-parameterized-core (< 2)
     rspec-support (3.13.1)
+    rspec_junit_formatter (0.6.0)
+      rspec-core (>= 2, < 4, != 2.12.0)
     rubocop (1.62.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
@@ -407,6 +409,7 @@ DEPENDENCIES
   pry-byebug (= 3.10.1)
   rake (~> 13.0)
   rspec (~> 3.12)
+  rspec_junit_formatter (~> 0.6.0)
   simplecov (~> 0.22)
   simplecov-cobertura (~> 2.1)
   solargraph (~> 0.41)

data/README.md

@@ -177,6 +177,20 @@ Usage: exe/failed-test-issues [options]
     -h, --help                       Show the usage
 ```
 
+### `exe/existing-test-health-issue`
+
+```shell
+Purpose: Checks whether tests coming from the rspec JSON report files has an existing test health issue opened.
+Usage: exe/existing-test-health-issue [options]
+    -i, --input-files INPUT_FILES    JSON rspec-retry report files
+    -p, --project PROJECT            Can be an integer or a group/project string
+    -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+        --health-problem-type PROBLEM_TYPE
+                                     Look for the given health problem type (failures, pass-after-retry, slow)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
 ### `exe/flaky-test-issues`
 
 ```shell

data/exe/existing-test-health-issue

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+HEALTH_PROBLEM_TYPES = GitlabQuality::TestTooling::Report::TestHealthIssueFinder::HEALTH_PROBLEM_TYPE_TO_LABEL.keys
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files INPUT_FILES', String, 'JSON rspec-retry report files') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
+    params[:token] = token
+  end
+
+  opts.on("--health-problem-type PROBLEM_TYPE", String, "Look for the given health problem type (#{HEALTH_PROBLEM_TYPES.join(', ')})") do |value|
+    raise ArgumentError, "Invalid health problem type: #{value}. Valid options are: #{HEALTH_PROBLEM_TYPES.join(', ')}" unless HEALTH_PROBLEM_TYPES.include?(value)
+
+    params[:health_problem_type] = value
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Checks whether tests coming from the rspec JSON report files has an existing test health issue opened."
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  raise ArgumentError, "No health problem type given. Valid options are: #{HEALTH_PROBLEM_TYPES.join(', ')}" unless params.key?(:health_problem_type)
+
+  if GitlabQuality::TestTooling::Report::TestHealthIssueFinder.new(**params).found_existing_unhealthy_test_issue?
+    exit 0
+  else
+    exit 1
+  end
+else
+  puts options
+  exit 1
+end

data/lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb

@@ -11,13 +11,15 @@ module GitlabQuality
           FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
           REPORT_ITEM_REGEX = /^1\. (?<report_date>\d{4}-\d{2}-\d{2}): #{JOB_URL_REGEX} \((?<pipeline_url>\S+)\) ?(?<extra_content>.*)$/
           LATEST_REPORTS_TO_SHOW = 10
+          DISPLAYED_HISTORY_REPORTS_THRESHOLD = 510 # https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/587
           DAILY_REPORTS_THRESHOLDS = 10
 
           class ReportsList
-            def initialize(preserved_content:, section_header:, reports:, extra_content:)
+            def initialize(preserved_content:, section_header:, reports:, total_reports_count:, extra_content:)
               @preserved_content = preserved_content
               @section_header = section_header
               @reports = reports
+              @total_reports_count = total_reports_count
               @extra_content = extra_content
             end
 
@@ -31,14 +33,14 @@ module GitlabQuality
             end
 
             def reports_count
-              reports.size
+              total_reports_count
             end
 
             def to_s
               [
                 preserved_content,
                 "#{section_header} (#{reports_count})",
-                reports_list,
+                reports_list(total_reports_count),
                 extra_content
               ].reject(&:blank?).compact.join("\n\n")
             end
@@ -53,15 +55,16 @@ module GitlabQuality
 
             private
 
-            attr_reader :preserved_content, :section_header, :reports, :extra_content
+            attr_reader :preserved_content, :section_header, :reports, :total_reports_count, :extra_content
 
-            def reports_list
+            def reports_list(total_reports_count)
               if sorted_reports.size > LATEST_REPORTS_TO_SHOW
+                max_displayed_reports = DISPLAYED_HISTORY_REPORTS_THRESHOLD - LATEST_REPORTS_TO_SHOW
                 [
                   "Last #{LATEST_REPORTS_TO_SHOW} reports:",
-                  sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
-                  "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
-                  sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
+                  displayed_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
+                  "<details><summary>With #{total_reports_count - LATEST_REPORTS_TO_SHOW} more reports (displaying up to #{max_displayed_reports} reports) </summary>",
+                  displayed_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
                   "</details>"
                 ].join("\n\n")
               else
@@ -70,7 +73,12 @@ module GitlabQuality
             end
 
             def sorted_reports
-              @sorted_reports ||= reports.sort.reverse
+              @sorted_reports ||=
+                reports.sort_by { |report| [report.report_date, report.job_id, report.to_s] }.reverse
+            end
+
+            def displayed_reports
+              sorted_reports[0...DISPLAYED_HISTORY_REPORTS_THRESHOLD]
             end
           end
 
@@ -88,8 +96,8 @@ module GitlabQuality
               "1. #{report_date}: #{job_url} (#{pipeline_url}) #{extra_content}".strip
             end
 
-            def <=>(other)
-              to_s <=> other.to_s
+            def job_id
+              job_url.split('/').last
             end
 
             private
@@ -122,14 +130,33 @@ module GitlabQuality
             preserved_content = current_reports_content.split(reports_section_header).first&.strip
             reports = report_lines(current_reports_content) + [ReportsList.report_list_item(test, item_extra_content: item_extra_content)]
 
+            total_reports_count = increment_total_reports_count(
+              reports_section_header: reports_section_header,
+              content: current_reports_content,
+              reports: reports
+            )
+
             ReportsList.new(
               preserved_content: preserved_content,
               section_header: reports_section_header,
               reports: reports,
+              total_reports_count: total_reports_count,
               extra_content: reports_extra_content
             )
           end
 
+          def increment_total_reports_count(reports_section_header:, content:, reports:)
+            reports_count = reports.count
+
+            return reports_count if reports_count < DISPLAYED_HISTORY_REPORTS_THRESHOLD
+
+            count_match = content.match(/#{Regexp.escape(reports_section_header)} \((\d+)\)/)
+
+            return reports_count unless count_match
+
+            count_match[1].to_i + 1
+          end
+
           def failed_issue_job_url(issue)
             job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
               # Legacy format

data/lib/gitlab_quality/test_tooling/report/test_health_issue_finder.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'http'
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      class TestHealthIssueFinder < ReportAsIssue
+        HEALTH_PROBLEM_TYPE_TO_LABEL = {
+          'pass-after-retry' => 'test-health:pass-after-retry',
+          'slow' => 'test-health:slow',
+          'failures' => 'test-health:failures'
+        }.freeze
+
+        def initialize(health_problem_type: [], **kwargs)
+          super(**kwargs)
+
+          @health_problem_type = health_problem_type
+        end
+
+        def found_existing_unhealthy_test_issue?
+          issue_url = invoke!
+
+          !issue_url.nil? && !issue_url.empty?
+        end
+
+        def run!
+          existing_issue_found = nil
+
+          applicable_tests.each do |test|
+            issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: search_labels)
+            next if issues.empty?
+
+            existing_issue_found = issues.first.web_url
+            puts "Found an existing test health issue of type #{health_problem_type} for test #{test.file}:#{test.line_number}: #{existing_issue_found}."
+            break
+          end
+
+          puts "Did not find an existing test health issue of type #{health_problem_type}." unless existing_issue_found
+
+          existing_issue_found
+        end
+
+        def applicable_tests
+          applicable_tests = []
+
+          TestResults::Builder.new(file_glob: files, token: token, project: project).test_results_per_file do |test_results|
+            applicable_tests = test_results.select { |test| test_is_applicable?(test) }
+          end
+
+          applicable_tests
+        end
+
+        private
+
+        attr_reader :health_problem_type
+
+        # Be mindful about the number of tests this method would return,
+        # as we will make at least one API request per test.
+        def test_is_applicable?(test)
+          expected_test_status =
+            case health_problem_type
+            when 'failures'
+              'failed'
+            else
+              'passed'
+            end
+
+          test.status == expected_test_status
+        end
+
+        def search_labels
+          ['test', HEALTH_PROBLEM_TYPE_TO_LABEL[health_problem_type]]
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_results/json_test_results.rb

@@ -16,9 +16,14 @@ module GitlabQuality
 
         def parse
           JSON.parse(File.read(path))
+        rescue JSON::ParserError
+          Runtime::Logger.debug("#{self.class.name}##{__method__} attempted to parse invalid JSON at path: #{path}")
+          {}
         end
 
         def process
+          return [] if results.empty?
+
           results['examples'].map do |test|
             GitlabQuality::TestTooling::TestResult::JsonTestResult.new(report: test, project: project, token: token)
           end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "2.0.0"
+    VERSION = "2.2.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.2.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-14 00:00:00.000000000 Z
+date: 2024-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -192,6 +192,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.7.0
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -400,6 +414,7 @@ description: A collection of test-related tools.
 email:
 - quality@gitlab.com
 executables:
+- existing-test-health-issue
 - failed-test-issues
 - flaky-test-issues
 - generate-test-session
@@ -429,6 +444,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/existing-test-health-issue
 - exe/failed-test-issues
 - exe/flaky-test-issues
 - exe/generate-test-session
@@ -477,6 +493,7 @@ files:
 - lib/gitlab_quality/test_tooling/report/results_in_issues.rb
 - lib/gitlab_quality/test_tooling/report/results_in_test_cases.rb
 - lib/gitlab_quality/test_tooling/report/slow_test_issue.rb
+- lib/gitlab_quality/test_tooling/report/test_health_issue_finder.rb
 - lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb
 - lib/gitlab_quality/test_tooling/runtime/env.rb
 - lib/gitlab_quality/test_tooling/runtime/logger.rb