gitlab_quality-test_tooling 1.3.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8faeb526c0c3996945e423ca2c2d944f57bda93b1941beac376e9d4478bf8230
-  data.tar.gz: '09a401ff0c2c60effc2a9c55466ed266dceb7eed8b3c83aa1970c31837ec36dd'
+  metadata.gz: 7ac9e211aa370a7163c4173c7a1dec921cbf237fc406d28e1c3c5cfe296e9750
+  data.tar.gz: 99a599cdb8f65c74a489210004f9b34918f03c0d3eb54cd47e3e2dc5a1ee01a1
 SHA512:
-  metadata.gz: e7ba58f29fb1d459a82e5bc078791ecd08b5e06737530b09f52facd3813e1f7de699b7227748e58a788ef16099c3a9e6aef8a97607a5b0ccb0adc7bc96a59ab0
-  data.tar.gz: 88f89b94c695450d2f0716e4927e11865dd5e9c3d356f1f6c106ece8ff86ab5797db88459dd7285acf8ee0b7b37952c829bc681ee1b5678f42f35643cc51aeb9
+  metadata.gz: '009549cacd5c20ec47ecc157801f272e00a4681decfa5fbf5ae62e252127f8482313857ad328cda1a68327ae7047c771ec7f6d39cfb0fa835834f9b1a53b519a'
+  data.tar.gz: 4bc8d5f83edf2c193ff2fadbc4ae525210a9d9c817dfc48ce35bfa9efcabdecd7801e59165e761a10c2a46d382bc276e1567d45e3ba64905649ce8abab530703

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (1.3.0)
+    gitlab_quality-test_tooling (1.5.0)
       activesupport (>= 6.1, < 7.2)
+      amatch (~> 0.4.1)
       gitlab (~> 4.19)
       http (~> 5.0)
       nokogiri (~> 1.10)
@@ -22,6 +23,9 @@ GEM
       zeitwerk (~> 2.3)
     addressable (2.8.4)
       public_suffix (>= 2.0.2, < 6.0)
+    amatch (0.4.1)
+      mize
+      tins (~> 1.0)
     ast (2.4.2)
     backport (1.2.0)
     benchmark (0.2.1)
@@ -132,6 +136,8 @@ GEM
     mini_mime (1.1.2)
     mini_portile2 (2.8.1)
     minitest (5.18.0)
+    mize (0.4.1)
+      protocol (~> 2.0)
     multi_xml (0.6.0)
     nap (1.1.0)
     nenv (0.3.0)
@@ -149,6 +155,8 @@ GEM
     parallel (1.23.0)
     parser (3.2.2.1)
       ast (~> 2.4.1)
+    protocol (2.0.0)
+      ruby_parser (~> 3.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -207,9 +215,12 @@ GEM
       rubocop-capybara (~> 2.17)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
+    ruby_parser (3.20.3)
+      sexp_processor (~> 4.16)
     sawyer (0.9.2)
       addressable (>= 2.3.5)
       faraday (>= 0.17.3, < 3)
+    sexp_processor (4.17.0)
     shellany (0.0.1)
     simplecov (0.22.0)
       docile (~> 1.1)
@@ -236,12 +247,15 @@ GEM
       thor (~> 1.0)
       tilt (~> 2.0)
       yard (~> 0.9, >= 0.9.24)
+    sync (0.5.0)
     table_print (1.5.7)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     thor (1.2.1)
     tilt (2.1.0)
     timecop (0.9.6)
+    tins (1.32.1)
+      sync
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unf (0.1.4)

data/README.md

@@ -26,7 +26,7 @@ $ gem install gitlab_quality-test_tooling
 
 The gem provides the following executables.
 
-### `generate-test-session`
+### `exe/generate-test-session`
 
 ```shell
 Purpose: Generate test session report based on RSpec report files (JSON or JUnit XML)
@@ -38,15 +38,15 @@ Usage: exe/generate-test-session [options]
         --ci-project-token
     -f ISSUE_URL_FILE,               Output the created test session issue URL
         --issue-url-file
+        --confidential               Makes test session issue confidential
         --dry-run                    Perform a dry-run (don't create or update issues or test cases)
     -v, --version                    Show the version
     -h, --help                       Show the usage
 ```
 
-### `post-to-slack`
+### `exe/post-to-slack`
 
 ```shell
-$ exe/post-to-slack -h
 Purpose: Post a message to Slack, and optionally add a test summary table based on RSpec report files (JUnit XML)
 Usage: exe/post-to-slack [options]
     -w SLACK_WEBHOOK_URL,            Slack webhook URL
@@ -61,10 +61,9 @@ Usage: exe/post-to-slack [options]
     -h, --help                       Show the usage
 ```
 
-### `prepare-stage-reports`
+### `exe/prepare-stage-reports`
 
 ```shell
-$ exe/prepare-stage-reports -h
 Purpose: Prepare separate reports for each DevOps stage from the provided RSpec report files (JUnit XML)
 Usage: exe/prepare-stage-reports [options]
     -i, --input-files INPUT_FILES    RSpec report files (JUnit XML)
@@ -72,10 +71,9 @@ Usage: exe/prepare-stage-reports [options]
     -h, --help                       Show the usage
 ```
 
-### `relate-failure-issue`
+### `exe/relate-failure-issue`
 
 ```shell
-$ exe/relate-failure-issue -h
 Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)
 Usage: exe/relate-failure-issue [options]
     -i, --input-files INPUT_FILES    RSpec report files (JSON or JUnit XML)
@@ -93,10 +91,9 @@ Usage: exe/relate-failure-issue [options]
     -h, --help                       Show the usage
 ```
 
-### `report-results`
+### `exe/report-results`
 
 ```shell
-$ exe/report-results -h
 Purpose: Report test results from RSpec report files (JSON or JUnit XML) in GitLab test cases and result issues
 Usage: exe/report-results [options]
     -i, --input-files INPUT_FILES    RSpec report files (JSON or JUnit XML)
@@ -113,10 +110,9 @@ Usage: exe/report-results [options]
     -h, --help                       Show the usage
 ```
 
-### `update-screenshot-paths`
+### `exe/update-screenshot-paths`
 
 ```shell
-$ exe/update-screenshot-paths -h
 Purpose: Update the path to screenshots to container's host from RSpec report files (JSON or JUnit XML)
 Usage: exe/update-screenshot-paths [options]
     -i, --input-files INPUT_FILES    RSpec report files (JSON or JUnit XML)
@@ -124,7 +120,7 @@ Usage: exe/update-screenshot-paths [options]
     -h, --help                       Show the usage
 ```
 
-### `slow-test-issues`
+### `exe/slow-test-issues`
 
 ```shell
 Purpose: Create slow test issues from JSON RSpec report files
@@ -137,10 +133,9 @@ Usage: exe/slow-test-issue [options]
     -h, --help                       Show the usage
 ```
 
-### `slow-test-merge-request-report-note`
+### `exe/slow-test-merge-request-report-note`
 
 ```shell
-$ exe/slow-test-merge-request-report-note -h
 Purpose: Create slow test note on merge requests from JSON RSpec report files
 Usage: exe/slow-test-merge-request-report-note [options]
     -i, --input-files INPUT_FILES    JSON RSpec report files JSON

data/exe/generate-test-session

@@ -31,6 +31,10 @@ options = OptionParser.new do |opts|
     params[:issue_url_file] = issue_url_file
   end
 
+  opts.on('--confidential', "Makes test session issue confidential") do
+    params[:confidential] = true
+  end
+
   opts.on('--dry-run', "Perform a dry-run (don't create or update issues or test cases)") do
     params[:dry_run] = true
   end

data/exe/relate-failure-issue

@@ -27,6 +27,10 @@ options = OptionParser.new do |opts|
     params[:token] = token
   end
 
+  opts.on('-r', '--related-issues-file RELATED_ISSUES_FILE', String, 'The file path for the related issues') do |related_issues_file|
+    params[:related_issues_file] = related_issues_file
+  end
+
   opts.on('--system-log-files SYSTEM_LOG_FILES', String,
     'Include errors from system logs in failure issues') do |system_log_files|
     params[:system_logs] = system_log_files

data/exe/slow-test-issues

@@ -23,6 +23,10 @@ options = OptionParser.new do |opts|
     params[:token] = token
   end
 
+  opts.on('-r', '--related-issues-file RELATED_ISSUES_FILE', String, 'The file path for the related issues') do |related_issues_file|
+    params[:related_issues_file] = related_issues_file
+  end
+
   opts.on('--dry-run', "Perform a dry-run (don't create issues)") do
     params[:dry_run] = true
   end

data/lib/gitlab_quality/test_tooling/gitlab_issue_dry_client.rb

@@ -3,8 +3,8 @@
 module GitlabQuality
   module TestTooling
     class GitlabIssueDryClient < GitlabIssueClient
-      def create_issue(title:, description:, labels:, issue_type: 'issue')
-        attrs = { description: description, labels: labels }
+      def create_issue(title:, description:, labels:, issue_type: 'issue', confidential: false)
+        attrs = { description: description, labels: labels, confidential: confidential }
 
         puts "The following #{issue_type} would have been created:"
         puts "project: #{project}, title: #{title}, attrs: #{attrs}"

data/lib/gitlab_quality/test_tooling/report/concerns/utils.rb

@@ -20,7 +20,10 @@ module GitlabQuality
           end
 
           def partial_file_path(path)
-            path.match(%r{((?:spec|ee|api|browser_ui)/.*)}i)[1]
+            matched = path&.match(%r{(?<partial_path>(?:spec|ee|api|browser_ui)/.*)}i)
+            return matched[:partial_path] if matched
+
+            path
           end
 
           def search_safe(value)

data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb

@@ -30,7 +30,8 @@ module GitlabQuality
           issue = gitlab.create_issue(
             title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
             description: generate_description(tests),
-            labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
+            labels: ['Quality', 'QA', 'triage report', pipeline_name_label],
+            confidential: confidential
           )
 
           # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/295493
@@ -137,6 +138,8 @@ module GitlabQuality
             growth
             fulfillment
             enablement
+            self-managed
+            saas
           ]
 
           tests.sort_by do |test|

data/lib/gitlab_quality/test_tooling/report/issue_logger.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      class IssueLogger
+        def initialize(file_path:)
+          @file_path = file_path
+          @data = File.exist?(file_path) ? JSON.parse(File.read(file_path)) : Hash.new { |h, k| h[k] = [] }
+        end
+
+        def collect(test, issues)
+          data[test.ci_job_url] += Array(issues).map(&:web_url)
+          data[test.ci_job_url].uniq!
+        end
+
+        def write
+          dirname = File.dirname(file_path)
+
+          FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
+
+          File.write(file_path, JSON.pretty_generate(data))
+        end
+
+        private
+
+        attr_reader :file_path, :data
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -4,6 +4,7 @@ require 'nokogiri'
 require 'active_support/core_ext/enumerable'
 require 'rubygems/text'
 require 'active_support/core_ext/integer/time'
+require 'amatch'
 
 module GitlabQuality
   module TestTooling
@@ -17,6 +18,7 @@ module GitlabQuality
       class RelateFailureIssue < ReportAsIssue
         include Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
+        include Amatch
 
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
         SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
@@ -53,15 +55,20 @@ module GitlabQuality
 
           TestResults::Builder.new(files).test_results_per_file do |test_results|
             puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
+            process_test_results(test_results)
+          end
 
-            systemic_failures = systemic_failures_for_test_results(test_results)
+          write_issues_log_file
+        end
 
-            test_results.each do |test|
-              relate_failure_to_issue(test) if should_report?(test, systemic_failures)
-            end
+        def process_test_results(test_results)
+          systemic_failures = systemic_failures_for_test_results(test_results)
 
-            test_results.write
+          test_results.each do |test|
+            collect_issues(test, relate_failure_to_issue(test)) if should_report?(test, systemic_failures)
           end
+
+          test_results.write
         end
 
         def systemic_failures_for_test_results(test_results)
@@ -177,9 +184,12 @@ module GitlabQuality
 
         def full_stacktrace(test)
           test.failures.each do |failure|
-            next if IGNORED_FAILURES.any? { |e| failure['message'].include?(e) }
+            message = failure['message'] || ""
+            message_lines = failure['message_lines'] || []
+
+            next if IGNORED_FAILURES.any? { |e| message.include?(e) }
 
-            return failure['message_lines'].empty? ? failure['message'] : failure['message_lines'].join("\n")
+            return message_lines.empty? ? message : message_lines.join("\n")
           end
         end
 
@@ -201,8 +211,7 @@ module GitlabQuality
         end
 
         def calculate_diff_ratio(stack_trace_first, stack_trace_second)
-          ld = Class.new.extend(Gem::Text).method(:levenshtein_distance)
-          distance = ld.call(stack_trace_first, stack_trace_second)
+          distance = Levenshtein.new(stack_trace_first).match(stack_trace_second)
           distance.zero? ? 0.0 : (distance.to_f / stack_trace_first.size).round(3)
         end
 

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -8,11 +8,12 @@ module GitlabQuality
       class ReportAsIssue
         include Concerns::Utils
 
-        def initialize(token:, input_files:, project: nil, confidential: false, dry_run: false, **_kwargs)
+        def initialize(token:, input_files:, related_issues_file: nil, project: nil, confidential: false, dry_run: false, **_kwargs)
           @project = project
           @gitlab = (dry_run ? GitlabIssueDryClient : GitlabIssueClient).new(token: token, project: project)
           @files = Array(input_files)
           @confidential = confidential
+          @issue_logger = IssueLogger.new(file_path: related_issues_file) if related_issues_file.present?
         end
 
         def invoke!
@@ -23,14 +24,22 @@ module GitlabQuality
 
         private
 
-        attr_reader :gitlab, :files, :project, :issue_type, :confidential
+        attr_reader :gitlab, :files, :project, :issue_type, :confidential, :issue_logger
 
         def run!
           raise NotImplementedError
         end
 
+        def collect_issues(test, issues)
+          issue_logger.collect(test, issues) if issue_logger.present?
+        end
+
+        def write_issues_log_file
+          issue_logger.write if issue_logger.present?
+        end
+
         def test_hash(test)
-          OpenSSL::Digest::SHA256.hexdigest(test.file + test.name)
+          OpenSSL::Digest.hexdigest('SHA256', "#{test.file}#{test.name}")
         end
 
         def new_issue_description(test)
@@ -131,7 +140,8 @@ module GitlabQuality
 
           gitlab.find_issues(options: search_options).find_all do |issue|
             issue_title = issue.title.strip
-            issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
+            test_file_path_found = !test.file.to_s.empty? && issue_title.include?(partial_file_path(test.file))
+            issue_title.include?(test.name) || test_file_path_found
           end
         end
 

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -30,6 +30,8 @@ module GitlabQuality
               create_slow_issue(test) if test.slow_test?
             end
           end
+
+          write_issues_log_file
         end
 
         def new_issue_title(test)
@@ -51,11 +53,15 @@ module GitlabQuality
 
           issues = find_issues(test, SEARCH_LABELS, state: 'opened')
 
-          issues.each do |issue|
-            puts "   => Existing issue link #{issue['web_url']}"
+          if issues.blank?
+            issues << create_issue(test)
+          else
+            issues.each do |issue|
+              puts "   => Existing issue link #{issue['web_url']}"
+            end
           end
 
-          create_issue(test) unless issues.any?
+          collect_issues(test, issues)
         rescue MultipleIssuesFound => e
           warn(e.message)
         end

data/lib/gitlab_quality/test_tooling/runtime/env.rb

@@ -50,7 +50,7 @@ module GitlabQuality
         end
 
         def pipeline_from_project_name
-          ci_project_name.to_s.start_with?('gitlab') ? default_branch : ci_project_name
+          %w[gitlab gitaly].any? { |str| ci_project_name.to_s.start_with?(str) } ? default_branch : ci_project_name
         end
 
         def run_id
@@ -88,6 +88,10 @@ module GitlabQuality
           "#{ci_project_name}-#{test_subset}"
         end
 
+        def file_base_url
+          env_var_value_if_defined('FILE_BASE_URL') || "https://gitlab.com/gitlab-org/gitlab/-/blob/master/"
+        end
+
         private
 
         def enabled?(value, default: true)

data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

@@ -11,7 +11,7 @@ module GitlabQuality
         end
 
         def stage
-          @stage ||= file[%r{(?:api|browser_ui)/(?:(?:\d+_)?(\w+))}, 1]
+          @stage ||= file[%r{(?:api|browser_ui)/(?:(?:\d+_)?(\w+))}, 1] || category
         end
 
         def name
@@ -22,6 +22,10 @@ module GitlabQuality
           raise NotImplementedError
         end
 
+        def category
+          raise NotImplementedError
+        end
+
         def skipped?
           raise NotImplementedError
         end
@@ -29,6 +33,10 @@ module GitlabQuality
         def failures
           raise NotImplementedError
         end
+
+        def failures?
+          failures.any?
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/test_result/j_unit_test_result.rb

@@ -11,7 +11,7 @@ module GitlabQuality
         end
 
         def file
-          report['file'].delete_prefix('./')
+          report['file']&.delete_prefix('./')
         end
 
         def skipped?
@@ -25,17 +25,85 @@ module GitlabQuality
           failures.map do |exception|
             trace = exception.content.split("\n").map(&:strip)
             spec_file_first_index = trace.rindex do |line|
-              line.include?(File.basename(report['file']))
+              report['file'] && line.include?(File.basename(report['file']))
             end
 
             exception['message'].gsub!(/(private_token=)[\w-]+/, '********')
-
+            exception.content = exception.content.gsub(/(private_token=)[\w-]+/, '********')
             {
               'message' => "#{exception['type']}: #{exception['message']}",
-              'stacktrace' => trace.slice(0..spec_file_first_index).join("\n")
+              'stacktrace' => trace.slice(0..spec_file_first_index).join("\n"),
+              'message_lines' => trace.slice(0..spec_file_first_index)
             }
           end
         end
+
+        def quarantine
+          !report['quarantine'].nil?
+        end
+
+        def quarantine?
+          # The value for 'quarantine' could be nil, a hash, a string,
+          # or true (if the test just has the :quarantine tag)
+          # But any non-nil or false value should means the test is in quarantine
+          !!quarantine
+        end
+
+        def line_number
+          report['line_number']
+        end
+
+        def level
+          report['level']
+        end
+
+        def run_time
+          report['run_time'].to_f.round(2)
+        end
+
+        def screenshot
+          report['screenshot']
+        end
+
+        def screenshot?
+          !!screenshot
+        end
+
+        def max_duration_for_test
+          ""
+        end
+
+        def ci_job_url
+          ENV.fetch('CI_JOB_URL', '')
+        end
+
+        def product_group
+          report['product_group'].to_s
+        end
+
+        def product_group?
+          product_group != ''
+        end
+
+        def feature_category
+          report['feature_category']
+        end
+
+        def failure_issue
+          report['failure_issue']
+        end
+
+        def failure_issue=(new_failure_issue)
+          report['failure_issue'] = new_failure_issue
+        end
+
+        def test_file_link
+          return "" if file.nil?
+
+          path_prefix = file.start_with?('qa/') ? 'qa/' : ''
+
+          "[`#{path_prefix}#{file}#L#{line_number}`](#{Runtime::Env.file_base_url}#{path_prefix}#{file}#L#{line_number})"
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/test_result/json_test_result.rb

@@ -4,8 +4,6 @@ module GitlabQuality
   module TestTooling
     module TestResult
       class JsonTestResult < BaseTestResult
-        FILE_BASE_URL = "https://gitlab.com/gitlab-org/gitlab/-/blob/master/"
-
         PRIVATE_TOKEN_REGEX = /(private_token=)[\w-]+/
 
         OTHER_TESTS_MAX_DURATION = 45.40 # seconds
@@ -90,6 +88,10 @@ module GitlabQuality
           report['feature_category']
         end
 
+        def category
+          report['category']
+        end
+
         def run_time
           report['run_time'].to_f.round(2)
         end
@@ -132,10 +134,6 @@ module GitlabQuality
             end
         end
 
-        def failures?
-          failures.any?
-        end
-
         def allowed_to_be_slow?
           !!report['allowed_to_be_slow']
         end
@@ -156,7 +154,7 @@ module GitlabQuality
         def test_file_link
           path_prefix = file.start_with?('qa/') ? 'qa/' : ''
 
-          "[`#{path_prefix}#{file}#L#{line_number}`](#{FILE_BASE_URL}#{path_prefix}#{file}#L#{line_number})"
+          "[`#{path_prefix}#{file}#L#{line_number}`](#{Runtime::Env.file_base_url}#{path_prefix}#{file}#L#{line_number})"
         end
 
         private

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "1.3.0"
+    VERSION = "1.5.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.5.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-10-09 00:00:00.000000000 Z
+date: 2023-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -198,6 +198,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '7.2'
+- !ruby/object:Gem::Dependency
+  name: amatch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: gitlab
   requirement: !ruby/object:Gem::Requirement
@@ -363,6 +377,7 @@ files:
 - lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
 - lib/gitlab_quality/test_tooling/report/concerns/utils.rb
 - lib/gitlab_quality/test_tooling/report/generate_test_session.rb
+- lib/gitlab_quality/test_tooling/report/issue_logger.rb
 - lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb
 - lib/gitlab_quality/test_tooling/report/prepare_stage_reports.rb
 - lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb