gitlab_quality-test_tooling 1.5.3 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ecfb78e3fc7f1f23f118c98b16cdf1c104774734d08c4ff04dd6e4c7b25ae703
-  data.tar.gz: 4ef1ed6d2f559f427f0c26e92b90f631b5f407b7b604914c3b17786563675966
+  metadata.gz: 6924fc592c0dbb5e1ae0aed8710f7f7714c336bdd1d1d31b343e6edddd358ead
+  data.tar.gz: 0ebb5fbe0aeff0417193d4521cf52a78d849b83d116e08118b9cf87ee2655a71
 SHA512:
-  metadata.gz: 345e33ac54c7a917bb735a715e63bb8fcbe23464186aab1d02697ae1904f894a31ee4c04e2fb793262a04a4e768518e21eb5d0214616a337ca4f526a3a14af42
-  data.tar.gz: 5cf0dff6f8b7ff0f9526027fb3cc1efef03cf2ca1cf655d29054ff8075217e7221936b666e619e18cc09eaa003d3b70a65f1927124c6efb8c2f1516da9f00f27
+  metadata.gz: f9a64cbae7e3a34ae66450ae7fe20181560dc6860950a92e53c5aaf61ab10343372044968b5a726bef51b082a45dd9150d6f161d93a45a4ccf75d550cf981be1
+  data.tar.gz: a77ea69469a02b3cbd089cade6be20347abc03cada1cf93fec8380456f000d083a914296e0b35faff573e60d73e4759abd1b5d224cb04e64b38862fe720cf65c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (1.5.3)
+    gitlab_quality-test_tooling (1.7.0)
       activesupport (>= 6.1, < 7.2)
       amatch (~> 0.4.1)
       gitlab (~> 4.19)
@@ -29,6 +29,7 @@ GEM
     ast (2.4.2)
     backport (1.2.0)
     benchmark (0.2.1)
+    byebug (11.1.3)
     claide (1.1.0)
     claide-plugins (0.9.2)
       cork
@@ -160,6 +161,9 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
+    pry-byebug (3.10.1)
+      byebug (~> 11.0)
+      pry (>= 0.13, < 0.15)
     public_suffix (5.0.1)
     racc (1.6.2)
     rack (3.0.7)
@@ -279,6 +283,7 @@ DEPENDENCIES
   gitlab_quality-test_tooling!
   guard-rspec (~> 4.7)
   lefthook (~> 1.3)
+  pry-byebug (= 3.10.1)
   rake (~> 13.0)
   rspec (~> 3.12)
   simplecov (~> 0.22)

data/README.md

@@ -81,10 +81,14 @@ Usage: exe/relate-failure-issue [options]
                                      Max stacktrace diff ratio for failure issues detection
     -p, --project PROJECT            Can be an integer or a group/project string
     -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+    -r RELATED_ISSUES_FILE,          The file path for the related issues
+        --related-issues-file
         --system-log-files SYSTEM_LOG_FILES
                                      Include errors from system logs in failure issues
         --base-issue-labels BASE_ISSUE_LABELS
                                      Labels to add to new failure issues
+        --exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH
+                                     Labels to exclude when searching for existing issues
         --confidential               Makes created new issues confidential
         --dry-run                    Perform a dry-run (don't create or update issues)
     -v, --version                    Show the version
@@ -133,6 +137,21 @@ Usage: exe/slow-test-issue [options]
     -h, --help                       Show the usage
 ```
 
+### `exe/flaky-test-issues`
+
+```shell
+Purpose: Purpose: Create flaky test issues for any passed test coming from rspec-retry JSON report files
+Usage: exe/flaky-test-issue [options]
+    -i, --input-files INPUT_FILES    JSON rspec-retry report files
+    -p, --project PROJECT            Can be an integer or a group/project string
+    -m MERGE_REQUEST_IID,            An integer merge request IID
+        --merge_request_iid
+    -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+        --dry-run                    Perform a dry-run (don't create note)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
 ### `exe/slow-test-merge-request-report-note`
 
 ```shell

data/exe/flaky-test-issues

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files INPUT_FILES', String, 'JSON rspec-retry report files') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-m', '--merge_request_iid MERGE_REQUEST_IID', String, 'An integer merge request IID') do |merge_request_iid|
+    params[:merge_request_iid] = merge_request_iid
+  end
+
+  opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
+    params[:token] = token
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create issues)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Create flaky test issues for any passed test coming from rspec-retry JSON report files."
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::FlakyTestIssue.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/relate-failure-issue

@@ -41,6 +41,11 @@ options = OptionParser.new do |opts|
     params[:base_issue_labels] = base_issue_labels.split(',')
   end
 
+  opts.on('--exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH', String,
+    'Labels to exclude when searching for existing issues') do |exclude_labels_for_search|
+    params[:exclude_labels_for_search] = exclude_labels_for_search.split(',')
+  end
+
   opts.on('--confidential', "Makes created new issues confidential") do
     params[:confidential] = true
   end

data/lib/gitlab_quality/test_tooling/gitlab_issue_client.rb

@@ -55,6 +55,16 @@ module GitlabQuality
         end
       end
 
+      def find_issues_by_hash(test_hash, &select)
+        select ||= :itself
+
+        handle_gitlab_client_exceptions do
+          client.search_in_project(project, 'issues', test_hash)
+            .auto_paginate
+            .select(&select)
+        end
+      end
+
       def find_issue_discussions(iid:)
         handle_gitlab_client_exceptions do
           client.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate

data/lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module Concerns
+        module IssueReports
+          JOB_URL_REGEX                = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
+          FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
+          REPORT_ITEM_REGEX            = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
+
+          def initial_reports_section(test)
+            <<~REPORTS
+            ### Reports (1)
+
+            #{report_list_item(test)}
+            REPORTS
+          end
+
+          def add_report_to_issue_description(issue, test)
+            issue_description = issue.description
+
+            # We include the number of reports in the header, for visibility.
+            new_issue_description =
+              if issue_description.include?('### Reports')
+                # We count the number of existing reports.
+                reports_count = issue_description
+                  .scan(REPORT_ITEM_REGEX)
+                  .size.to_i + 1
+                issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
+              else # For issue with the legacy format, we add the Reports section
+                update_legacy_issue_description(issue_description)
+              end
+
+            [new_issue_description, report_list_item(test)].join("\n")
+          end
+
+          def failed_issue_job_url(issue)
+            job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
+              # Legacy format
+              job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX).last
+          end
+
+          def failed_issue_job_urls(issue)
+            job_urls_from_description(issue.description, REPORT_ITEM_REGEX) +
+              # Legacy format
+              job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX)
+          end
+
+          private
+
+          def report_list_item(test)
+            "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
+          end
+
+          def job_urls_from_description(issue_description, regex)
+            issue_description.lines.filter_map do |line|
+              match = line.match(regex)
+              match[:job_url] if match
+            end
+          end
+
+          def update_legacy_issue_description(issue_description)
+            test_captures = issue_description.scan(JOB_URL_REGEX)
+            reports_count = test_captures.size.to_i + 1
+
+            updated_description = "#{issue_description}\n\n### Reports (#{reports_count})\n"
+            updated_description = [updated_description, *test_captures_to_report_items(test_captures)].join("\n") unless test_captures.empty?
+            updated_description
+          end
+
+          def test_captures_to_report_items(test_captures)
+            test_captures.map do |ci_job_url, _, _|
+              report_list_item(GitlabQuality::TestTooling::TestResult::JsonTestResult.new(
+                'ci_job_url' => ci_job_url
+              ))
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      # Uses the API to create GitLab issues for any passed test coming from JSON test reports.
+      # We expect the test reports to come from rspec-retry, or from a new RSpec process where
+      # we retried failing specs.
+      #
+      # - Takes the JSON test reports like rspec-*.json (typically from rspec-retry gem)`
+      # - Takes a project where flaky test issues should be created
+      # - For every passed test in the report:
+      #   - Find issue by test hash
+      #   - Reopen issue if it already exists, but is closed
+      class FlakyTestIssue < ReportAsIssue
+        include Concerns::IssueReports
+
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', 'failure::flaky-test']).freeze
+        FOUND_IN_MR_LABEL     = 'found:in MR'
+        FOUND_IN_MASTER_LABEL = 'found:master'
+
+        def initialize(token:, input_files:, project: nil, merge_request_iid: nil, confidential: false, dry_run: false, **_kwargs)
+          super(token: token, input_files: input_files, project: project, confidential: confidential, dry_run: dry_run)
+          @merge_request_iid = merge_request_iid
+        end
+
+        private
+
+        attr_reader :merge_request_iid
+
+        def run!
+          puts "Reporting flaky tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          TestResults::Builder.new(files).test_results_per_file do |test_results|
+            puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
+
+            test_results.each do |test|
+              next if test.status != 'passed' # We only want failed tests that passed in the end
+
+              create_flaky_issue(test)
+            end
+          end
+        end
+
+        def new_issue_title(test)
+          "Flaky test in #{super}"
+        end
+
+        def new_issue_labels(_test)
+          found_label =
+            if !merge_request_iid || merge_request_iid.empty?
+              FOUND_IN_MASTER_LABEL
+            else
+              FOUND_IN_MR_LABEL
+            end
+
+          NEW_ISSUE_LABELS + [found_label]
+        end
+
+        def new_issue_description(test)
+          super + [
+            "Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html) " \
+            "to learn more about how to reproduce them.",
+            initial_reports_section(test)
+          ].compact.join("\n\n")
+        end
+
+        def create_flaky_issue(test)
+          puts "   => Finding existing issues for flaky test '#{test.name}' (run time: #{test.run_time} seconds)..."
+
+          issues = find_issues_by_hash(test_hash(test))
+          issues.each do |issue|
+            puts "   => Existing issue link #{issue.web_url}."
+
+            puts "   => Adding the flaky test to the existing issue..."
+            add_report_to_issue(issue, test)
+
+            if issue.state == 'closed'
+              puts "   => Issue is closed. Reopening it."
+              reopen_issue(issue)
+            end
+          end
+
+          create_issue(test) unless issues.any?
+        end
+
+        def add_report_to_issue(issue, test)
+          gitlab.edit_issue(iid: issue.iid, options: { description: add_report_to_issue_description(issue, test) })
+        end
+
+        def reopen_issue(issue)
+          gitlab.edit_issue(iid: issue.iid, options: { state_event: 'reopen' })
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb

@@ -7,7 +7,6 @@ module GitlabQuality
         SLOW_TEST_MESSAGE = '<!-- slow-test -->'
         SLOW_TEST_LABEL = '/label ~"rspec:slow test detected"'
         SLOW_TEST_NOTE_SOURCE_CODE = 'Generated by [`gitlab_quality-test_tooling`](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/blob/main/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb).'
-        SLOW_TEST_NOTE_FEEDBACK = 'Please [share your feedback and suggestions](https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/289).'
 
         def initialize(token:, input_files:, merge_request_iid:, project: nil, dry_run: false, **_kwargs)
           @project = project
@@ -70,7 +69,6 @@ module GitlabQuality
             SLOW_TEST_MESSAGE,
             SLOW_TEST_LABEL,
             ":tools: #{SLOW_TEST_NOTE_SOURCE_CODE}\n",
-            ":recycle: #{SLOW_TEST_NOTE_FEEDBACK}\n",
             "---\n",
             ":snail: Slow tests detected in this merge request. These slow tests might be related to this merge request's changes.",
             "<details><summary>Click to expand</summary>\n",

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -18,6 +18,7 @@ module GitlabQuality
       class RelateFailureIssue < ReportAsIssue
         include Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
+        include Concerns::IssueReports
         include Amatch
 
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
@@ -25,9 +26,7 @@ module GitlabQuality
         SPAM_THRESHOLD_FOR_FAILURE_ISSUES = 3
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m
-        JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
-        FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
-        REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
+
         NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
         IGNORED_FAILURES = [
           'Net::ReadTimeout',
@@ -37,18 +36,24 @@ module GitlabQuality
 
         MultipleIssuesFound = Class.new(StandardError)
 
-        def initialize(max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION, system_logs: [], base_issue_labels: Set.new, **kwargs)
+        def initialize(
+          max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION,
+          system_logs: [],
+          base_issue_labels: nil,
+          exclude_labels_for_search: nil,
+          **kwargs)
           super
           @max_diff_ratio = max_diff_ratio.to_f
           @system_logs = Dir.glob(system_logs)
           @base_issue_labels = Set.new(base_issue_labels)
+          @exclude_labels_for_search = Set.new(exclude_labels_for_search)
           @issue_type = 'issue'
           @commented_issue_list = Set.new
         end
 
         private
 
-        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels
+        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search
 
         def run!
           puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
@@ -86,7 +91,9 @@ module GitlabQuality
           begin
             issue = find_issue_and_update_reports(test)
 
-            create_issue(test) unless issue || test.quarantine?
+            issue = create_issue(test) unless issue || test.quarantine?
+
+            issue
           rescue MultipleIssuesFound => e
             warn(e.message)
           end
@@ -134,35 +141,15 @@ module GitlabQuality
 
         def pipeline_issues_with_similar_stacktrace(test)
           search_labels = (base_issue_labels + Set.new(%w[test failure::new])).to_a
+          not_labels = exclude_labels_for_search.to_a
           gitlab.find_issues(options: { labels: search_labels,
+                                        not: { labels: not_labels },
                                         created_after: past_timestamp(2) }).select do |issue|
             job_url_from_issue = failed_issue_job_url(issue)
 
             next if pipeline != pipeline_env_from_job_url(job_url_from_issue)
 
-            stack_trace_from_issue = cleaned_stack_trace_from_issue(issue)
-            stack_trace_from_test = cleaned_stacktrace_from_test(test)
-            diff_ratio = compare_stack_traces(stack_trace_from_test, stack_trace_from_issue)
-            diff_ratio < max_diff_ratio
-          end
-        end
-
-        def failed_issue_job_url(issue)
-          job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
-            # Legacy format
-            job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX).last
-        end
-
-        def failed_issue_job_urls(issue)
-          job_urls_from_description(issue.description, REPORT_ITEM_REGEX) +
-            # Legacy format
-            job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX)
-        end
-
-        def job_urls_from_description(issue_description, regex)
-          issue_description.lines.filter_map do |line|
-            match = line.match(regex)
-            match[:job_url] if match
+            compare_stack_traces(cleaned_stack_trace_from_test(test), cleaned_stack_trace_from_issue(issue)) < max_diff_ratio
           end
         end
 
@@ -182,7 +169,11 @@ module GitlabQuality
         end
 
         def failure_issues(test)
-          find_issues(test, (base_issue_labels + Set.new(%w[test])).to_a) # Search for opened and closed issues
+          find_issues_for_test(
+            test,
+            labels: base_issue_labels + Set.new(%w[test]),
+            not_labels: exclude_labels_for_search
+          )
         end
 
         def full_stacktrace(test)
@@ -203,7 +194,7 @@ module GitlabQuality
           remove_unique_resource_names(relevant_issue_stacktrace)
         end
 
-        def cleaned_stacktrace_from_test(test)
+        def cleaned_stack_trace_from_test(test)
           test_failure_stacktrace = sanitize_stacktrace(full_stacktrace(test),
             FAILURE_STACKTRACE_REGEX) || full_stacktrace(test)
           remove_unique_resource_names(test_failure_stacktrace)
@@ -219,7 +210,7 @@ module GitlabQuality
         end
 
         def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
-          clean_first_test_failure_stacktrace = cleaned_stacktrace_from_test(test)
+          clean_first_test_failure_stacktrace = cleaned_stack_trace_from_test(test)
           # Search with the `search` param returns 500 errors, so we filter by `base_issue_labels` and then filter further in Ruby
           failure_issues(test).each_with_object({}) do |issue, memo|
             clean_relevant_issue_stacktrace = cleaned_stack_trace_from_issue(issue)
@@ -287,7 +278,7 @@ module GitlabQuality
             "```\n#{full_stacktrace(test)}\n```",
             screenshot_section(test),
             system_log_errors_section(test),
-            reports_section(test)
+            initial_reports_section(test)
           ].compact.join("\n\n")
         end
 
@@ -310,18 +301,6 @@ module GitlabQuality
           section
         end
 
-        def reports_section(test)
-          <<~REPORTS
-          ### Reports (1)
-
-          #{report_list_item(test)}
-          REPORTS
-        end
-
-        def report_list_item(test)
-          "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
-        end
-
         def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
           (Set.new(base_issue_labels) + (super << pipeline_name_label)).to_a
         end
@@ -346,7 +325,7 @@ module GitlabQuality
           state_event = issue.state == 'closed' ? 'reopen' : nil
 
           issue_attrs = {
-            description: up_to_date_issue_description(issue.description, test),
+            description: add_report_to_issue_description(issue, test),
             labels: up_to_date_labels(test: test, issue: issue)
           }
           issue_attrs[:state_event] = state_event if state_event
@@ -355,26 +334,6 @@ module GitlabQuality
           puts "  => Added a report in '#{issue.title}': #{issue.web_url}!"
         end
 
-        def up_to_date_issue_description(issue_description, test)
-          # We include the number of reports in the header, for visibility.
-          new_issue_description =
-            if issue_description.include?('### Reports')
-              # We count the number of existing reports.
-              reports_count = issue_description
-                .scan(REPORT_ITEM_REGEX)
-                .size.to_i + 1
-              issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
-            else # For issue with the legacy format, we add the Reports section
-              reports_count = issue_description
-                .scan(JOB_URL_REGEX)
-                .size.to_i + 1
-
-              "#{issue_description}\n\n### Reports (#{reports_count})"
-            end
-
-          "#{new_issue_description}\n#{report_list_item(test)}"
-        end
-
         def new_issue_title(test)
           "Failure in #{super}"
         end

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -39,7 +39,10 @@ module GitlabQuality
         end
 
         def test_hash(test)
-          OpenSSL::Digest.hexdigest('SHA256', "#{test.file}#{test.name}")
+          # Should not be more than 50 characters if we want it indexed.
+          #
+          # See https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/issues/27#note_1607276486
+          OpenSSL::Digest.hexdigest('SHA256', "#{test.file}#{test.name}")[..40]
         end
 
         def new_issue_description(test)
@@ -48,7 +51,8 @@ module GitlabQuality
 
           | Field | Value |
           | ------ | ------ |
-          | File | #{test.test_file_link} |
+          | File URL | #{test.test_file_link} |
+          | Filename | `#{test.file}` |
           | Description | `#{test.name}` |
           | Test level | #{test.level} |
           | Hash | `#{test_hash(test)}` |
@@ -134,15 +138,21 @@ module GitlabQuality
           labels
         end
 
-        def find_issues(test, labels, state: nil)
-          search_options = { labels: labels.to_a }
+        def find_issues_by_hash(test_hash)
+          gitlab.find_issues_by_hash(test_hash)
+        end
+
+        def find_issues_for_test(test, labels:, not_labels: Set.new, state: nil)
+          search_options = { labels: labels.to_a, not: { labels: not_labels.to_a } }
           search_options[:state] = state if state
 
-          gitlab.find_issues(options: search_options).find_all do |issue|
-            issue_title = issue.title.strip
-            test_file_path_found = !test.file.to_s.empty? && issue_title.include?(partial_file_path(test.file))
-            issue_title.include?(test.name) || test_file_path_found
-          end
+          gitlab.find_issues(options: search_options).find_all { |issue| issue_match_test?(issue, test) }
+        end
+
+        def issue_match_test?(issue, test)
+          issue_title = issue.title.strip
+          test_file_path_found = !test.file.to_s.empty? && issue_title.include?(partial_file_path(test.file))
+          issue_title.include?(test.name) || test_file_path_found
         end
 
         def pipeline_name_label
@@ -160,7 +170,7 @@ module GitlabQuality
           when 'customers-gitlab-com'
             'found:customers.stg.gitlab.com'
           else
-            raise "No `found:*` label for the `#{pipeline}` pipeline!"
+            puts "  => [DEBUG] No `found:*` label for the `#{pipeline}` pipeline!"
           end
         end
 

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -68,7 +68,7 @@ module GitlabQuality
         def create_slow_issue(test)
           puts "   => Finding existing issues for slow test '#{test.name}' (run time: #{test.run_time} seconds)..."
 
-          issues = find_issues(test, SEARCH_LABELS, state: 'opened')
+          issues = find_issues_for_test(test, labels: SEARCH_LABELS, state: 'opened')
 
           if issues.blank?
             issues << create_issue(test)

data/lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb

@@ -16,8 +16,8 @@ module GitlabQuality
 
         def invoke!
           Dir.glob(input_files).each do |input_file|
-            rewrite_schreenshot_paths_in_junit_file(input_file)
-            rewrite_schreenshot_paths_in_json_file(input_file.gsub('.xml', '.json'))
+            rewrite_screenshot_paths_in_junit_file(input_file)
+            rewrite_screenshot_paths_in_json_file(input_file.gsub('.xml', '.json'))
           end
         end
 
@@ -25,7 +25,7 @@ module GitlabQuality
 
         attr_reader :input_files
 
-        def rewrite_schreenshot_paths_in_junit_file(junit_file)
+        def rewrite_screenshot_paths_in_junit_file(junit_file)
           File.write(
             junit_file,
             rewrite_each_junit_screenshot_path(junit_file).to_s
@@ -34,7 +34,7 @@ module GitlabQuality
           puts "Saved #{junit_file}"
         end
 
-        def rewrite_schreenshot_paths_in_json_file(json_file)
+        def rewrite_screenshot_paths_in_json_file(json_file)
           File.write(
             json_file,
             JSON.pretty_generate(
@@ -58,7 +58,7 @@ module GitlabQuality
             examples = report['examples']
 
             examples.each do |example|
-              next unless example['screenshot'].present?
+              next unless example['screenshot'].present? && example['screenshot']['image'].present?
 
               example['screenshot']['image'] =
                 remove_container_absolute_path_prefix(example.dig('screenshot', 'image'), test_artifacts_directory(json_file))

data/lib/gitlab_quality/test_tooling/test_result/json_test_result.rb

@@ -18,7 +18,25 @@ module GitlabQuality
         ].freeze
 
         def name
-          report.fetch('full_description')
+          # If we see a string representation of an object in a test full_description, we discard it.
+          #
+          # This is to ensure that tests would have a reproducible name, in case they don't have a name.
+          #
+          # Test example:
+          #
+          # it { is_expected.to eq(secondary_node) }
+          #
+          # Would have its full_description as follows:
+          #
+          # Gitlab::Geo.proxied_site on a primary for a proxied request with a proxy extra data header
+          # for an existing site is expected to eq #<GeoNode id: 116, primary: false, oauth_application_id: 97
+          # , enabled: true, access_key: [FILTERED], e...pdated_at: "2023-10-10 08:49:49.797128469 +0000",
+          # sync_object_storage: true, secret_access_key: nil>
+          #
+          # Which would change for every test run due to the timestamps.
+          #
+          # See https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/merge_requests/77#note_1608793804
+          report.fetch('full_description').split('#<').first
         end
 
         def file

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "1.5.3"
+    VERSION = "1.7.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 1.5.3
+  version: 1.7.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-11-17 00:00:00.000000000 Z
+date: 2023-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.10.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -332,6 +346,7 @@ description: A collection of test-related tools.
 email:
 - quality@gitlab.com
 executables:
+- flaky-test-issues
 - generate-test-session
 - post-to-slack
 - prepare-stage-reports
@@ -357,6 +372,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/flaky-test-issues
 - exe/generate-test-session
 - exe/post-to-slack
 - exe/prepare-stage-reports
@@ -374,8 +390,10 @@ files:
 - lib/gitlab_quality/test_tooling/labels_inference.rb
 - lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb
 - lib/gitlab_quality/test_tooling/report/concerns/group_and_category_labels.rb
+- lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb
 - lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
 - lib/gitlab_quality/test_tooling/report/concerns/utils.rb
+- lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb
 - lib/gitlab_quality/test_tooling/report/generate_test_session.rb
 - lib/gitlab_quality/test_tooling/report/issue_logger.rb
 - lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb