gitlab_quality-test_tooling 1.5.4 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: acc009f4f6d5f4be05fa85cd3fffc68778d415d970ca96f22ace03c8a86f9488
-  data.tar.gz: 5c31655a8245c632e595431fb99ec14852bfd4f9b9778d09e8b8d0ae3393dbc1
+  metadata.gz: 4ee3407afd5274be75c1bb63c7f7aeeb0d2353cf62f0fb75c1393bd2103c8c1e
+  data.tar.gz: a4965dd9d979a58c4202c6f25315d60e08d597d51322db491a077a0ae797cf34
 SHA512:
-  metadata.gz: 754b2ed722bc050515ceb9f4691bc49da1e63c27c1ff44b3cd97bc597d6691f947a2390b63b41e980b4229332817671bc9cad351fee4455d424ce66f1d54cf03
-  data.tar.gz: ff2b356903435078c53b6234e69d57e272c9f367cd01518158fda38a94372ceeb1fe1e2f2cebec16e2aaa0dd4db9c98592c36631c858119d3340cf4bcd961bda
+  metadata.gz: 1619807ec2456f02c9d4a1c09dfb0aeba598de7d51fe7a0377c1e5a9f3f9fc82cf607fa34387bb5ee1691ad28b5637e75c18c38ce068269c3bba76e3c902df22
+  data.tar.gz: '0940057d402dfb8fbcd3a6d2d09f7ff698e26d140ebdda659fcc5347445574fa7f398b2bd91e8a39c872b0f8a9f368fa1f734e59e10262790ed2d507fb97e514'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (1.5.4)
+    gitlab_quality-test_tooling (1.8.0)
       activesupport (>= 6.1, < 7.2)
       amatch (~> 0.4.1)
       gitlab (~> 4.19)
@@ -29,6 +29,7 @@ GEM
     ast (2.4.2)
     backport (1.2.0)
     benchmark (0.2.1)
+    byebug (11.1.3)
     claide (1.1.0)
     claide-plugins (0.9.2)
       cork
@@ -160,6 +161,9 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
+    pry-byebug (3.10.1)
+      byebug (~> 11.0)
+      pry (>= 0.13, < 0.15)
     public_suffix (5.0.1)
     racc (1.6.2)
     rack (3.0.7)
@@ -279,6 +283,7 @@ DEPENDENCIES
   gitlab_quality-test_tooling!
   guard-rspec (~> 4.7)
   lefthook (~> 1.3)
+  pry-byebug (= 3.10.1)
   rake (~> 13.0)
   rspec (~> 3.12)
   simplecov (~> 0.22)

data/README.md

@@ -77,14 +77,20 @@ Usage: exe/prepare-stage-reports [options]
 Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)
 Usage: exe/relate-failure-issue [options]
     -i, --input-files INPUT_FILES    RSpec report files (JSON or JUnit XML)
+    -m METRICS_FILES,                Test metrics files (JSON)
+        --metrics-files
         --max-diff-ratio MAX_DIFF_RATO
                                      Max stacktrace diff ratio for failure issues detection
     -p, --project PROJECT            Can be an integer or a group/project string
     -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+    -r RELATED_ISSUES_FILE,          The file path for the related issues
+        --related-issues-file
         --system-log-files SYSTEM_LOG_FILES
                                      Include errors from system logs in failure issues
         --base-issue-labels BASE_ISSUE_LABELS
                                      Labels to add to new failure issues
+        --exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH
+                                     Labels to exclude when searching for existing issues
         --confidential               Makes created new issues confidential
         --dry-run                    Perform a dry-run (don't create or update issues)
     -v, --version                    Show the version
@@ -133,6 +139,21 @@ Usage: exe/slow-test-issue [options]
     -h, --help                       Show the usage
 ```
 
+### `exe/flaky-test-issues`
+
+```shell
+Purpose: Purpose: Create flaky test issues for any passed test coming from rspec-retry JSON report files
+Usage: exe/flaky-test-issue [options]
+    -i, --input-files INPUT_FILES    JSON rspec-retry report files
+    -p, --project PROJECT            Can be an integer or a group/project string
+    -m MERGE_REQUEST_IID,            An integer merge request IID
+        --merge_request_iid
+    -t, --token TOKEN                A valid access token with `api` scope and Maintainer permission in PROJECT
+        --dry-run                    Perform a dry-run (don't create note)
+    -v, --version                    Show the version
+    -h, --help                       Show the usage
+```
+
 ### `exe/slow-test-merge-request-report-note`
 
 ```shell

data/exe/flaky-test-issues

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+params = {}
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-i', '--input-files INPUT_FILES', String, 'JSON rspec-retry report files') do |input_files|
+    params[:input_files] = input_files
+  end
+
+  opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
+    params[:project] = project
+  end
+
+  opts.on('-m', '--merge_request_iid MERGE_REQUEST_IID', String, 'An integer merge request IID') do |merge_request_iid|
+    params[:merge_request_iid] = merge_request_iid
+  end
+
+  opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
+    params[:token] = token
+  end
+
+  opts.on('--dry-run', "Perform a dry-run (don't create issues)") do
+    params[:dry_run] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show the usage') do
+    puts "Purpose: Create flaky test issues for any passed test coming from rspec-retry JSON report files."
+    puts opts
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any?
+  GitlabQuality::TestTooling::Report::FlakyTestIssue.new(**params).invoke!
+else
+  puts options
+  exit 1
+end

data/exe/relate-failure-issue

@@ -15,6 +15,10 @@ options = OptionParser.new do |opts|
     params[:input_files] = input_files
   end
 
+  opts.on('-m', '--metrics-files METRICS_FILES', String, 'Test metrics files (JSON)') do |metrics_files|
+    params[:metrics_files] = metrics_files
+  end
+
   opts.on('--max-diff-ratio MAX_DIFF_RATO', Float, 'Max stacktrace diff ratio for failure issues detection') do |max_diff_ratio|
     params[:max_diff_ratio] = max_diff_ratio
   end
@@ -41,6 +45,11 @@ options = OptionParser.new do |opts|
     params[:base_issue_labels] = base_issue_labels.split(',')
   end
 
+  opts.on('--exclude-labels-for-search EXCLUDE_LABELS_FOR_SEARCH', String,
+    'Labels to exclude when searching for existing issues') do |exclude_labels_for_search|
+    params[:exclude_labels_for_search] = exclude_labels_for_search.split(',')
+  end
+
   opts.on('--confidential', "Makes created new issues confidential") do
     params[:confidential] = true
   end

data/lib/gitlab_quality/test_tooling/gitlab_issue_client.rb

@@ -55,6 +55,16 @@ module GitlabQuality
         end
       end
 
+      def find_issues_by_hash(test_hash, &select)
+        select ||= :itself
+
+        handle_gitlab_client_exceptions do
+          client.search_in_project(project, 'issues', test_hash)
+            .auto_paginate
+            .select(&select)
+        end
+      end
+
       def find_issue_discussions(iid:)
         handle_gitlab_client_exceptions do
           client.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate

data/lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      module Concerns
+        module IssueReports
+          JOB_URL_REGEX                = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
+          FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
+          REPORT_ITEM_REGEX            = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
+
+          def initial_reports_section(test)
+            <<~REPORTS
+            ### Reports (1)
+
+            #{report_list_item(test)}
+            REPORTS
+          end
+
+          def add_report_to_issue_description(issue, test)
+            issue_description = issue.description
+
+            # We include the number of reports in the header, for visibility.
+            new_issue_description =
+              if issue_description.include?('### Reports')
+                # We count the number of existing reports.
+                reports_count = issue_description
+                  .scan(REPORT_ITEM_REGEX)
+                  .size.to_i + 1
+                issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
+              else # For issue with the legacy format, we add the Reports section
+                update_legacy_issue_description(issue_description)
+              end
+
+            [new_issue_description, report_list_item(test)].join("\n")
+          end
+
+          def failed_issue_job_url(issue)
+            job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
+              # Legacy format
+              job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX).last
+          end
+
+          def failed_issue_job_urls(issue)
+            job_urls_from_description(issue.description, REPORT_ITEM_REGEX) +
+              # Legacy format
+              job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX)
+          end
+
+          private
+
+          def report_list_item(test)
+            "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
+          end
+
+          def job_urls_from_description(issue_description, regex)
+            issue_description.lines.filter_map do |line|
+              match = line.match(regex)
+              match[:job_url] if match
+            end
+          end
+
+          def update_legacy_issue_description(issue_description)
+            test_captures = issue_description.scan(JOB_URL_REGEX)
+            reports_count = test_captures.size.to_i + 1
+
+            updated_description = "#{issue_description}\n\n### Reports (#{reports_count})\n"
+            updated_description = [updated_description, *test_captures_to_report_items(test_captures)].join("\n") unless test_captures.empty?
+            updated_description
+          end
+
+          def test_captures_to_report_items(test_captures)
+            test_captures.map do |ci_job_url, _, _|
+              report_list_item(GitlabQuality::TestTooling::TestResult::JsonTestResult.new(
+                'ci_job_url' => ci_job_url
+              ))
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Report
+      # Uses the API to create GitLab issues for any passed test coming from JSON test reports.
+      # We expect the test reports to come from rspec-retry, or from a new RSpec process where
+      # we retried failing specs.
+      #
+      # - Takes the JSON test reports like rspec-*.json (typically from rspec-retry gem)`
+      # - Takes a project where flaky test issues should be created
+      # - For every passed test in the report:
+      #   - Find issue by test hash
+      #   - Reopen issue if it already exists, but is closed
+      class FlakyTestIssue < ReportAsIssue
+        include Concerns::IssueReports
+
+        NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'priority::3', 'severity::3', 'failure::flaky-test']).freeze
+        FOUND_IN_MR_LABEL     = 'found:in MR'
+        FOUND_IN_MASTER_LABEL = 'found:master'
+
+        def initialize(token:, input_files:, project: nil, merge_request_iid: nil, confidential: false, dry_run: false, **_kwargs)
+          super(token: token, input_files: input_files, project: project, confidential: confidential, dry_run: dry_run)
+          @merge_request_iid = merge_request_iid
+        end
+
+        private
+
+        attr_reader :merge_request_iid
+
+        def run!
+          puts "Reporting flaky tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
+
+          TestResults::Builder.new(files).test_results_per_file do |test_results|
+            puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
+
+            test_results.each do |test|
+              next if test.status != 'passed' # We only want failed tests that passed in the end
+
+              create_flaky_issue(test)
+            end
+          end
+        end
+
+        def new_issue_title(test)
+          "Flaky test in #{super}"
+        end
+
+        def new_issue_labels(_test)
+          found_label =
+            if !merge_request_iid || merge_request_iid.empty?
+              FOUND_IN_MASTER_LABEL
+            else
+              FOUND_IN_MR_LABEL
+            end
+
+          NEW_ISSUE_LABELS + [found_label]
+        end
+
+        def new_issue_description(test)
+          super + [
+            "Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html) " \
+            "to learn more about how to reproduce them.",
+            initial_reports_section(test)
+          ].compact.join("\n\n")
+        end
+
+        def create_flaky_issue(test)
+          puts "   => Finding existing issues for flaky test '#{test.name}' (run time: #{test.run_time} seconds)..."
+
+          issues = find_issues_by_hash(test_hash(test))
+          issues.each do |issue|
+            puts "   => Existing issue link #{issue.web_url}."
+
+            puts "   => Adding the flaky test to the existing issue..."
+            add_report_to_issue(issue, test)
+
+            if issue.state == 'closed'
+              puts "   => Issue is closed. Reopening it."
+              reopen_issue(issue)
+            end
+          end
+
+          create_issue(test) unless issues.any?
+        end
+
+        def add_report_to_issue(issue, test)
+          gitlab.edit_issue(iid: issue.iid, options: { description: add_report_to_issue_description(issue, test) })
+        end
+
+        def reopen_issue(issue)
+          gitlab.edit_issue(iid: issue.iid, options: { state_event: 'reopen' })
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb

@@ -7,7 +7,6 @@ module GitlabQuality
         SLOW_TEST_MESSAGE = '<!-- slow-test -->'
         SLOW_TEST_LABEL = '/label ~"rspec:slow test detected"'
         SLOW_TEST_NOTE_SOURCE_CODE = 'Generated by [`gitlab_quality-test_tooling`](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/blob/main/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb).'
-        SLOW_TEST_NOTE_FEEDBACK = 'Please [share your feedback and suggestions](https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/289).'
 
         def initialize(token:, input_files:, merge_request_iid:, project: nil, dry_run: false, **_kwargs)
           @project = project
@@ -70,7 +69,6 @@ module GitlabQuality
             SLOW_TEST_MESSAGE,
             SLOW_TEST_LABEL,
             ":tools: #{SLOW_TEST_NOTE_SOURCE_CODE}\n",
-            ":recycle: #{SLOW_TEST_NOTE_FEEDBACK}\n",
             "---\n",
             ":snail: Slow tests detected in this merge request. These slow tests might be related to this merge request's changes.",
             "<details><summary>Click to expand</summary>\n",

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -18,6 +18,7 @@ module GitlabQuality
       class RelateFailureIssue < ReportAsIssue
         include Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
+        include Concerns::IssueReports
         include Amatch
 
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
@@ -25,9 +26,7 @@ module GitlabQuality
         SPAM_THRESHOLD_FOR_FAILURE_ISSUES = 3
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m
-        JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
-        FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
-        REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
+
         NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
         IGNORED_FAILURES = [
           'Net::ReadTimeout',
@@ -37,18 +36,26 @@ module GitlabQuality
 
         MultipleIssuesFound = Class.new(StandardError)
 
-        def initialize(max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION, system_logs: [], base_issue_labels: Set.new, **kwargs)
+        def initialize(
+          max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION,
+          system_logs: [],
+          base_issue_labels: nil,
+          exclude_labels_for_search: nil,
+          metrics_files: [],
+          **kwargs)
           super
           @max_diff_ratio = max_diff_ratio.to_f
           @system_logs = Dir.glob(system_logs)
           @base_issue_labels = Set.new(base_issue_labels)
+          @exclude_labels_for_search = Set.new(exclude_labels_for_search)
           @issue_type = 'issue'
           @commented_issue_list = Set.new
+          @metrics_files = Array(metrics_files)
         end
 
         private
 
-        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels
+        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files
 
         def run!
           puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
@@ -61,16 +68,39 @@ module GitlabQuality
           write_issues_log_file
         end
 
+        def test_metric_collections
+          @test_metric_collections ||= Dir.glob(metrics_files).map do |path|
+            TestMetrics::JsonTestMetricCollection.new(path)
+          end
+        end
+
         def process_test_results(test_results)
           systemic_failures = systemic_failures_for_test_results(test_results)
 
           test_results.each do |test|
             collect_issues(test, relate_failure_to_issue(test)) if should_report?(test, systemic_failures)
+
+            copy_failure_issue_to_test_metrics(test) if metrics_files.any?
           end
 
           test_results.write
         end
 
+        def copy_failure_issue_to_test_metrics(test)
+          failure_issue = test.failure_issue
+
+          return unless failure_issue
+
+          test_metric_collections.find do |test_metric_collection|
+            test_metric = test_metric_collection.metric_for_test_id(test.example_id)
+
+            if test_metric
+              test_metric.fields['failure_issue'] = failure_issue
+              test_metric_collection.write
+            end
+          end
+        end
+
         def systemic_failures_for_test_results(test_results)
           test_results
             .flat_map { |test| test.failures.map { |failure| failure['message'].lines.first.chomp } }
@@ -136,35 +166,15 @@ module GitlabQuality
 
         def pipeline_issues_with_similar_stacktrace(test)
           search_labels = (base_issue_labels + Set.new(%w[test failure::new])).to_a
+          not_labels = exclude_labels_for_search.to_a
           gitlab.find_issues(options: { labels: search_labels,
+                                        not: { labels: not_labels },
                                         created_after: past_timestamp(2) }).select do |issue|
             job_url_from_issue = failed_issue_job_url(issue)
 
             next if pipeline != pipeline_env_from_job_url(job_url_from_issue)
 
-            stack_trace_from_issue = cleaned_stack_trace_from_issue(issue)
-            stack_trace_from_test = cleaned_stacktrace_from_test(test)
-            diff_ratio = compare_stack_traces(stack_trace_from_test, stack_trace_from_issue)
-            diff_ratio < max_diff_ratio
-          end
-        end
-
-        def failed_issue_job_url(issue)
-          job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
-            # Legacy format
-            job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX).last
-        end
-
-        def failed_issue_job_urls(issue)
-          job_urls_from_description(issue.description, REPORT_ITEM_REGEX) +
-            # Legacy format
-            job_urls_from_description(issue.description, FAILED_JOB_DESCRIPTION_REGEX)
-        end
-
-        def job_urls_from_description(issue_description, regex)
-          issue_description.lines.filter_map do |line|
-            match = line.match(regex)
-            match[:job_url] if match
+            compare_stack_traces(cleaned_stack_trace_from_test(test), cleaned_stack_trace_from_issue(issue)) < max_diff_ratio
           end
         end
 
@@ -184,7 +194,11 @@ module GitlabQuality
         end
 
         def failure_issues(test)
-          find_issues(test, (base_issue_labels + Set.new(%w[test])).to_a) # Search for opened and closed issues
+          find_issues_for_test(
+            test,
+            labels: base_issue_labels + Set.new(%w[test]),
+            not_labels: exclude_labels_for_search
+          )
         end
 
         def full_stacktrace(test)
@@ -205,7 +219,7 @@ module GitlabQuality
           remove_unique_resource_names(relevant_issue_stacktrace)
         end
 
-        def cleaned_stacktrace_from_test(test)
+        def cleaned_stack_trace_from_test(test)
           test_failure_stacktrace = sanitize_stacktrace(full_stacktrace(test),
             FAILURE_STACKTRACE_REGEX) || full_stacktrace(test)
           remove_unique_resource_names(test_failure_stacktrace)
@@ -221,7 +235,7 @@ module GitlabQuality
         end
 
         def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
-          clean_first_test_failure_stacktrace = cleaned_stacktrace_from_test(test)
+          clean_first_test_failure_stacktrace = cleaned_stack_trace_from_test(test)
           # Search with the `search` param returns 500 errors, so we filter by `base_issue_labels` and then filter further in Ruby
           failure_issues(test).each_with_object({}) do |issue, memo|
             clean_relevant_issue_stacktrace = cleaned_stack_trace_from_issue(issue)
@@ -289,7 +303,7 @@ module GitlabQuality
             "```\n#{full_stacktrace(test)}\n```",
             screenshot_section(test),
             system_log_errors_section(test),
-            reports_section(test)
+            initial_reports_section(test)
           ].compact.join("\n\n")
         end
 
@@ -312,18 +326,6 @@ module GitlabQuality
           section
         end
 
-        def reports_section(test)
-          <<~REPORTS
-          ### Reports (1)
-
-          #{report_list_item(test)}
-          REPORTS
-        end
-
-        def report_list_item(test)
-          "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
-        end
-
         def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
           (Set.new(base_issue_labels) + (super << pipeline_name_label)).to_a
         end
@@ -348,7 +350,7 @@ module GitlabQuality
           state_event = issue.state == 'closed' ? 'reopen' : nil
 
           issue_attrs = {
-            description: up_to_date_issue_description(issue.description, test),
+            description: add_report_to_issue_description(issue, test),
             labels: up_to_date_labels(test: test, issue: issue)
           }
           issue_attrs[:state_event] = state_event if state_event
@@ -357,26 +359,6 @@ module GitlabQuality
           puts "  => Added a report in '#{issue.title}': #{issue.web_url}!"
         end
 
-        def up_to_date_issue_description(issue_description, test)
-          # We include the number of reports in the header, for visibility.
-          new_issue_description =
-            if issue_description.include?('### Reports')
-              # We count the number of existing reports.
-              reports_count = issue_description
-                .scan(REPORT_ITEM_REGEX)
-                .size.to_i + 1
-              issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
-            else # For issue with the legacy format, we add the Reports section
-              reports_count = issue_description
-                .scan(JOB_URL_REGEX)
-                .size.to_i + 1
-
-              "#{issue_description}\n\n### Reports (#{reports_count})"
-            end
-
-          "#{new_issue_description}\n#{report_list_item(test)}"
-        end
-
         def new_issue_title(test)
           "Failure in #{super}"
         end

data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb

@@ -39,7 +39,10 @@ module GitlabQuality
         end
 
         def test_hash(test)
-          OpenSSL::Digest.hexdigest('SHA256', "#{test.file}#{test.name}")
+          # Should not be more than 50 characters if we want it indexed.
+          #
+          # See https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/issues/27#note_1607276486
+          OpenSSL::Digest.hexdigest('SHA256', "#{test.file}#{test.name}")[..40]
         end
 
         def new_issue_description(test)
@@ -48,7 +51,8 @@ module GitlabQuality
 
           | Field | Value |
           | ------ | ------ |
-          | File | #{test.test_file_link} |
+          | File URL | #{test.test_file_link} |
+          | Filename | `#{test.file}` |
           | Description | `#{test.name}` |
           | Test level | #{test.level} |
           | Hash | `#{test_hash(test)}` |
@@ -134,15 +138,21 @@ module GitlabQuality
           labels
         end
 
-        def find_issues(test, labels, state: nil)
-          search_options = { labels: labels.to_a }
+        def find_issues_by_hash(test_hash)
+          gitlab.find_issues_by_hash(test_hash)
+        end
+
+        def find_issues_for_test(test, labels:, not_labels: Set.new, state: nil)
+          search_options = { labels: labels.to_a, not: { labels: not_labels.to_a } }
           search_options[:state] = state if state
 
-          gitlab.find_issues(options: search_options).find_all do |issue|
-            issue_title = issue.title.strip
-            test_file_path_found = !test.file.to_s.empty? && issue_title.include?(partial_file_path(test.file))
-            issue_title.include?(test.name) || test_file_path_found
-          end
+          gitlab.find_issues(options: search_options).find_all { |issue| issue_match_test?(issue, test) }
+        end
+
+        def issue_match_test?(issue, test)
+          issue_title = issue.title.strip
+          test_file_path_found = !test.file.to_s.empty? && issue_title.include?(partial_file_path(test.file))
+          issue_title.include?(test.name) || test_file_path_found
         end
 
         def pipeline_name_label

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -68,7 +68,7 @@ module GitlabQuality
         def create_slow_issue(test)
           puts "   => Finding existing issues for slow test '#{test.name}' (run time: #{test.run_time} seconds)..."
 
-          issues = find_issues(test, SEARCH_LABELS, state: 'opened')
+          issues = find_issues_for_test(test, labels: SEARCH_LABELS, state: 'opened')
 
           if issues.blank?
             issues << create_issue(test)

data/lib/gitlab_quality/test_tooling/test_metric/json_test_metric.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module TestMetric
+      class JsonTestMetric
+        attr_reader :metric
+
+        def initialize(metric)
+          @metric = metric
+        end
+
+        def name
+          metric.fetch('name')
+        end
+
+        def time
+          metric.fetch('time')
+        end
+
+        def tags
+          @tags ||= metric.fetch('tags')
+        end
+
+        def fields
+          @fields ||= metric.fetch('fields')
+        end
+
+        def to_json(*options)
+          as_json.to_json(*options)
+        end
+
+        private
+
+        def as_json
+          {
+            name: name,
+            time: time,
+            tags: tags,
+            fields: fields
+          }
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module TestMetrics
+      class JsonTestMetricCollection
+        include Enumerable
+
+        attr_reader :path, :metrics
+
+        def initialize(path)
+          @path = path
+          @metrics = process
+        end
+
+        def metric_for_test_id(test_id)
+          metrics.find do |metric|
+            metric.fields['id'] == test_id
+          end
+        end
+
+        def write
+          File.write(path, JSON.pretty_generate(metrics))
+        end
+
+        private
+
+        def parse
+          JSON.parse(File.read(path))
+        rescue JSON::ParserError
+          Runtime::Logger.debug("#{self.class.name}##{__method__} attempted to parse invalid JSON at path: #{path}")
+          {}
+        end
+
+        def process
+          parse.map do |test|
+            GitlabQuality::TestTooling::TestMetric::JsonTestMetric.new(test)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_result/json_test_result.rb

@@ -18,7 +18,25 @@ module GitlabQuality
         ].freeze
 
         def name
-          report.fetch('full_description')
+          # If we see a string representation of an object in a test full_description, we discard it.
+          #
+          # This is to ensure that tests would have a reproducible name, in case they don't have a name.
+          #
+          # Test example:
+          #
+          # it { is_expected.to eq(secondary_node) }
+          #
+          # Would have its full_description as follows:
+          #
+          # Gitlab::Geo.proxied_site on a primary for a proxied request with a proxy extra data header
+          # for an existing site is expected to eq #<GeoNode id: 116, primary: false, oauth_application_id: 97
+          # , enabled: true, access_key: [FILTERED], e...pdated_at: "2023-10-10 08:49:49.797128469 +0000",
+          # sync_object_storage: true, secret_access_key: nil>
+          #
+          # Which would change for every test run due to the timestamps.
+          #
+          # See https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/merge_requests/77#note_1608793804
+          report.fetch('full_description').split('#<').first
         end
 
         def file

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "1.5.4"
+    VERSION = "1.8.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 1.5.4
+  version: 1.8.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-11-22 00:00:00.000000000 Z
+date: 2023-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.10.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -332,6 +346,7 @@ description: A collection of test-related tools.
 email:
 - quality@gitlab.com
 executables:
+- flaky-test-issues
 - generate-test-session
 - post-to-slack
 - prepare-stage-reports
@@ -357,6 +372,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/flaky-test-issues
 - exe/generate-test-session
 - exe/post-to-slack
 - exe/prepare-stage-reports
@@ -374,8 +390,10 @@ files:
 - lib/gitlab_quality/test_tooling/labels_inference.rb
 - lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb
 - lib/gitlab_quality/test_tooling/report/concerns/group_and_category_labels.rb
+- lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb
 - lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
 - lib/gitlab_quality/test_tooling/report/concerns/utils.rb
+- lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb
 - lib/gitlab_quality/test_tooling/report/generate_test_session.rb
 - lib/gitlab_quality/test_tooling/report/issue_logger.rb
 - lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb
@@ -404,6 +422,8 @@ files:
 - lib/gitlab_quality/test_tooling/system_logs/log_types/rails/graphql_log.rb
 - lib/gitlab_quality/test_tooling/system_logs/shared_fields.rb
 - lib/gitlab_quality/test_tooling/system_logs/system_logs_formatter.rb
+- lib/gitlab_quality/test_tooling/test_metric/json_test_metric.rb
+- lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb
 - lib/gitlab_quality/test_tooling/test_result/base_test_result.rb
 - lib/gitlab_quality/test_tooling/test_result/j_unit_test_result.rb
 - lib/gitlab_quality/test_tooling/test_result/json_test_result.rb