gitlab_quality-test_tooling 1.22.0 → 1.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 755ed1e2f1e0a7ac2e1d1264eeed686c76fb23c90d5eb6ebfbcaa64f931045f6
-  data.tar.gz: 5bbc1d0b7b93d8db268e2b0b0081b3878ff4117e97b2a3426023180cdec945ec
+  metadata.gz: '009e41374e42e637a571410da7cd95aa23ab4d5a6f1599f080a0ccea8432b820'
+  data.tar.gz: b81d4043e16bc5da888a102d3765250691acb705bb3edac3762c6baad0fe490f
 SHA512:
-  metadata.gz: 3078229a51d390624b6e148444735f38581383319e377db26986d7d1b77d2b53447b440d1fc3eecb8eaf52e92a3cc0c927e4ab44930e34bfb83165e88b011097
-  data.tar.gz: a0ac07071de3a7fe50144f2527faff4a6e5c5530477681b5ebf1137fc1d89aa8991f2aa88ef255bd0ffa84c3a3d8440ab91e3b3b7ef4c28fed9ae3b11c7709e0
+  metadata.gz: f1d34b6b4862000282c042e25b5f0cd02e23e72dee4307d0eb9d2309a0e5d352dfeeeadab2f5da7d01b91fe798a192bf8f2ed075608e9f73854842d107bef55a
+  data.tar.gz: eb3746a140cb622b9ede27514cb26ea7681b1bafffc5320dd9ce6dce83951db2e752f5458fa7af09906488fd6ef871c589c232794a4bf3d8a6962613cc35e484

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (1.22.0)
+    gitlab_quality-test_tooling (1.24.0)
       activesupport (>= 6.1, < 7.2)
       amatch (~> 0.4.1)
       gitlab (~> 4.19)
@@ -130,6 +130,7 @@ GEM
       multi_xml (>= 0.5.2)
     i18n (1.14.4)
       concurrent-ruby (~> 1.0)
+    influxdb-client (3.1.0)
     jaro_winkler (1.5.6)
     json (2.7.1)
     kramdown (2.4.0)
@@ -312,11 +313,13 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activesupport (>= 6.1, < 7.2)
   climate_control (~> 1.2)
   gitlab-dangerfiles (~> 3.8)
   gitlab-styles (~> 10.0)
   gitlab_quality-test_tooling!
   guard-rspec (~> 4.7)
+  influxdb-client (~> 3.1)
   lefthook (~> 1.3)
   pry-byebug (= 3.10.1)
   rake (~> 13.0)
@@ -328,4 +331,4 @@ DEPENDENCIES
   webmock (= 3.7.0)
 
 BUNDLED WITH
-   2.5.4
+   2.5.6

data/lib/gitlab_quality/test_tooling/concerns/influxdb_tools.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Concerns
+      module InfluxdbTools
+        # InfluxDb client
+        #
+        # @return [InfluxDB2::Client]
+        def influx_client(url:, token:, bucket:)
+          @influx_client ||= InfluxDB2::Client.new(
+            url || raise('Missing influxdb_url'),
+            token || raise('Missing influxdb_token'),
+            bucket: bucket || raise('Missing influxdb_bucket'),
+            org: "gitlab-qa",
+            precision: InfluxDB2::WritePrecision::NANOSECOND
+          )
+        end
+
+        # Write client
+        #
+        # @return [WriteApi]
+        def write_api(url:, token:, bucket:)
+          @write_api ||= influx_client(url: url, token: token, bucket: bucket).create_write_api
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/concerns/test_metrics.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Concerns
+      module TestMetrics
+        # Single common timestamp for all exported example metrics to keep data points consistently grouped
+        #
+        # @return [Time]
+        def time
+          return @time if defined?(@time)
+
+          created_at = Time.strptime(env('CI_PIPELINE_CREATED_AT'), '%Y-%m-%dT%H:%M:%S%z') if env('CI_PIPELINE_CREATED_AT')
+          @time = (created_at || Time.now).utc.strftime('%Y-%m-%dT%H:%M:%S%z')
+        end
+
+        # rubocop:disable Metrics/AbcSize
+        # Metrics tags
+        #
+        # @param [RSpec::Core::Example] example
+        # @param [Array<String>] custom_keys
+        # @param [String]
+        # @return [Hash]
+        def tags(example, custom_keys, run_type)
+          {
+            name: example.full_description,
+            file_path: example.metadata[:file_path].sub(/\A./, ''),
+            status: status(example),
+            quarantined: quarantined(example),
+            job_name: job_name,
+            merge_request: merge_request,
+            run_type: run_type,
+            stage: example.metadata[:product_stage] || example.metadata[:product_category],
+            product_group: example.metadata[:product_group],
+            exception_class: example.execution_result.exception&.class&.to_s,
+            **custom_metrics(example.metadata, custom_keys)
+          }.compact
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        # Metrics fields
+        #
+        # @param [RSpec::Core::Example] example
+        # @param [Array<String>] custom_keys
+        # @return [Hash]
+        def fields(example, custom_keys)
+          {
+            id: example.id,
+            run_time: (example.execution_result.run_time * 1000).round,
+            job_url: Runtime::Env.ci_job_url,
+            pipeline_url: env('CI_PIPELINE_URL'),
+            pipeline_id: env('CI_PIPELINE_ID'),
+            job_id: env('CI_JOB_ID'),
+            merge_request_iid: merge_request_iid,
+            failure_exception: example.execution_result.exception.to_s.delete("\n"),
+            **custom_metrics(example.metadata, custom_keys)
+          }.compact
+        end
+
+        # Return a more detailed status
+        #
+        # - if test is failed or pending, return rspec status
+        # - if test passed but had more than 1 attempt, consider test flaky
+        #
+        # @param [RSpec::Core::Example] example
+        # @return [Symbol]
+        def status(example)
+          rspec_status = example.execution_result.status
+          return rspec_status if [:pending, :failed].include?(rspec_status)
+
+          retry_attempts(example.metadata).positive? ? :flaky : :passed
+        end
+
+        # Retry attempts
+        #
+        # @param [Hash] example
+        # @return [Integer]
+        def retry_attempts(metadata)
+          metadata[:retry_attempts] || 0
+        end
+
+        # Checks if spec is quarantined
+        #
+        # @param [RSpec::Core::Example] example
+        # @return [String]
+        def quarantined(example)
+          return "false" unless example.metadata.key?(:quarantine)
+
+          # if quarantine key is present and status is pending, consider it quarantined
+          (example.execution_result.status == :pending).to_s
+        end
+
+        # Base ci job name
+        #
+        # @return [String]
+        def job_name
+          @job_name ||= Runtime::Env.ci_job_name&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+        end
+
+        # Check if it is a merge request execution
+        #
+        # @return [String]
+        def merge_request
+          (!!merge_request_iid).to_s
+        end
+
+        # Merge request iid
+        #
+        # @return [String]
+        def merge_request_iid
+          env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID')
+        end
+
+        # Custom test metrics
+        #
+        # @param [Hash] metadata
+        # @param [Array] array of custom metrics keys
+        # @return [Hash]
+        def custom_metrics(metadata, custom_keys)
+          return {} if custom_keys.nil?
+
+          custom_metrics = {}
+          custom_keys.each do |k|
+            value = metadata[k.to_sym]
+            v = value.is_a?(Numeric) || value.nil? ? value : value.to_s
+
+            custom_metrics[k.to_sym] = v
+          end
+
+          custom_metrics
+        end
+
+        # Return non empty environment variable value
+        #
+        # @param [String] name
+        # @return [String, nil]
+        def env(name)
+          return unless ENV[name] && !ENV[name].empty?
+
+          ENV.fetch(name)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/concerns/issue_reports.rb

@@ -12,11 +12,57 @@ module GitlabQuality
           REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>\S+)\)/
           LATEST_REPORTS_TO_SHOW = 10
 
+          class ReportsList
+            def initialize(preserved_content:, section_header:, reports:, extra_content:)
+              @preserved_content = preserved_content
+              @section_header = section_header
+              @reports = reports
+              @extra_content = extra_content
+            end
+
+            def self.report_list_item(test, item_extra_content: nil)
+              "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')}) #{item_extra_content}".strip
+            end
+
+            def reports_count
+              reports.size
+            end
+
+            def to_s
+              [
+                preserved_content,
+                "#{section_header} (#{reports_count})",
+                reports_list(reports),
+                extra_content
+              ].reject(&:blank?).compact.join("\n\n")
+            end
+
+            private
+
+            attr_reader :preserved_content, :section_header, :reports, :extra_content
+
+            def reports_list(reports)
+              sorted_reports = reports.sort.reverse
+
+              if sorted_reports.size > LATEST_REPORTS_TO_SHOW
+                [
+                  "Last 10 reports:",
+                  sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
+                  "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
+                  sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
+                  "</details>"
+                ].join("\n\n")
+              else
+                sorted_reports.join("\n")
+              end
+            end
+          end
+
           def initial_reports_section(test)
             <<~REPORTS
             ### Reports (1)
 
-            #{report_list_item(test)}
+            #{ReportsList.report_list_item(test)}
             REPORTS
           end
 
@@ -27,14 +73,14 @@ module GitlabQuality
             item_extra_content: nil,
             reports_extra_content: nil)
             preserved_content = current_reports_content.split(reports_section_header).first&.strip
-            reports = report_lines(current_reports_content) + [report_list_item(test, item_extra_content: item_extra_content)]
-
-            [
-              preserved_content,
-              "#{reports_section_header} (#{reports.size})",
-              reports_list(reports),
-              reports_extra_content
-            ].reject(&:blank?).compact.join("\n\n")
+            reports = report_lines(current_reports_content) + [ReportsList.report_list_item(test, item_extra_content: item_extra_content)]
+
+            ReportsList.new(
+              preserved_content: preserved_content,
+              section_header: reports_section_header,
+              reports: reports,
+              extra_content: reports_extra_content
+            )
           end
 
           def failed_issue_job_url(issue)
@@ -55,40 +101,12 @@ module GitlabQuality
             content.lines.grep(REPORT_ITEM_REGEX).map(&:strip)
           end
 
-          def reports_list(reports)
-            sorted_reports = reports.sort.reverse
-
-            if sorted_reports.size > LATEST_REPORTS_TO_SHOW
-              [
-                "Last 10 reports:",
-                sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
-                "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
-                sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
-                "</details>"
-              ].join("\n\n")
-            else
-              sorted_reports.join("\n")
-            end
-          end
-
-          def report_list_item(test, item_extra_content: nil)
-            "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')}) #{item_extra_content}".strip
-          end
-
           def job_urls_from_description(issue_description, regex)
             issue_description.lines.filter_map do |line|
               match = line.match(regex)
               match[:job_url] if match
             end
           end
-
-          def test_captures_to_report_items(test_captures)
-            test_captures.map do |ci_job_url, _, _|
-              report_list_item(GitlabQuality::TestTooling::TestResult::JsonTestResult.new(
-                'ci_job_url' => ci_job_url
-              ))
-            end
-          end
         end
       end
     end

data/lib/gitlab_quality/test_tooling/report/failed_test_issue.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'amatch'
-
 module GitlabQuality
   module TestTooling
     module Report
@@ -15,7 +13,6 @@ module GitlabQuality
       class FailedTestIssue < ReportAsIssue
         include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports
-        include Amatch
 
         IDENTITY_LABELS       = ['test', 'automation:bot-authored'].freeze
         NEW_ISSUE_LABELS      = Set.new(['type::maintenance', 'failure::new', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
@@ -25,10 +22,6 @@ module GitlabQuality
         REPORTS_DISCUSSION_HEADER = '### Failure reports'
         REPORT_SECTION_HEADER = '#### Failure reports'
 
-        IGNORED_FAILURES = [
-          'Net::ReadTimeout',
-          '403 Forbidden - Your account has been blocked'
-        ].freeze
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
         ISSUE_STACKTRACE_REGEX = /##### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*/m
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
@@ -92,20 +85,21 @@ module GitlabQuality
           end
         end
 
-        def add_report_to_issue(issue:, test:, related_issues:)
+        def add_report_to_issue(issue:, test:, related_issues:) # rubocop:disable Metrics/AbcSize:
           reports_discussion = find_or_create_reports_discussion(issue: issue)
-          failure_discussion_note = find_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion)
+          current_reports_note = find_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion)
+          new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test)
 
           note_body = [
-            report_body(reports_note: failure_discussion_note, test: test),
+            new_reports_list.to_s,
             identity_labels_quick_action,
             relate_issues_quick_actions(related_issues)
           ].join("\n")
 
-          if failure_discussion_note
+          if current_reports_note
             gitlab.edit_issue_note(
               issue_iid: issue.iid,
-              note_id: failure_discussion_note.id,
+              note_id: current_reports_note.id,
               note: note_body
             )
           else
@@ -158,6 +152,8 @@ module GitlabQuality
 
           clean_test_stacktrace = cleaned_stack_trace_from_test(test: test)
 
+          # We're skipping the first note of the discussion as this is the "non-collapsible note", aka
+          # the "header note", which doesn't contain any stack trace.
           reports_discussion.notes[1..].each_with_object({}) do |note, memo|
             clean_note_stacktrace = cleaned_stack_trace_from_note(issue: issue, note: note)
             diff_ratio = diff_ratio_between_test_and_note_stacktraces(
@@ -171,7 +167,7 @@ module GitlabQuality
         end
 
         def cleaned_stack_trace_from_test(test:)
-          sanitize_stacktrace(stacktrace: full_stacktrace(test: test), regex: FAILURE_STACKTRACE_REGEX) || full_stacktrace(test: test)
+          sanitize_stacktrace(stacktrace: test.full_stacktrace, regex: FAILURE_STACKTRACE_REGEX) || test.full_stacktrace
         end
 
         def cleaned_stack_trace_from_note(issue:, note:)
@@ -191,53 +187,33 @@ module GitlabQuality
           end
         end
 
-        def full_stacktrace(test:)
-          test.failures.each do |failure|
-            message = failure['message'] || ""
-            message_lines = failure['message_lines'] || []
-
-            next if IGNORED_FAILURES.any? { |e| message.include?(e) }
-
-            return message_lines.empty? ? message : message_lines.join("\n")
-          end
-        end
-
         def diff_ratio_between_test_and_note_stacktraces(issue:, note:, test_stacktrace:, note_stacktrace:)
           return if note_stacktrace.nil?
 
-          diff_ratio = compare_stack_traces(test_stacktrace, note_stacktrace)
+          stack_trace_comparator = StackTraceComparator.new(test_stacktrace, note_stacktrace)
 
-          if diff_ratio <= max_diff_ratio
-            puts "  => [DEBUG] Note #{issue.web_url}#note_#{note.id} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
+          if stack_trace_comparator.lower_or_equal_to_diff_ratio?(max_diff_ratio)
+            puts "  => [DEBUG] Note #{issue.web_url}#note_#{note.id} has an acceptable diff ratio of #{stack_trace_comparator.diff_percent}%."
             # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
             # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
             # See:
             # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
             # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
-            diff_ratio
+            stack_trace_comparator.diff_ratio
           else
-            puts "  => [DEBUG] Found note #{issue.web_url}#note_#{note.id} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
+            puts "  => [DEBUG] Found note #{issue.web_url}#note_#{note.id} but stacktraces are too different (#{stack_trace_comparator.diff_percent}%).\n"
             puts "  => [DEBUG] Issue stacktrace:\n----------------\n#{note_stacktrace}\n----------------\n"
             puts "  => [DEBUG] Failure stacktrace:\n----------------\n#{test_stacktrace}\n----------------\n"
           end
         end
 
-        def compare_stack_traces(stack_trace_first, stack_trace_second)
-          calculate_diff_ratio(stack_trace_first, stack_trace_second)
-        end
-
-        def calculate_diff_ratio(stack_trace_first, stack_trace_second)
-          distance = Levenshtein.new(stack_trace_first).match(stack_trace_second)
-          distance.zero? ? 0.0 : (distance.to_f / stack_trace_first.size).round(3)
-        end
-
-        def report_body(reports_note:, test:)
+        def add_report_for_test(current_reports_content:, test:)
           increment_reports(
-            current_reports_content: reports_note&.body.to_s,
+            current_reports_content: current_reports_content,
             test: test,
             reports_section_header: REPORT_SECTION_HEADER,
             item_extra_content: found_label,
-            reports_extra_content: "##### Stack trace\n\n```\n#{full_stacktrace(test: test)}\n```"
+            reports_extra_content: "##### Stack trace\n\n```\n#{test.full_stacktrace}\n```"
           )
         end
 

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -22,7 +22,7 @@ module GitlabQuality
         FOUND_IN_MASTER_LABEL = '~"found:master"'
         REPORT_SECTION_HEADER = '### Flakiness reports'
         REPORTS_DOCUMENTATION = <<~DOC
-          Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html)
+          Flaky tests were detected. Please refer to the [Flaky tests reproducibility instructions](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#how-to-reproduce-a-flaky-test-locally)
           to learn more about how to reproduce them.
         DOC
 
@@ -83,17 +83,20 @@ module GitlabQuality
         end
 
         def add_report_to_issue(issue:, test:, related_issues:)
-          reports_note = existing_reports_note(issue: issue)
+          current_reports_note = existing_reports_note(issue: issue)
+          new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test)
+
           note_body = [
-            report_body(reports_note: reports_note, test: test),
+            new_reports_list.to_s,
+            flakiness_status_labels_quick_action(new_reports_list.reports_count),
             identity_labels_quick_action,
             relate_issues_quick_actions(related_issues)
           ].join("\n")
 
-          if reports_note
+          if current_reports_note
             gitlab.edit_issue_note(
               issue_iid: issue.iid,
-              note_id: reports_note.id,
+              note_id: current_reports_note.id,
               note: note_body
             )
           else
@@ -107,9 +110,9 @@ module GitlabQuality
           end
         end
 
-        def report_body(reports_note:, test:)
+        def add_report_for_test(current_reports_content:, test:)
           increment_reports(
-            current_reports_content: reports_note&.body.to_s,
+            current_reports_content: current_reports_content,
             test: test,
             reports_section_header: REPORT_SECTION_HEADER,
             item_extra_content: found_label,
@@ -125,6 +128,28 @@ module GitlabQuality
           end
         end
 
+        # The report count is based on the percentiles of flakiness issues.
+        #
+        # See https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/scripts/unhealthy_test_issues_statistics.rb
+        # to gather these statistics.
+        #
+        # P75       => flakiness::4
+        # P90       => flakiness::3
+        # P95       => flakiness::2
+        # Above P95 => flakiness::1
+        def flakiness_status_labels_quick_action(reports_count)
+          case reports_count
+          when 42..Float::INFINITY
+            '/label ~"flakiness::1"'
+          when 22..41
+            '/label ~"flakiness::2"'
+          when 4..21
+            '/label ~"flakiness::3"'
+          else
+            '/label ~"flakiness::4"'
+          end
+        end
+
         def identity_labels_quick_action
           labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
           %(/label #{labels_list})

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -2,7 +2,6 @@
 
 require 'nokogiri'
 require 'rubygems/text'
-require 'amatch'
 
 module GitlabQuality
   module TestTooling
@@ -17,7 +16,6 @@ module GitlabQuality
         include TestTooling::Concerns::FindSetDri
         include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports
-        include Amatch
 
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
         SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
@@ -26,10 +24,6 @@ module GitlabQuality
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*\n###/m
 
         NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
-        IGNORED_FAILURES = [
-          'Net::ReadTimeout',
-          '403 Forbidden - Your account has been blocked'
-        ].freeze
         SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
 
         MultipleIssuesFound = Class.new(StandardError)
@@ -168,7 +162,9 @@ module GitlabQuality
 
             next if pipeline != pipeline_env_from_job_url(job_url_from_issue)
 
-            compare_stack_traces(cleaned_stack_trace_from_test(test), cleaned_stack_trace_from_issue(issue)) < max_diff_ratio
+            stack_trace_comparator = StackTraceComparator.new(cleaned_stack_trace_from_test(test), cleaned_stack_trace_from_issue(issue))
+
+            stack_trace_comparator.lower_than_diff_ratio?(max_diff_ratio)
           end
         end
 
@@ -196,17 +192,6 @@ module GitlabQuality
           )
         end
 
-        def full_stacktrace(test)
-          test.failures.each do |failure|
-            message = failure['message'] || ""
-            message_lines = failure['message_lines'] || []
-
-            next if IGNORED_FAILURES.any? { |e| message.include?(e) }
-
-            return message_lines.empty? ? message : message_lines.join("\n")
-          end
-        end
-
         def cleaned_stack_trace_from_issue(issue)
           relevant_issue_stacktrace = find_issue_stacktrace(issue)
           return unless relevant_issue_stacktrace
@@ -215,20 +200,11 @@ module GitlabQuality
         end
 
         def cleaned_stack_trace_from_test(test)
-          test_failure_stacktrace = sanitize_stacktrace(full_stacktrace(test),
-            FAILURE_STACKTRACE_REGEX) || full_stacktrace(test)
+          test_failure_stacktrace = sanitize_stacktrace(test.full_stacktrace,
+            FAILURE_STACKTRACE_REGEX) || test.full_stacktrace
           remove_unique_resource_names(test_failure_stacktrace)
         end
 
-        def compare_stack_traces(stack_trace_first, stack_trace_second)
-          calculate_diff_ratio(stack_trace_first, stack_trace_second)
-        end
-
-        def calculate_diff_ratio(stack_trace_first, stack_trace_second)
-          distance = Levenshtein.new(stack_trace_first).match(stack_trace_second)
-          distance.zero? ? 0.0 : (distance.to_f / stack_trace_first.size).round(3)
-        end
-
         def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
           clean_first_test_failure_stacktrace = cleaned_stack_trace_from_test(test)
           # Search with the `search` param returns 500 errors, so we filter by `base_issue_labels` and then filter further in Ruby
@@ -236,17 +212,18 @@ module GitlabQuality
             clean_relevant_issue_stacktrace = cleaned_stack_trace_from_issue(issue)
             next if clean_relevant_issue_stacktrace.nil?
 
-            diff_ratio = compare_stack_traces(clean_first_test_failure_stacktrace, clean_relevant_issue_stacktrace)
-            if diff_ratio <= max_diff_ratio
-              puts "  => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
+            stack_trace_comparator = StackTraceComparator.new(clean_first_test_failure_stacktrace, clean_relevant_issue_stacktrace)
+
+            if stack_trace_comparator.lower_or_equal_to_diff_ratio?(max_diff_ratio)
+              puts "  => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{stack_trace_comparator.diff_percent}%."
               # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
               # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
               # See:
               # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
               # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
-              memo[issue.to_h] = diff_ratio
+              memo[issue.to_h] = stack_trace_comparator.diff_ratio
             else
-              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
+              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{stack_trace_comparator.diff_percent}%).\n"
               puts "  => [DEBUG] Issue stacktrace:\n----------------\n#{clean_relevant_issue_stacktrace}\n----------------\n"
               puts "  => [DEBUG] Failure stacktrace:\n----------------\n#{clean_first_test_failure_stacktrace}\n----------------\n"
             end
@@ -295,7 +272,7 @@ module GitlabQuality
         def new_issue_description(test)
           super + [
             "\n### Stack trace",
-            "```\n#{full_stacktrace(test)}\n```",
+            "```\n#{test.full_stacktrace}\n```",
             screenshot_section(test),
             system_log_errors_section(test),
             initial_reports_section(test)
@@ -345,7 +322,7 @@ module GitlabQuality
           state_event = issue.state == 'closed' ? 'reopen' : nil
 
           issue_attrs = {
-            description: increment_reports(current_reports_content: issue.description, test: test),
+            description: increment_reports(current_reports_content: issue.description, test: test).to_s,
             labels: up_to_date_labels(test: test, issue: issue)
           }
           issue_attrs[:state_event] = state_event if state_event
@@ -357,7 +334,7 @@ module GitlabQuality
         def screenshot_section(test)
           return unless test.screenshot?
 
-          failure = full_stacktrace(test)
+          failure = test.full_stacktrace
           return if SCREENSHOT_IGNORED_ERRORS.any? { |e| failure.include?(e) }
 
           relative_url = gitlab.upload_file(file_fullpath: test.screenshot_image)
@@ -374,7 +351,7 @@ module GitlabQuality
           return false unless test.failures?
 
           puts "  => Systemic failures detected: #{systemic_failure_messages}" if systemic_failure_messages.any?
-          failure_to_ignore = IGNORED_FAILURES + systemic_failure_messages
+          failure_to_ignore = TestResult::BaseTestResult::IGNORED_FAILURES + systemic_failure_messages
 
           reason = ignored_failure_reason(test.failures, failure_to_ignore)
 

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -64,17 +64,20 @@ module GitlabQuality
         end
 
         def add_report_to_issue(issue:, test:, related_issues:)
-          reports_note = existing_reports_note(issue: issue)
+          current_reports_note = existing_reports_note(issue: issue)
+          new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test)
+
           note_body = [
-            report_body(reports_note: reports_note, test: test),
+            new_reports_list.to_s,
+            slowness_status_labels_quick_action(new_reports_list.reports_count),
             identity_labels_quick_action,
             relate_issues_quick_actions(related_issues)
           ].join("\n")
 
-          if reports_note
+          if current_reports_note
             gitlab.edit_issue_note(
               issue_iid: issue.iid,
-              note_id: reports_note.id,
+              note_id: current_reports_note.id,
               note: note_body
             )
           else
@@ -88,9 +91,9 @@ module GitlabQuality
           end
         end
 
-        def report_body(reports_note:, test:)
+        def add_report_for_test(current_reports_content:, test:)
           increment_reports(
-            current_reports_content: reports_note&.body.to_s,
+            current_reports_content: current_reports_content,
             test: test,
             reports_section_header: REPORT_SECTION_HEADER,
             item_extra_content: "(#{test.run_time} seconds) #{found_label}",
@@ -106,6 +109,28 @@ module GitlabQuality
           end
         end
 
+        # The report count is based on the percentiles of slowness issues.
+        #
+        # See https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/scripts/unhealthy_test_issues_statistics.rb
+        # to gather these statistics.
+        #
+        # P75       => slowness::4
+        # P90       => slowness::3
+        # P95       => slowness::2
+        # Above P95 => slowness::1
+        def slowness_status_labels_quick_action(reports_count)
+          case reports_count
+          when 40..Float::INFINITY
+            '/label ~"slowness::1"'
+          when 28..39
+            '/label ~"slowness::2"'
+          when 12..27
+            '/label ~"slowness::3"'
+          else
+            '/label ~"slowness::4"'
+          end
+        end
+
         def identity_labels_quick_action
           labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
           %(/label #{labels_list})

data/lib/gitlab_quality/test_tooling/stack_trace_comparator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'amatch'
+
+module GitlabQuality
+  module TestTooling
+    class StackTraceComparator
+      include Amatch
+
+      def initialize(first_trace, second_trace)
+        @first_trace = first_trace
+        @second_trace = second_trace
+      end
+
+      def diff_ratio
+        @diff_ratio ||= (1 - first_trace.levenshtein_similar(second_trace))
+      end
+
+      def diff_percent
+        (diff_ratio * 100).round(2)
+      end
+
+      def lower_than_diff_ratio?(max_diff_ratio)
+        diff_ratio < max_diff_ratio
+      end
+
+      def lower_or_equal_to_diff_ratio?(max_diff_ratio)
+        diff_ratio <= max_diff_ratio
+      end
+
+      private
+
+      attr_reader :first_trace, :second_trace
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_metric/log_test_metrics.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module TestMetric
+      class LogTestMetrics
+        include Concerns::TestMetrics
+        include Concerns::InfluxdbTools
+
+        CUSTOM_METRICS_KEY = :custom_test_metrics
+
+        def initialize(examples:, influxdb_url: nil, influxdb_token: nil, influxdb_bucket: nil, run_type: nil)
+          @examples = examples
+          @influxdb_url = influxdb_url
+          @influxdb_token = influxdb_token
+          @influxdb_bucket = influxdb_bucket
+          @run_type = run_type
+        end
+
+        # Push test execution metrics to influxdb
+        #
+        # @param [Array<String>] custom_keys_tags
+        # @param [Array<String>] custom_keys_fields
+        # @return [nil]
+        def push_test_metrics(custom_keys_tags: nil, custom_keys_fields: nil)
+          @test_metrics ||= examples.filter_map { |example| parse_test_results(example, custom_keys_tags, custom_keys_fields) }
+
+          write_api(url: influxdb_url, token: influxdb_token, bucket: influxdb_bucket).write(data: test_metrics)
+          Runtime::Logger.debug("Pushed #{test_metrics.length} test execution entries to influxdb")
+        rescue StandardError => e
+          Runtime::Logger.error("Failed to push test execution metrics to influxdb, error: #{e}")
+        end
+
+        # Save metrics in json file
+        #
+        # @param [String] file_name
+        # @param [Array<String>] custom_keys_tags
+        # @param [Array<String>] custom_keys_fields
+        # @return [nil]
+        def save_test_metrics(file_name, custom_keys_tags: nil, custom_keys_fields: nil)
+          @test_metrics ||= examples.filter_map { |example| parse_test_results(example, custom_keys_tags, custom_keys_fields) }
+          file = "tmp/#{file_name}"
+
+          File.write(file, test_metrics.to_json) && Runtime::Logger.debug("Saved test metrics to #{file}")
+        rescue StandardError => e
+          Runtime::Logger.error("Failed to save test execution metrics, error: #{e}")
+        end
+
+        private
+
+        attr_reader :examples, :test_metrics, :influxdb_url, :influxdb_token, :influxdb_bucket, :run_type
+
+        # Transform example to influxdb compatible metrics data
+        # https://github.com/influxdata/influxdb-client-ruby#data-format
+        #
+        # @param [RSpec::Core::Example] example
+        # @param [Array<String>] custom_keys_tags
+        # @param [Array<String>] custom_keys_fields
+        # @return [Hash]
+        def parse_test_results(example, custom_keys_tags, custom_keys_fields)
+          {
+            name: 'test-stats',
+            time: time,
+            tags: tags(example, custom_keys_tags, run_type),
+            fields: fields(example, custom_keys_fields)
+          }
+        rescue StandardError => e
+          Runtime::Logger.error("Failed to transform example '#{example.id}', error: #{e}")
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

@@ -4,6 +4,11 @@ module GitlabQuality
   module TestTooling
     module TestResult
       class BaseTestResult
+        IGNORED_FAILURES = [
+          'Net::ReadTimeout',
+          '403 Forbidden - Your account has been blocked'
+        ].freeze
+
         attr_reader :report
 
         def initialize(report)
@@ -41,6 +46,17 @@ module GitlabQuality
         def failures?
           failures.any?
         end
+
+        def full_stacktrace
+          failures.each do |failure|
+            message = failure['message'] || ""
+            message_lines = failure['message_lines'] || []
+
+            next if IGNORED_FAILURES.any? { |e| message.include?(e) }
+
+            return message_lines.empty? ? message : message_lines.join("\n")
+          end
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "1.22.0"
+    VERSION = "1.24.0"
   end
 end

metadata

@@ -1,15 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 1.22.0
+  version: 1.24.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-10 00:00:00.000000000 Z
+date: 2024-04-29 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.2'
 - !ruby/object:Gem::Dependency
   name: climate_control
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +86,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.7'
+- !ruby/object:Gem::Dependency
+  name: influxdb-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: lefthook
   requirement: !ruby/object:Gem::Requirement
@@ -404,6 +438,8 @@ files:
 - lefthook.yml
 - lib/gitlab_quality/test_tooling.rb
 - lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb
+- lib/gitlab_quality/test_tooling/concerns/influxdb_tools.rb
+- lib/gitlab_quality/test_tooling/concerns/test_metrics.rb
 - lib/gitlab_quality/test_tooling/failed_jobs_table.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/branches_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/branches_dry_client.rb
@@ -441,6 +477,7 @@ files:
 - lib/gitlab_quality/test_tooling/runtime/logger.rb
 - lib/gitlab_quality/test_tooling/slack/post_to_slack.rb
 - lib/gitlab_quality/test_tooling/slack/post_to_slack_dry.rb
+- lib/gitlab_quality/test_tooling/stack_trace_comparator.rb
 - lib/gitlab_quality/test_tooling/summary_table.rb
 - lib/gitlab_quality/test_tooling/support/http_request.rb
 - lib/gitlab_quality/test_tooling/system_logs/finders/json_log_finder.rb
@@ -460,6 +497,7 @@ files:
 - lib/gitlab_quality/test_tooling/test_meta/processor/meta_processor.rb
 - lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb
 - lib/gitlab_quality/test_tooling/test_metric/json_test_metric.rb
+- lib/gitlab_quality/test_tooling/test_metric/log_test_metrics.rb
 - lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb
 - lib/gitlab_quality/test_tooling/test_result/base_test_result.rb
 - lib/gitlab_quality/test_tooling/test_result/j_unit_test_result.rb
@@ -492,7 +530,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.3.27
 signing_key:
 specification_version: 4
 summary: A collection of test-related tools.