gitlab_quality-test_tooling 1.23.0 → 1.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c63a7a25d0e3a05f515954353898bb8d0dcb9e7a90634087e738f5cb16174c57
-  data.tar.gz: 386833833dc0fd5cdc6334d3d613a38544fbc885c5ccb396070614f0e889555f
+  metadata.gz: '009e41374e42e637a571410da7cd95aa23ab4d5a6f1599f080a0ccea8432b820'
+  data.tar.gz: b81d4043e16bc5da888a102d3765250691acb705bb3edac3762c6baad0fe490f
 SHA512:
-  metadata.gz: bb13a1c06a626b1152115262e0606480e78eb9aaafd6ed5e673cd72d6b1a268ab7659c020dbab743ebb872e71c1332905319d0ed6970cbde8bf0168c208fa2c3
-  data.tar.gz: 40652c63561438f37f5610e5b22f93c6a0404c53d4dc65b17713bbf5469bf769dd13691361f02f5aa8229aab706f7e349db6abf663cf45745aaa040456461db7
+  metadata.gz: f1d34b6b4862000282c042e25b5f0cd02e23e72dee4307d0eb9d2309a0e5d352dfeeeadab2f5da7d01b91fe798a192bf8f2ed075608e9f73854842d107bef55a
+  data.tar.gz: eb3746a140cb622b9ede27514cb26ea7681b1bafffc5320dd9ce6dce83951db2e752f5458fa7af09906488fd6ef871c589c232794a4bf3d8a6962613cc35e484

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (1.23.0)
+    gitlab_quality-test_tooling (1.24.0)
       activesupport (>= 6.1, < 7.2)
       amatch (~> 0.4.1)
       gitlab (~> 4.19)
@@ -130,6 +130,7 @@ GEM
       multi_xml (>= 0.5.2)
     i18n (1.14.4)
       concurrent-ruby (~> 1.0)
+    influxdb-client (3.1.0)
     jaro_winkler (1.5.6)
     json (2.7.1)
     kramdown (2.4.0)
@@ -312,11 +313,13 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activesupport (>= 6.1, < 7.2)
   climate_control (~> 1.2)
   gitlab-dangerfiles (~> 3.8)
   gitlab-styles (~> 10.0)
   gitlab_quality-test_tooling!
   guard-rspec (~> 4.7)
+  influxdb-client (~> 3.1)
   lefthook (~> 1.3)
   pry-byebug (= 3.10.1)
   rake (~> 13.0)
@@ -328,4 +331,4 @@ DEPENDENCIES
   webmock (= 3.7.0)
 
 BUNDLED WITH
-   2.5.4
+   2.5.6

data/lib/gitlab_quality/test_tooling/concerns/influxdb_tools.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Concerns
+      module InfluxdbTools
+        # InfluxDb client
+        #
+        # @return [InfluxDB2::Client]
+        def influx_client(url:, token:, bucket:)
+          @influx_client ||= InfluxDB2::Client.new(
+            url || raise('Missing influxdb_url'),
+            token || raise('Missing influxdb_token'),
+            bucket: bucket || raise('Missing influxdb_bucket'),
+            org: "gitlab-qa",
+            precision: InfluxDB2::WritePrecision::NANOSECOND
+          )
+        end
+
+        # Write client
+        #
+        # @return [WriteApi]
+        def write_api(url:, token:, bucket:)
+          @write_api ||= influx_client(url: url, token: token, bucket: bucket).create_write_api
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/concerns/test_metrics.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module Concerns
+      module TestMetrics
+        # Single common timestamp for all exported example metrics to keep data points consistently grouped
+        #
+        # @return [Time]
+        def time
+          return @time if defined?(@time)
+
+          created_at = Time.strptime(env('CI_PIPELINE_CREATED_AT'), '%Y-%m-%dT%H:%M:%S%z') if env('CI_PIPELINE_CREATED_AT')
+          @time = (created_at || Time.now).utc.strftime('%Y-%m-%dT%H:%M:%S%z')
+        end
+
+        # rubocop:disable Metrics/AbcSize
+        # Metrics tags
+        #
+        # @param [RSpec::Core::Example] example
+        # @param [Array<String>] custom_keys
+        # @param [String]
+        # @return [Hash]
+        def tags(example, custom_keys, run_type)
+          {
+            name: example.full_description,
+            file_path: example.metadata[:file_path].sub(/\A./, ''),
+            status: status(example),
+            quarantined: quarantined(example),
+            job_name: job_name,
+            merge_request: merge_request,
+            run_type: run_type,
+            stage: example.metadata[:product_stage] || example.metadata[:product_category],
+            product_group: example.metadata[:product_group],
+            exception_class: example.execution_result.exception&.class&.to_s,
+            **custom_metrics(example.metadata, custom_keys)
+          }.compact
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        # Metrics fields
+        #
+        # @param [RSpec::Core::Example] example
+        # @param [Array<String>] custom_keys
+        # @return [Hash]
+        def fields(example, custom_keys)
+          {
+            id: example.id,
+            run_time: (example.execution_result.run_time * 1000).round,
+            job_url: Runtime::Env.ci_job_url,
+            pipeline_url: env('CI_PIPELINE_URL'),
+            pipeline_id: env('CI_PIPELINE_ID'),
+            job_id: env('CI_JOB_ID'),
+            merge_request_iid: merge_request_iid,
+            failure_exception: example.execution_result.exception.to_s.delete("\n"),
+            **custom_metrics(example.metadata, custom_keys)
+          }.compact
+        end
+
+        # Return a more detailed status
+        #
+        # - if test is failed or pending, return rspec status
+        # - if test passed but had more than 1 attempt, consider test flaky
+        #
+        # @param [RSpec::Core::Example] example
+        # @return [Symbol]
+        def status(example)
+          rspec_status = example.execution_result.status
+          return rspec_status if [:pending, :failed].include?(rspec_status)
+
+          retry_attempts(example.metadata).positive? ? :flaky : :passed
+        end
+
+        # Retry attempts
+        #
+        # @param [Hash] example
+        # @return [Integer]
+        def retry_attempts(metadata)
+          metadata[:retry_attempts] || 0
+        end
+
+        # Checks if spec is quarantined
+        #
+        # @param [RSpec::Core::Example] example
+        # @return [String]
+        def quarantined(example)
+          return "false" unless example.metadata.key?(:quarantine)
+
+          # if quarantine key is present and status is pending, consider it quarantined
+          (example.execution_result.status == :pending).to_s
+        end
+
+        # Base ci job name
+        #
+        # @return [String]
+        def job_name
+          @job_name ||= Runtime::Env.ci_job_name&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+        end
+
+        # Check if it is a merge request execution
+        #
+        # @return [String]
+        def merge_request
+          (!!merge_request_iid).to_s
+        end
+
+        # Merge request iid
+        #
+        # @return [String]
+        def merge_request_iid
+          env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID')
+        end
+
+        # Custom test metrics
+        #
+        # @param [Hash] metadata
+        # @param [Array] array of custom metrics keys
+        # @return [Hash]
+        def custom_metrics(metadata, custom_keys)
+          return {} if custom_keys.nil?
+
+          custom_metrics = {}
+          custom_keys.each do |k|
+            value = metadata[k.to_sym]
+            v = value.is_a?(Numeric) || value.nil? ? value : value.to_s
+
+            custom_metrics[k.to_sym] = v
+          end
+
+          custom_metrics
+        end
+
+        # Return non empty environment variable value
+        #
+        # @param [String] name
+        # @return [String, nil]
+        def env(name)
+          return unless ENV[name] && !ENV[name].empty?
+
+          ENV.fetch(name)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb

@@ -22,7 +22,7 @@ module GitlabQuality
         FOUND_IN_MASTER_LABEL = '~"found:master"'
         REPORT_SECTION_HEADER = '### Flakiness reports'
         REPORTS_DOCUMENTATION = <<~DOC
-          Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html)
+          Flaky tests were detected. Please refer to the [Flaky tests reproducibility instructions](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#how-to-reproduce-a-flaky-test-locally)
           to learn more about how to reproduce them.
         DOC
 
@@ -128,13 +128,22 @@ module GitlabQuality
           end
         end
 
+        # The report count is based on the percentiles of flakiness issues.
+        #
+        # See https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/scripts/unhealthy_test_issues_statistics.rb
+        # to gather these statistics.
+        #
+        # P75       => flakiness::4
+        # P90       => flakiness::3
+        # P95       => flakiness::2
+        # Above P95 => flakiness::1
         def flakiness_status_labels_quick_action(reports_count)
           case reports_count
-          when 1000..Float::INFINITY
+          when 42..Float::INFINITY
             '/label ~"flakiness::1"'
-          when 500..999
+          when 22..41
             '/label ~"flakiness::2"'
-          when 10..499
+          when 4..21
             '/label ~"flakiness::3"'
           else
             '/label ~"flakiness::4"'

data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb

@@ -69,6 +69,7 @@ module GitlabQuality
 
           note_body = [
             new_reports_list.to_s,
+            slowness_status_labels_quick_action(new_reports_list.reports_count),
             identity_labels_quick_action,
             relate_issues_quick_actions(related_issues)
           ].join("\n")
@@ -108,6 +109,28 @@ module GitlabQuality
           end
         end
 
+        # The report count is based on the percentiles of slowness issues.
+        #
+        # See https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/scripts/unhealthy_test_issues_statistics.rb
+        # to gather these statistics.
+        #
+        # P75       => slowness::4
+        # P90       => slowness::3
+        # P95       => slowness::2
+        # Above P95 => slowness::1
+        def slowness_status_labels_quick_action(reports_count)
+          case reports_count
+          when 40..Float::INFINITY
+            '/label ~"slowness::1"'
+          when 28..39
+            '/label ~"slowness::2"'
+          when 12..27
+            '/label ~"slowness::3"'
+          else
+            '/label ~"slowness::4"'
+          end
+        end
+
         def identity_labels_quick_action
           labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
           %(/label #{labels_list})

data/lib/gitlab_quality/test_tooling/test_metric/log_test_metrics.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module TestMetric
+      class LogTestMetrics
+        include Concerns::TestMetrics
+        include Concerns::InfluxdbTools
+
+        CUSTOM_METRICS_KEY = :custom_test_metrics
+
+        def initialize(examples:, influxdb_url: nil, influxdb_token: nil, influxdb_bucket: nil, run_type: nil)
+          @examples = examples
+          @influxdb_url = influxdb_url
+          @influxdb_token = influxdb_token
+          @influxdb_bucket = influxdb_bucket
+          @run_type = run_type
+        end
+
+        # Push test execution metrics to influxdb
+        #
+        # @param [Array<String>] custom_keys_tags
+        # @param [Array<String>] custom_keys_fields
+        # @return [nil]
+        def push_test_metrics(custom_keys_tags: nil, custom_keys_fields: nil)
+          @test_metrics ||= examples.filter_map { |example| parse_test_results(example, custom_keys_tags, custom_keys_fields) }
+
+          write_api(url: influxdb_url, token: influxdb_token, bucket: influxdb_bucket).write(data: test_metrics)
+          Runtime::Logger.debug("Pushed #{test_metrics.length} test execution entries to influxdb")
+        rescue StandardError => e
+          Runtime::Logger.error("Failed to push test execution metrics to influxdb, error: #{e}")
+        end
+
+        # Save metrics in json file
+        #
+        # @param [String] file_name
+        # @param [Array<String>] custom_keys_tags
+        # @param [Array<String>] custom_keys_fields
+        # @return [nil]
+        def save_test_metrics(file_name, custom_keys_tags: nil, custom_keys_fields: nil)
+          @test_metrics ||= examples.filter_map { |example| parse_test_results(example, custom_keys_tags, custom_keys_fields) }
+          file = "tmp/#{file_name}"
+
+          File.write(file, test_metrics.to_json) && Runtime::Logger.debug("Saved test metrics to #{file}")
+        rescue StandardError => e
+          Runtime::Logger.error("Failed to save test execution metrics, error: #{e}")
+        end
+
+        private
+
+        attr_reader :examples, :test_metrics, :influxdb_url, :influxdb_token, :influxdb_bucket, :run_type
+
+        # Transform example to influxdb compatible metrics data
+        # https://github.com/influxdata/influxdb-client-ruby#data-format
+        #
+        # @param [RSpec::Core::Example] example
+        # @param [Array<String>] custom_keys_tags
+        # @param [Array<String>] custom_keys_fields
+        # @return [Hash]
+        def parse_test_results(example, custom_keys_tags, custom_keys_fields)
+          {
+            name: 'test-stats',
+            time: time,
+            tags: tags(example, custom_keys_tags, run_type),
+            fields: fields(example, custom_keys_fields)
+          }
+        rescue StandardError => e
+          Runtime::Logger.error("Failed to transform example '#{example.id}', error: #{e}")
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "1.23.0"
+    VERSION = "1.24.0"
   end
 end

metadata

@@ -1,15 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 1.23.0
+  version: 1.24.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-15 00:00:00.000000000 Z
+date: 2024-04-29 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.2'
 - !ruby/object:Gem::Dependency
   name: climate_control
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +86,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.7'
+- !ruby/object:Gem::Dependency
+  name: influxdb-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: lefthook
   requirement: !ruby/object:Gem::Requirement
@@ -404,6 +438,8 @@ files:
 - lefthook.yml
 - lib/gitlab_quality/test_tooling.rb
 - lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb
+- lib/gitlab_quality/test_tooling/concerns/influxdb_tools.rb
+- lib/gitlab_quality/test_tooling/concerns/test_metrics.rb
 - lib/gitlab_quality/test_tooling/failed_jobs_table.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/branches_client.rb
 - lib/gitlab_quality/test_tooling/gitlab_client/branches_dry_client.rb
@@ -461,6 +497,7 @@ files:
 - lib/gitlab_quality/test_tooling/test_meta/processor/meta_processor.rb
 - lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb
 - lib/gitlab_quality/test_tooling/test_metric/json_test_metric.rb
+- lib/gitlab_quality/test_tooling/test_metric/log_test_metrics.rb
 - lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb
 - lib/gitlab_quality/test_tooling/test_result/base_test_result.rb
 - lib/gitlab_quality/test_tooling/test_result/j_unit_test_result.rb
@@ -493,7 +530,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.3.27
 signing_key:
 specification_version: 4
 summary: A collection of test-related tools.