gitlab_quality-test_tooling 2.16.0 → 2.20.2

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb

@@ -8,6 +8,34 @@ module GitlabQuality
       class Config
         include Singleton
 
+        # GCS client configuration object used to push metrics as json file to gcs bucket
+        #
+        class GCS
+          def initialize(project_id:, credentials:, bucket_name:, metrics_file_name:, dry_run: false)
+            @project_id = project_id
+            @credentials = credentials
+            @bucket_name = bucket_name
+            @metrics_file_name = metrics_file_name
+            @dry_run = dry_run
+          end
+
+          attr_reader :project_id, :credentials, :bucket_name, :metrics_file_name, :dry_run
+        end
+
+        # ClickHouse client configuration object
+        #
+        class ClickHouse
+          def initialize(url:, database:, table_name:, username:, password:)
+            @url = url
+            @database = database
+            @table_name = table_name
+            @username = username
+            @password = password
+          end
+
+          attr_reader :url, :database, :table_name, :username, :password
+        end
+
         class << self
           def configuration
             Config.instance
@@ -18,25 +46,70 @@ module GitlabQuality
           end
         end
 
-        attr_accessor :influxdb_url,
-          :influxdb_token,
-          :influxdb_bucket,
-          :gcs_bucket,
-          :gcs_project_id,
-          :gcs_credentials,
-          :gcs_metrics_file_name,
-          :test_metric_file_name,
-          :run_type
+        attr_reader :gcs_config, :clickhouse_config
+        attr_accessor :run_type
+        attr_writer :extra_rspec_metadata_keys,
+          :skip_record_proc,
+          :test_retried_proc,
+          :custom_metrics_proc,
+          :logger
 
-        attr_writer :custom_keys_tags,
-          :custom_keys_fields
+        # rubocop:disable Style/TrivialAccessors -- allows documenting that setting config enables the export as well as document input class type
+
+        # Enable metrics export to gcs bucket by setting configuration object
+        #
+        # @param config [Config::GCS]
+        # @return [GCS]
+        def gcs_config=(config)
+          @gcs_config = config
+        end
+
+        # Enable metrics export to clickhouse by setting configuration object
+        #
+        # @param config [Config::ClickHouse]
+        # @return [ClickHouse]
+        def clickhouse_config=(config)
+          @clickhouse_config = config
+        end
+
+        # rubocop:enable Style/TrivialAccessors
+
+        # Extra rspec metadata keys to include in exported metrics
+        #
+        # @return [Array<Symbol>]
+        def extra_rspec_metadata_keys
+          @extra_rspec_metadata_keys ||= []
+        end
+
+        # A lambda that determines whether to skip recording a test result
+        #
+        # This is useful when you would want to skip initial failure when retrying specs is set up in a separate process
+        # and you want to avoid duplicate records
+        #
+        # @return [Proc]
+        def skip_record_proc
+          @skip_record_proc ||= ->(_example) { false }
+        end
+
+        # A lambda that determines whether a test was retried or not
+        #
+        # @return [Proc]
+        def test_retried_proc
+          @test_retried_proc ||= ->(_example) { false }
+        end
 
-        def custom_keys_tags
-          @custom_keys_tags || []
+        # A lambda that return hash with additional custom metrics
+        #
+        # @return [Proc]
+        def custom_metrics_proc
+          @custom_metrics_proc ||= ->(_example) { {} }
         end
 
-        def custom_keys_fields
-          @custom_keys_fields || []
+        # Logger instance
+        #
+        # @return [Logger]
+        def logger
+          @logger ||= Logger.new($stdout, level: Logger::INFO)
         end
       end
     end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb

@@ -1,56 +1,93 @@
 # frozen_string_literal: true
 
+require "rspec/core/formatters/base_formatter"
+
+# rubocop:disable Metrics/AbcSize
 module GitlabQuality
   module TestTooling
     module TestMetricsExporter
       class Formatter < RSpec::Core::Formatters::BaseFormatter
         RSpec::Core::Formatters.register(self, :stop)
 
+        LOG_PREFIX = "[MetricsExporter]"
+
         def stop(notification)
-          setup_test_metrics_exporter(notification.examples)
-
-          log_test_metrics.push_test_metrics(
-            custom_keys_tags: config.custom_keys_tags,
-            custom_keys_fields: config.custom_keys_fields
-          )
-
-          log_test_metrics.save_test_metrics(
-            file_name: config.test_metric_file_name,
-            custom_keys_tags: config.custom_keys_tags,
-            custom_keys_fields: config.custom_keys_fields
-          )
+          data = notification.examples.filter_map do |example|
+            next if config.skip_record_proc.call(example)
+
+            TestMetrics.new(example, time).data
+          end
+          return logger.warn("#{LOG_PREFIX} No test execution records found, metrics will not be exported!") if data.empty?
+
+          push_to_gcs(data)
+          push_to_clickhouse(data)
         end
 
         private
 
-        attr_reader :log_test_metrics
-
+        # Configuration instance
+        #
+        # @return [Config]
         def config
           Config.configuration
         end
 
-        # rubocop:disable Metrics/AbcSize
-        def setup_test_metrics_exporter(examples)
-          @log_test_metrics = LogTestMetrics.new(
-            examples: examples,
-            run_type: config.run_type
-          )
-
-          @log_test_metrics.configure_influxdb_client(
-            influxdb_url: config.influxdb_url,
-            influxdb_token: config.influxdb_token,
-            influxdb_bucket: config.influxdb_bucket
-          )
-
-          @log_test_metrics.configure_gcs_client(
-            gcs_bucket: config.gcs_bucket,
-            gcs_project_id: config.gcs_project_id,
-            gcs_credentials: config.gcs_credentials,
-            gcs_metrics_file_name: config.gcs_metrics_file_name
-          )
+        # Logger instance
+        #
+        # @return [Logger]
+        def logger
+          config.logger
+        end
+
+        # Single common timestamp for all exported example metrics to keep data points consistently grouped
+        #
+        # @return [Time]
+        def time
+          return @time if @time
+
+          ci_created_at = Runtime::Env.ci_pipeline_created_at
+          @time = ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc
+        end
+
+        # Push data to gcs
+        #
+        # @param data [Array]
+        # @return [void]
+        def push_to_gcs(data)
+          return logger.debug("#{LOG_PREFIX} GCS configuration missing, skipping gcs export!") unless config.gcs_config
+
+          gcs_config = config.gcs_config
+          GcsTools.gcs_client(
+            project_id: gcs_config.project_id,
+            credentials: gcs_config.credentials,
+            dry_run: gcs_config.dry_run
+          ).put_object(gcs_config.bucket_name, gcs_config.metrics_file_name, data.to_json)
+          logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to GCS bucket!")
+        rescue StandardError => e
+          logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to GCS: #{e.message}")
+        end
+
+        # Push data to clickhouse
+        #
+        # @param data [Array<Hash>]
+        # @return [void]
+        def push_to_clickhouse(data)
+          return logger.debug("ClickHouse configuration missing, skipping ClickHouse export!") unless config.clickhouse_config
+
+          clickhouse_config = config.clickhouse_config
+          ClickHouse::Client.new(
+            url: clickhouse_config.url,
+            database: clickhouse_config.database,
+            username: clickhouse_config.username,
+            password: clickhouse_config.password,
+            logger: logger
+          ).insert_json_data(clickhouse_config.table_name, data)
+          logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to ClickHouse!")
+        rescue StandardError => e
+          logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to ClickHouse: #{e.message}")
         end
-        # rubocop:enable Metrics/AbcSize
       end
     end
   end
 end
+# rubocop:enable Metrics/AbcSize

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb

@@ -5,131 +5,138 @@ require 'time'
 module GitlabQuality
   module TestTooling
     module TestMetricsExporter
-      module TestMetrics
-        # Single common timestamp for all exported example metrics to keep data points consistently grouped
-        #
-        # @return [Time]
-        def time
-          return @time if defined?(@time)
-
-          created_at = Time.strptime(env('CI_PIPELINE_CREATED_AT'), '%Y-%m-%dT%H:%M:%S%z') if env('CI_PIPELINE_CREATED_AT')
-          @time = Time.parse((created_at || Time.now).utc.strftime('%Y-%m-%d %H:%M:%S %z'))
+      class TestMetrics
+        def initialize(example, timestamp)
+          @example = example
+          @timestamp = timestamp
         end
 
-        # rubocop:disable Metrics/AbcSize
-        # Metrics tags
+        # Test data hash
         #
-        # @param [RSpec::Core::Example] example
-        # @param [Array<String>] custom_keys
-        # @param [String]
         # @return [Hash]
-        def tags(example, custom_keys, run_type)
+        def data
           {
-            name: example.full_description,
-            file_path: example.metadata[:file_path].sub(/\A./, ''),
-            status: status(example),
-            quarantined: quarantined(example),
-            job_name: job_name,
-            merge_request: merge_request,
-            run_type: run_type,
-            feature_category: example.metadata[:feature_category],
-            product_group: example.metadata[:product_group],
-            exception_class: example.execution_result.exception&.class&.to_s,
-            **custom_metrics(example.metadata, custom_keys)
+            time: timestamp,
+            **rspec_metrics,
+            **ci_metrics,
+            **custom_metrics
           }.compact
         end
-        # rubocop:enable Metrics/AbcSize
 
-        # Metrics fields
+        private
+
+        attr_reader :example, :timestamp
+
+        # Exporter configuration
+        #
+        # @return [Config]
+        def config
+          Config.configuration
+        end
+
+        # Rspec related metrics
         #
-        # @param [RSpec::Core::Example] example
-        # @param [Array<String>] custom_keys
         # @return [Hash]
-        def fields(example, custom_keys)
+        def rspec_metrics # rubocop:disable Metrics/AbcSize
           {
             id: example.id,
+            name: example.full_description,
+            file_path: example.metadata[:file_path].sub(/\A./, ''),
+            status: example.execution_result.status,
             run_time: (example.execution_result.run_time * 1000).round,
-            job_url: Runtime::Env.ci_job_url,
-            pipeline_url: env('CI_PIPELINE_URL'),
-            pipeline_id: env('CI_PIPELINE_ID'),
-            job_id: env('CI_JOB_ID'),
-            merge_request_iid: merge_request_iid,
-            failure_exception: example.execution_result.exception.to_s.delete("\n"),
-            **custom_metrics(example.metadata, custom_keys)
-          }.compact
+            location: example_location,
+            exception_class: exception_class,
+            failure_exception: failure_exception,
+            quarantined: quarantined?,
+            test_retried: config.test_retried_proc.call(example)
+          }
         end
 
-        # Return a more detailed status
-        #
-        # - if test is failed or pending, return rspec status
-        # - if test passed but had more than 1 attempt, consider test flaky
+        # CI related metrics
         #
-        # @param [RSpec::Core::Example] example
-        # @return [Symbol]
-        def status(example)
-          rspec_status = example.execution_result.status
-          return rspec_status if [:pending, :failed].include?(rspec_status)
-
-          retry_attempts(example.metadata).positive? ? :flaky : :passed
+        # @return [Hash]
+        def ci_metrics
+          {
+            ci_project_id: env("CI_PROJECT_ID")&.to_i,
+            ci_project_path: env("CI_PROJECT_PATH"),
+            ci_job_name: ci_job_name,
+            ci_job_id: env('CI_JOB_ID')&.to_i,
+            ci_pipeline_id: env('CI_PIPELINE_ID')&.to_i,
+            ci_merge_request_iid: (env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID'))&.to_i,
+            ci_branch: env("CI_COMMIT_REF_NAME"),
+            ci_target_branch: env("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
+          }
         end
 
-        # Retry attempts
+        # Additional custom metrics
         #
-        # @param [Hash] example
-        # @return [Integer]
-        def retry_attempts(metadata)
-          metadata[:retry_attempts] || 0
+        # @return [Hash]
+        def custom_metrics
+          metrics = example.metadata
+            .slice(*config.extra_rspec_metadata_keys)
+            .merge(config.custom_metrics_proc.call(example))
+
+          metrics.each_with_object({}) do |(k, value), custom_metrics|
+            custom_metrics[k.to_sym] = metrics_value(value)
+          end
         end
 
         # Checks if spec is quarantined
         #
-        # @param [RSpec::Core::Example] example
         # @return [String]
-        def quarantined(example)
-          return "false" unless example.metadata.key?(:quarantine)
+        def quarantined?
+          return false unless example.metadata.key?(:quarantine)
 
           # if quarantine key is present and status is pending, consider it quarantined
-          (example.execution_result.status == :pending).to_s
+          example.execution_result.status == :pending
         end
 
         # Base ci job name
         #
         # @return [String]
-        def job_name
-          @job_name ||= Runtime::Env.ci_job_name&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+        def ci_job_name
+          env("CI_JOB_NAME")&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
         end
 
-        # Check if it is a merge request execution
+        # Example location
         #
         # @return [String]
-        def merge_request
-          (!!merge_request_iid).to_s
+        def example_location
+          # ensures that location will be correct even in case of shared examples
+          file = example
+                 .metadata
+                 .fetch(:shared_group_inclusion_backtrace)
+                 .last
+                 &.formatted_inclusion_location
+
+          return example.location unless file
+
+          file
         end
 
-        # Merge request iid
+        # Failure exception class
         #
         # @return [String]
-        def merge_request_iid
-          env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID')
+        def exception_class
+          example.execution_result.exception&.class&.to_s
         end
 
-        # Custom test metrics
+        # Truncated exception stacktrace
         #
-        # @param [Hash] metadata
-        # @param [Array] array of custom metrics keys
-        # @return [Hash]
-        def custom_metrics(metadata, custom_keys)
-          return {} if custom_keys.nil?
-
-          custom_metrics = {}
-          custom_keys.each do |k|
-            value = metadata[k.to_sym]
-            v = value.is_a?(Numeric) || value.nil? ? value : value.to_s
+        # @return [String]
+        def failure_exception
+          example.execution_result.exception.then do |exception|
+            next unless exception
 
-            custom_metrics[k.to_sym] = v
+            exception.to_s.tr("\n", " ").slice(0, 1000)
           end
+        end
 
-          custom_metrics
+        # Test run type | suite name
+        #
+        # @return [String]
+        def run_type
+          config.run_type || ci_job_name || "unknown"
         end
 
         # Return non empty environment variable value
@@ -141,6 +148,24 @@ module GitlabQuality
 
           ENV.fetch(name)
         end
+
+        # Metrics value cast to a valid type
+        #
+        # @param value [Object]
+        # @return [Object]
+        def metrics_value(value)
+          return value if value.is_a?(Numeric) || value.is_a?(String) || bool?(value) || value.nil?
+
+          value.to_s
+        end
+
+        # Value is a true or false
+        #
+        # @param val [Object]
+        # @return [Boolean]
+        def bool?(val)
+          [true, false].include?(val)
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

@@ -73,6 +73,10 @@ module GitlabQuality
           product_group != ''
         end
 
+        def feature_category?
+          feature_category && !feature_category.empty?
+        end
+
         def failure_issue
           report['failure_issue']
         end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "2.16.0"
+    VERSION = "2.20.2"
   end
 end

data/lib/gitlab_quality/test_tooling.rb

@@ -7,6 +7,8 @@ module GitlabQuality
   module TestTooling
     Error = Class.new(StandardError)
     loader = Zeitwerk::Loader.new
+    loader.push_dir(__dir__.to_s, namespace: GitlabQuality)
+    loader.ignore("#{__dir__}/test_tooling.rb")
     loader.push_dir("#{__dir__}/test_tooling", namespace: GitlabQuality::TestTooling)
     loader.ignore("#{__dir__}/test_tooling/version.rb")