gitlab_quality-test_tooling 2.16.0 → 2.21.0

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb

@@ -8,6 +8,34 @@ module GitlabQuality
       class Config
         include Singleton
 
+        # GCS client configuration object used to push metrics as json file to gcs bucket
+        #
+        class GCS
+          def initialize(project_id:, credentials:, bucket_name:, metrics_file_name:, dry_run: false)
+            @project_id = project_id
+            @credentials = credentials
+            @bucket_name = bucket_name
+            @metrics_file_name = metrics_file_name
+            @dry_run = dry_run
+          end
+
+          attr_reader :project_id, :credentials, :bucket_name, :metrics_file_name, :dry_run
+        end
+
+        # ClickHouse client configuration object
+        #
+        class ClickHouse
+          def initialize(url:, database:, table_name:, username:, password:)
+            @url = url
+            @database = database
+            @table_name = table_name
+            @username = username
+            @password = password
+          end
+
+          attr_reader :url, :database, :table_name, :username, :password
+        end
+
         class << self
           def configuration
             Config.instance
@@ -18,25 +46,97 @@ module GitlabQuality
           end
         end
 
-        attr_accessor :influxdb_url,
-          :influxdb_token,
-          :influxdb_bucket,
-          :gcs_bucket,
-          :gcs_project_id,
-          :gcs_credentials,
-          :gcs_metrics_file_name,
-          :test_metric_file_name,
-          :run_type
+        attr_reader :gcs_config, :clickhouse_config, :initial_run
+        attr_accessor :run_type
+        attr_writer :extra_rspec_metadata_keys,
+          :skip_record_proc,
+          :test_retried_proc,
+          :custom_metrics_proc,
+          :logger
+
+        # rubocop:disable Style/TrivialAccessors -- allows documenting that setting config enables the export as well as document input class type
+
+        # Enable metrics export to gcs bucket by setting configuration object
+        #
+        # @param config [Config::GCS]
+        # @return [GCS]
+        def gcs_config=(config)
+          @gcs_config = config
+        end
+
+        # Enable metrics export to clickhouse by setting configuration object
+        #
+        # @param config [Config::ClickHouse]
+        # @return [ClickHouse]
+        def clickhouse_config=(config)
+          @clickhouse_config = config
+        end
 
-        attr_writer :custom_keys_tags,
-          :custom_keys_fields
+        # Additional columns to be created in the table if initial_run setup is used
+        # Columns should be defined in the format used for ALTER TABLE query, example;
+        #   [
+        #     "feature_category LowCardinality(String) DEFAULT ''",
+        #     "level LowCardinality(String) DEFAULT ''"
+        #   ]
+        #
+        # @param columns [Array]
+        # @return [Array]
+        def extra_metadata_columns=(columns)
+          @extra_metadata_columns = columns
+        end
+
+        # rubocop:enable Style/TrivialAccessors
+
+        # Marks execution as initial run and performs setup tasks before running tests, like creating database in ClickHouse
+        #
+        # @return [Boolean]
+        def initial_run!
+          @initial_run = true
+        end
+
+        # Additional metadata columns used during initial table creation
+        #
+        # @return [Array]
+        def extra_metadata_columns
+          @extra_metadata_columns ||= []
+        end
+
+        # Extra rspec metadata keys to include in exported metrics
+        #
+        # @return [Array<Symbol>]
+        def extra_rspec_metadata_keys
+          @extra_rspec_metadata_keys ||= []
+        end
+
+        # A lambda that determines whether to skip recording a test result
+        #
+        # This is useful when you would want to skip initial failure when retrying specs is set up in a separate process
+        # and you want to avoid duplicate records
+        #
+        # @return [Proc]
+        def skip_record_proc
+          @skip_record_proc ||= ->(_example) { false }
+        end
+
+        # A lambda that determines whether a test was retried or not
+        #
+        # @return [Proc]
+        def test_retried_proc
+          @test_retried_proc ||= ->(_example) { false }
+        end
 
-        def custom_keys_tags
-          @custom_keys_tags || []
+        # A lambda that return hash with additional custom metrics
+        #
+        # @return [Proc]
+        def custom_metrics_proc
+          @custom_metrics_proc ||= ->(_example) { {} }
         end
 
-        def custom_keys_fields
-          @custom_keys_fields || []
+        # Logger instance
+        #
+        # @return [Logger]
+        def logger
+          @logger ||= Logger.new($stdout, level: Logger::INFO)
         end
       end
     end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb

@@ -1,55 +1,76 @@
 # frozen_string_literal: true
 
+require "rspec/core/formatters/base_formatter"
+
 module GitlabQuality
   module TestTooling
     module TestMetricsExporter
       class Formatter < RSpec::Core::Formatters::BaseFormatter
-        RSpec::Core::Formatters.register(self, :stop)
+        include Utils
+
+        RSpec::Core::Formatters.register(self, *[:stop, Utils.config.initial_run ? :start : nil].compact)
+
+        LOG_PREFIX = "[MetricsExporter]"
+
+        def start(_notification)
+          logger.debug("#{LOG_PREFIX} Running initial setup for metrics export")
+          return logger.warn("#{LOG_PREFIX} Initial setup is enabled, but clickhouse configuration is missing!") unless clickhouse_config
+
+          create_clickhouse_metrics_table
+        end
 
         def stop(notification)
-          setup_test_metrics_exporter(notification.examples)
-
-          log_test_metrics.push_test_metrics(
-            custom_keys_tags: config.custom_keys_tags,
-            custom_keys_fields: config.custom_keys_fields
-          )
-
-          log_test_metrics.save_test_metrics(
-            file_name: config.test_metric_file_name,
-            custom_keys_tags: config.custom_keys_tags,
-            custom_keys_fields: config.custom_keys_fields
-          )
+          logger.debug("#{LOG_PREFIX} Starting test metrics export")
+          data = notification.examples.filter_map do |example|
+            next if config.skip_record_proc.call(example)
+
+            TestMetrics.new(example, time).data
+          end
+          return logger.warn("#{LOG_PREFIX} No test execution records found, metrics will not be exported!") if data.empty?
+
+          push_to_gcs(data)
+          push_to_clickhouse(data)
         end
 
         private
 
-        attr_reader :log_test_metrics
+        delegate :gcs_config, :clickhouse_config, to: :config
+
+        # Single common timestamp for all exported example metrics to keep data points consistently grouped
+        #
+        # @return [Time]
+        def time
+          return @time if @time
+
+          ci_created_at = Runtime::Env.ci_pipeline_created_at
+          @time = ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc
+        end
+
+        # Push data to gcs
+        #
+        # @param data [Array]
+        # @return [void]
+        def push_to_gcs(data)
+          return logger.debug("#{LOG_PREFIX} GCS configuration missing, skipping gcs export!") unless gcs_config
 
-        def config
-          Config.configuration
+          gcs_client.put_object(gcs_config.bucket_name, gcs_config.metrics_file_name, data.to_json)
+          logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to GCS bucket!")
+        rescue StandardError => e
+          logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to GCS: #{e.message}")
         end
 
-        # rubocop:disable Metrics/AbcSize
-        def setup_test_metrics_exporter(examples)
-          @log_test_metrics = LogTestMetrics.new(
-            examples: examples,
-            run_type: config.run_type
-          )
-
-          @log_test_metrics.configure_influxdb_client(
-            influxdb_url: config.influxdb_url,
-            influxdb_token: config.influxdb_token,
-            influxdb_bucket: config.influxdb_bucket
-          )
-
-          @log_test_metrics.configure_gcs_client(
-            gcs_bucket: config.gcs_bucket,
-            gcs_project_id: config.gcs_project_id,
-            gcs_credentials: config.gcs_credentials,
-            gcs_metrics_file_name: config.gcs_metrics_file_name
-          )
+        # Push data to clickhouse
+        #
+        # @param data [Array<Hash>]
+        # @return [void]
+        def push_to_clickhouse(data)
+          return logger.debug("ClickHouse configuration missing, skipping ClickHouse export!") unless clickhouse_config
+
+          clickhouse_client.insert_json_data(clickhouse_config.table_name, data)
+          logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to ClickHouse!")
+        rescue StandardError => e
+          logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to ClickHouse: #{e.message}")
         end
-        # rubocop:enable Metrics/AbcSize
       end
     end
   end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb

@@ -5,131 +5,149 @@ require 'time'
 module GitlabQuality
   module TestTooling
     module TestMetricsExporter
-      module TestMetrics
-        # Single common timestamp for all exported example metrics to keep data points consistently grouped
-        #
-        # @return [Time]
-        def time
-          return @time if defined?(@time)
-
-          created_at = Time.strptime(env('CI_PIPELINE_CREATED_AT'), '%Y-%m-%dT%H:%M:%S%z') if env('CI_PIPELINE_CREATED_AT')
-          @time = Time.parse((created_at || Time.now).utc.strftime('%Y-%m-%d %H:%M:%S %z'))
+      class TestMetrics
+        def initialize(example, timestamp)
+          @example = example
+          @timestamp = timestamp
         end
 
-        # rubocop:disable Metrics/AbcSize
-        # Metrics tags
+        # Test data hash
         #
-        # @param [RSpec::Core::Example] example
-        # @param [Array<String>] custom_keys
-        # @param [String]
         # @return [Hash]
-        def tags(example, custom_keys, run_type)
+        def data
           {
-            name: example.full_description,
-            file_path: example.metadata[:file_path].sub(/\A./, ''),
-            status: status(example),
-            quarantined: quarantined(example),
-            job_name: job_name,
-            merge_request: merge_request,
-            run_type: run_type,
-            feature_category: example.metadata[:feature_category],
-            product_group: example.metadata[:product_group],
-            exception_class: example.execution_result.exception&.class&.to_s,
-            **custom_metrics(example.metadata, custom_keys)
+            time: timestamp,
+            **rspec_metrics,
+            **ci_metrics,
+            **custom_metrics
           }.compact
         end
-        # rubocop:enable Metrics/AbcSize
 
-        # Metrics fields
+        private
+
+        attr_reader :example, :timestamp
+
+        # Exporter configuration
+        #
+        # @return [Config]
+        def config
+          Config.configuration
+        end
+
+        # Rspec related metrics
         #
-        # @param [RSpec::Core::Example] example
-        # @param [Array<String>] custom_keys
         # @return [Hash]
-        def fields(example, custom_keys)
+        def rspec_metrics # rubocop:disable Metrics/AbcSize
           {
-            id: example.id,
+            id: without_relative_path(example.id),
+            name: example.full_description,
+            hash: OpenSSL::Digest.hexdigest("SHA256", "#{file_path}#{example.full_description}")[..40],
+            file_path: file_path,
+            status: example.execution_result.status,
             run_time: (example.execution_result.run_time * 1000).round,
-            job_url: Runtime::Env.ci_job_url,
-            pipeline_url: env('CI_PIPELINE_URL'),
-            pipeline_id: env('CI_PIPELINE_ID'),
-            job_id: env('CI_JOB_ID'),
-            merge_request_iid: merge_request_iid,
-            failure_exception: example.execution_result.exception.to_s.delete("\n"),
-            **custom_metrics(example.metadata, custom_keys)
-          }.compact
+            location: example_location,
+            exception_class: exception_class,
+            failure_exception: failure_exception,
+            quarantined: quarantined?,
+            feature_category: example.metadata[:feature_category] || "",
+            test_retried: config.test_retried_proc.call(example)
+          }
         end
 
-        # Return a more detailed status
+        # CI related metrics
         #
-        # - if test is failed or pending, return rspec status
-        # - if test passed but had more than 1 attempt, consider test flaky
-        #
-        # @param [RSpec::Core::Example] example
-        # @return [Symbol]
-        def status(example)
-          rspec_status = example.execution_result.status
-          return rspec_status if [:pending, :failed].include?(rspec_status)
-
-          retry_attempts(example.metadata).positive? ? :flaky : :passed
+        # @return [Hash]
+        def ci_metrics
+          {
+            ci_project_id: env("CI_PROJECT_ID")&.to_i,
+            ci_project_path: env("CI_PROJECT_PATH"),
+            ci_job_name: ci_job_name,
+            ci_job_id: env('CI_JOB_ID')&.to_i,
+            ci_pipeline_id: env('CI_PIPELINE_ID')&.to_i,
+            ci_merge_request_iid: (env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID'))&.to_i,
+            ci_branch: env("CI_COMMIT_REF_NAME"),
+            ci_target_branch: env("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
+          }
         end
 
-        # Retry attempts
+        # Additional custom metrics
         #
-        # @param [Hash] example
-        # @return [Integer]
-        def retry_attempts(metadata)
-          metadata[:retry_attempts] || 0
+        # @return [Hash]
+        def custom_metrics
+          metrics = example.metadata
+            .slice(*config.extra_rspec_metadata_keys)
+            .merge(config.custom_metrics_proc.call(example))
+
+          metrics.each_with_object({}) do |(k, value), custom_metrics|
+            custom_metrics[k.to_sym] = metrics_value(value)
+          end
         end
 
         # Checks if spec is quarantined
         #
-        # @param [RSpec::Core::Example] example
         # @return [String]
-        def quarantined(example)
-          return "false" unless example.metadata.key?(:quarantine)
+        def quarantined?
+          return false unless example.metadata.key?(:quarantine)
 
           # if quarantine key is present and status is pending, consider it quarantined
-          (example.execution_result.status == :pending).to_s
+          example.execution_result.status == :pending
         end
 
         # Base ci job name
         #
         # @return [String]
-        def job_name
-          @job_name ||= Runtime::Env.ci_job_name&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+        def ci_job_name
+          env("CI_JOB_NAME")&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
         end
 
-        # Check if it is a merge request execution
+        # Example location
         #
         # @return [String]
-        def merge_request
-          (!!merge_request_iid).to_s
+        def example_location
+          return @example_location if @example_location
+
+          # ensures that location will be correct even in case of shared examples
+          file = example
+                 .metadata
+                 .fetch(:shared_group_inclusion_backtrace)
+                 .last
+                 &.formatted_inclusion_location
+
+          return without_relative_path(example.location) unless file
+
+          @example_location = without_relative_path(file)
         end
 
-        # Merge request iid
+        # File path based on actual test location, not shared example location
         #
         # @return [String]
-        def merge_request_iid
-          env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID')
+        def file_path
+          @file_path ||= example_location.gsub(/:\d+$/, "")
         end
 
-        # Custom test metrics
+        # Failure exception class
         #
-        # @param [Hash] metadata
-        # @param [Array] array of custom metrics keys
-        # @return [Hash]
-        def custom_metrics(metadata, custom_keys)
-          return {} if custom_keys.nil?
+        # @return [String]
+        def exception_class
+          example.execution_result.exception&.class&.to_s
+        end
 
-          custom_metrics = {}
-          custom_keys.each do |k|
-            value = metadata[k.to_sym]
-            v = value.is_a?(Numeric) || value.nil? ? value : value.to_s
+        # Truncated exception stacktrace
+        #
+        # @return [String]
+        def failure_exception
+          example.execution_result.exception.then do |exception|
+            next unless exception
 
-            custom_metrics[k.to_sym] = v
+            exception.to_s.tr("\n", " ").slice(0, 1000)
           end
+        end
 
-          custom_metrics
+        # Test run type | suite name
+        #
+        # @return [String]
+        def run_type
+          config.run_type || ci_job_name || "unknown"
         end
 
         # Return non empty environment variable value
@@ -141,6 +159,32 @@ module GitlabQuality
 
           ENV.fetch(name)
         end
+
+        # Metrics value cast to a valid type
+        #
+        # @param value [Object]
+        # @return [Object]
+        def metrics_value(value)
+          return value if value.is_a?(Numeric) || value.is_a?(String) || bool?(value) || value.nil?
+
+          value.to_s
+        end
+
+        # Value is a true or false
+        #
+        # @param val [Object]
+        # @return [Boolean]
+        def bool?(val)
+          [true, false].include?(val)
+        end
+
+        # Path without leading ./
+        #
+        # @param path [String]
+        # @return [String]
+        def without_relative_path(path)
+          path.gsub(%r{^\./}, "")
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/utils.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module TestMetricsExporter
+      module Utils
+        # Instance of metrics exporter configuration
+        #
+        # @return [Config]
+        def config
+          Config.configuration
+        end
+
+        # Configured logger instance
+        #
+        # @return [Logger]
+        def logger
+          config.logger
+        end
+
+        # Configured clickhouse client
+        #
+        # @return [ClickHouse::Client]
+        def clickhouse_client
+          ClickHouse::Client.new(
+            url: config.clickhouse_config.url,
+            database: config.clickhouse_config.database,
+            username: config.clickhouse_config.username,
+            password: config.clickhouse_config.password,
+            logger: logger
+          )
+        end
+
+        # GCS client instance
+        #
+        # @return [Fog::Storage::Google, GcsTools::GCSMockClient]
+        def gcs_client
+          GcsTools.gcs_client(
+            project_id: config.gcs_config.project_id,
+            credentials: config.gcs_config.credentials,
+            dry_run: config.gcs_config.dry_run
+          )
+        end
+
+        # Create table for metrics export using current ClickHouse configuration
+        #
+        # This method is mostly for schema documentation but it can be used together with initial_run! method in
+        #
+        # @return [void]
+        def create_clickhouse_metrics_table
+          table_name = config.clickhouse_config.table_name
+
+          clickhouse_client.query(<<~SQL)
+            CREATE TABLE IF NOT EXISTS #{table_name}
+            (
+                timestamp DateTime64(6, 'UTC'),
+                id String,
+                name String,
+                hash String,
+                file_path String,
+                status LowCardinality(String),
+                run_time UInt32,
+                location String,
+                quarantined Bool,
+                test_retried Bool,
+                feature_category LowCardinality(String) DEFAULT '',
+                ci_project_id UInt32,
+                ci_job_name LowCardinality(String),
+                ci_job_id UInt64,
+                ci_pipeline_id UInt64,
+                ci_merge_request_iid UInt32 DEFAULT 0,
+                ci_project_path LowCardinality(String),
+                ci_branch String,
+                ci_target_branch LowCardinality(String),
+                exception_class String DEFAULT '',
+                failure_exception String DEFAULT ''
+            )
+            ENGINE = MergeTree()
+            PARTITION BY toYYYYMM(timestamp)
+            ORDER BY (ci_project_path, timestamp, status, feature_category, file_path, ci_pipeline_id)
+            SETTINGS index_granularity = 8192;
+          SQL
+          return if config.extra_metadata_columns.empty?
+
+          clickhouse_client.query(
+            "ALTER TABLE #{table_name} #{config.extra_metadata_columns.map { |column| "ADD COLUMN IF NOT EXISTS #{column}" }.join(', ')};"
+          )
+        end
+
+        module_function :config, :logger, :clickhouse_client
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

@@ -73,6 +73,10 @@ module GitlabQuality
           product_group != ''
         end
 
+        def feature_category?
+          feature_category && !feature_category.empty?
+        end
+
         def failure_issue
           report['failure_issue']
         end
@@ -115,7 +119,7 @@ module GitlabQuality
         end
 
         def file_base_url
-          @file_base_url ||= "https://gitlab.com/#{project == 'gitlab-org/quality/e2e-test-issues' ? 'gitlab-org/gitlab' : project}/-/blob/#{ref}/"
+          @file_base_url ||= "https://gitlab.com/#{mapped_project}/-/blob/#{ref}/"
         end
 
         def test_file_link
@@ -154,7 +158,7 @@ module GitlabQuality
         attr_reader :token, :project, :ref
 
         def mapped_project
-          if project == 'gitlab-org/quality/e2e-test-issues'
+          if ['gitlab-org/quality/e2e-test-issues', 'gitlab-org/quality/test-failure-issues'].include?(project)
             'gitlab-org/gitlab'
           else
             project

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "2.16.0"
+    VERSION = "2.21.0"
   end
 end

data/lib/gitlab_quality/test_tooling.rb

@@ -2,11 +2,14 @@
 
 require 'rainbow/refinement'
 require 'zeitwerk'
+require 'active_support/core_ext/module/delegation'
 
 module GitlabQuality
   module TestTooling
     Error = Class.new(StandardError)
     loader = Zeitwerk::Loader.new
+    loader.push_dir(__dir__.to_s, namespace: GitlabQuality)
+    loader.ignore("#{__dir__}/test_tooling.rb")
     loader.push_dir("#{__dir__}/test_tooling", namespace: GitlabQuality::TestTooling)
     loader.ignore("#{__dir__}/test_tooling/version.rb")