gitlab_quality-test_tooling 2.20.0 → 2.20.1

data/lib/gitlab_quality/test_tooling/gitlab_client/group_labels_client.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module GitlabClient
+      class GroupLabelsClient < GitlabClient
+        def initialize(token:, group:, endpoint: nil, **_kwargs)
+          @token = token
+          @group = group
+          @endpoint = endpoint
+        end
+
+        def group_labels(options: {})
+          client.group_labels(group, options)
+        end
+
+        def create_group_label(name:, color: '#428BCA', description: nil)
+          client.create_group_label(group, name, color, description: description)
+        end
+
+        private
+
+        attr_reader :group, :token, :endpoint
+
+        def client
+          @client ||= Gitlab.client(
+            endpoint: endpoint || ENV['GITLAB_API_BASE'] || Runtime::Env.gitlab_api_base,
+            private_token: token
+          )
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb

@@ -31,7 +31,7 @@ module GitlabQuality
           tests = tests.select { |test| pipeline_stages.include? test.report["stage"] } unless pipeline_stages.empty?
 
           issue = gitlab.create_issue(
-            title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
+            title: "#{Time.now.to_date.iso8601} Test session report | #{Runtime::Env.qa_run_type}",
             description: generate_description(tests),
             labels: ['automation:bot-authored', 'E2E', 'triage report', pipeline_name_label, 'suppress-contributor-links'],
             confidential: confidential

data/lib/gitlab_quality/test_tooling/report/health_problem_reporter.rb

@@ -15,7 +15,6 @@ module GitlabQuality
       class HealthProblemReporter < ReportAsIssue
         include Concerns::GroupAndCategoryLabels
         include Concerns::IssueReports
-        include TestMetricsExporter::Support::GcsTools
 
         BASE_SEARCH_LABELS    = ['test'].freeze
         FOUND_IN_MR_LABEL     = '~"found:in MR"'
@@ -151,7 +150,7 @@ module GitlabQuality
         def push_test_to_gcs(tests_data, test_results_filename)
           Runtime::Logger.info "will push the test data to GCS"
 
-          gcs_client(project_id: gcs_project_id, credentials: gcs_credentials, dry_run: dry_run).put_object(
+          GcsTools.gcs_client(project_id: gcs_project_id, credentials: gcs_credentials, dry_run: dry_run).put_object(
             gcs_bucket,
             gcs_metrics_file_name(test_results_filename),
             tests_data.to_json,
@@ -163,7 +162,7 @@ module GitlabQuality
         end
 
         def gcs_metrics_file_name(test_results_filename)
-          today = Time.now.strftime('%Y-%m-%d')
+          today = Time.now.to_date.iso8601
 
           "#{today}-#{test_results_filename}"
         end

data/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb

@@ -78,13 +78,9 @@ module GitlabQuality
         end
 
         def slow_test_rows(slow_test)
-          rows = []
-
-          slow_test.each do |test|
-            rows << slow_test_table_row(test)
+          slow_test.map do |test|
+            slow_test_table_row(test)
           end
-
-          rows
         end
 
         def build_note(slow_test)

data/lib/gitlab_quality/test_tooling/runtime/env.rb

@@ -22,10 +22,11 @@ module GitlabQuality
           'CI_PROJECT_ID' => :ci_project_id,
           'CI_PROJECT_NAME' => :ci_project_name,
           'CI_PROJECT_PATH' => :ci_project_path,
+          'CI_PIPELINE_CREATED_AT' => :ci_pipeline_created_at,
           'DEPLOY_VERSION' => :deploy_version,
           'GITLAB_QA_ISSUE_URL' => :qa_issue_url,
           'QA_GITLAB_CI_TOKEN' => :gitlab_ci_token,
-          'SLACK_QA_CHANNEL' => :slack_qa_channel
+          'SLACK_ALERTS_CHANNEL' => :slack_alerts_channel
         }.freeze
 
         ENV_VARIABLES.each do |env_name, method_name|
@@ -112,12 +113,12 @@ module GitlabQuality
         end
 
         def env_var_value_if_defined(variable)
-          return ENV.fetch(variable) if env_var_value_valid?(variable)
+          ENV.fetch(variable) if env_var_value_valid?(variable)
         end
 
         def env_var_name_if_defined(variable)
           # Pass through the variables if they are defined and not empty in the environment
-          return "$#{variable}" if env_var_value_valid?(variable)
+          "$#{variable}" if env_var_value_valid?(variable)
         end
       end
     end

data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/api_log_finder.rb

@@ -7,7 +7,7 @@ module GitlabQuality
         module Rails
           class ApiLogFinder < JsonLogFinder
             def initialize(base_path, file_path = 'gitlab-rails/api_json.log')
-              super(base_path, file_path)
+              super
             end
 
             def new_log(data)

data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/application_log_finder.rb

@@ -7,7 +7,7 @@ module GitlabQuality
         module Rails
           class ApplicationLogFinder < JsonLogFinder
             def initialize(base_path, file_path = 'gitlab-rails/application_json.log')
-              super(base_path, file_path)
+              super
             end
 
             def new_log(data)

data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/exception_log_finder.rb

@@ -7,7 +7,7 @@ module GitlabQuality
         module Rails
           class ExceptionLogFinder < JsonLogFinder
             def initialize(base_path, file_path = 'gitlab-rails/exceptions_json.log')
-              super(base_path, file_path)
+              super
             end
 
             def new_log(data)

data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/graphql_log_finder.rb

@@ -7,7 +7,7 @@ module GitlabQuality
         module Rails
           class GraphqlLogFinder < JsonLogFinder
             def initialize(base_path, file_path = 'gitlab-rails/graphql_json.log')
-              super(base_path, file_path)
+              super
             end
 
             def new_log(data)

data/lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb

@@ -10,7 +10,7 @@ module GitlabQuality
 
         attr_reader :project, :ref, :report_issue, :processed_commits, :token, :specs_file, :dry_run, :processor
 
-        TEST_PLATFORM_MAINTAINERS_SLACK_CHANNEL_ID = 'C0437FV9KBN' # test-platform-maintainers
+        TEST_TOOLING_ALERTS_SLACK_CHANNEL_ID = 'C09HQ5BN07J' # test-tooling-alerts
 
         def initialize(token:, project:, specs_file:, processor:, ref: 'master', dry_run: false)
           @specs_file = specs_file
@@ -294,7 +294,7 @@ module GitlabQuality
         # @param [String] message the message to post
         # @return [HTTP::Response]
         def post_message_on_slack(message)
-          channel = ENV.fetch('SLACK_QA_CHANNEL', nil) || TEST_PLATFORM_MAINTAINERS_SLACK_CHANNEL_ID
+          channel = ENV.fetch('SLACK_ALERTS_CHANNEL', nil) || TEST_TOOLING_ALERTS_SLACK_CHANNEL_ID
           slack_options = {
             slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
             channel: channel,

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb

@@ -8,6 +8,34 @@ module GitlabQuality
       class Config
         include Singleton
 
+        # GCS client configuration object used to push metrics as json file to gcs bucket
+        #
+        class GCS
+          def initialize(project_id:, credentials:, bucket_name:, metrics_file_name:, dry_run: false)
+            @project_id = project_id
+            @credentials = credentials
+            @bucket_name = bucket_name
+            @metrics_file_name = metrics_file_name
+            @dry_run = dry_run
+          end
+
+          attr_reader :project_id, :credentials, :bucket_name, :metrics_file_name, :dry_run
+        end
+
+        # ClickHouse client configuration object
+        #
+        class ClickHouse
+          def initialize(url:, database:, table_name:, username:, password:)
+            @url = url
+            @database = database
+            @table_name = table_name
+            @username = username
+            @password = password
+          end
+
+          attr_reader :url, :database, :table_name, :username, :password
+        end
+
         class << self
           def configuration
             Config.instance
@@ -18,25 +46,70 @@ module GitlabQuality
           end
         end
 
-        attr_accessor :influxdb_url,
-          :influxdb_token,
-          :influxdb_bucket,
-          :gcs_bucket,
-          :gcs_project_id,
-          :gcs_credentials,
-          :gcs_metrics_file_name,
-          :test_metric_file_name,
-          :run_type
+        attr_reader :gcs_config, :clickhouse_config
+        attr_accessor :run_type
+        attr_writer :extra_rspec_metadata_keys,
+          :skip_record_proc,
+          :test_retried_proc,
+          :custom_metrics_proc,
+          :logger
 
-        attr_writer :custom_keys_tags,
-          :custom_keys_fields
+        # rubocop:disable Style/TrivialAccessors -- allows documenting that setting config enables the export as well as document input class type
+
+        # Enable metrics export to gcs bucket by setting configuration object
+        #
+        # @param config [Config::GCS]
+        # @return [GCS]
+        def gcs_config=(config)
+          @gcs_config = config
+        end
+
+        # Enable metrics export to clickhouse by setting configuration object
+        #
+        # @param config [Config::ClickHouse]
+        # @return [ClickHouse]
+        def clickhouse_config=(config)
+          @clickhouse_config = config
+        end
+
+        # rubocop:enable Style/TrivialAccessors
+
+        # Extra rspec metadata keys to include in exported metrics
+        #
+        # @return [Array<Symbol>]
+        def extra_rspec_metadata_keys
+          @extra_rspec_metadata_keys ||= []
+        end
+
+        # A lambda that determines whether to skip recording a test result
+        #
+        # This is useful when you would want to skip initial failure when retrying specs is set up in a separate process
+        # and you want to avoid duplicate records
+        #
+        # @return [Proc]
+        def skip_record_proc
+          @skip_record_proc ||= ->(_example) { false }
+        end
+
+        # A lambda that determines whether a test was retried or not
+        #
+        # @return [Proc]
+        def test_retried_proc
+          @test_retried_proc ||= ->(_example) { false }
+        end
 
-        def custom_keys_tags
-          @custom_keys_tags || []
+        # A lambda that return hash with additional custom metrics
+        #
+        # @return [Proc]
+        def custom_metrics_proc
+          @custom_metrics_proc ||= ->(_example) { {} }
         end
 
-        def custom_keys_fields
-          @custom_keys_fields || []
+        # Logger instance
+        #
+        # @return [Logger]
+        def logger
+          @logger ||= Logger.new($stdout, level: Logger::INFO)
         end
       end
     end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb

@@ -1,56 +1,93 @@
 # frozen_string_literal: true
 
+require "rspec/core/formatters/base_formatter"
+
+# rubocop:disable Metrics/AbcSize
 module GitlabQuality
   module TestTooling
     module TestMetricsExporter
       class Formatter < RSpec::Core::Formatters::BaseFormatter
         RSpec::Core::Formatters.register(self, :stop)
 
+        LOG_PREFIX = "[MetricsExporter]"
+
         def stop(notification)
-          setup_test_metrics_exporter(notification.examples)
-
-          log_test_metrics.push_test_metrics(
-            custom_keys_tags: config.custom_keys_tags,
-            custom_keys_fields: config.custom_keys_fields
-          )
-
-          log_test_metrics.save_test_metrics(
-            file_name: config.test_metric_file_name,
-            custom_keys_tags: config.custom_keys_tags,
-            custom_keys_fields: config.custom_keys_fields
-          )
+          data = notification.examples.filter_map do |example|
+            next if config.skip_record_proc.call(example)
+
+            TestMetrics.new(example, time).data
+          end
+          return logger.warn("#{LOG_PREFIX} No test execution records found, metrics will not be exported!") if data.empty?
+
+          push_to_gcs(data)
+          push_to_clickhouse(data)
         end
 
         private
 
-        attr_reader :log_test_metrics
-
+        # Configuration instance
+        #
+        # @return [Config]
         def config
           Config.configuration
         end
 
-        # rubocop:disable Metrics/AbcSize
-        def setup_test_metrics_exporter(examples)
-          @log_test_metrics = LogTestMetrics.new(
-            examples: examples,
-            run_type: config.run_type
-          )
-
-          @log_test_metrics.configure_influxdb_client(
-            influxdb_url: config.influxdb_url,
-            influxdb_token: config.influxdb_token,
-            influxdb_bucket: config.influxdb_bucket
-          )
-
-          @log_test_metrics.configure_gcs_client(
-            gcs_bucket: config.gcs_bucket,
-            gcs_project_id: config.gcs_project_id,
-            gcs_credentials: config.gcs_credentials,
-            gcs_metrics_file_name: config.gcs_metrics_file_name
-          )
+        # Logger instance
+        #
+        # @return [Logger]
+        def logger
+          config.logger
+        end
+
+        # Single common timestamp for all exported example metrics to keep data points consistently grouped
+        #
+        # @return [Time]
+        def time
+          return @time if @time
+
+          ci_created_at = Runtime::Env.ci_pipeline_created_at
+          @time = ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc
+        end
+
+        # Push data to gcs
+        #
+        # @param data [Array]
+        # @return [void]
+        def push_to_gcs(data)
+          return logger.debug("#{LOG_PREFIX} GCS configuration missing, skipping gcs export!") unless config.gcs_config
+
+          gcs_config = config.gcs_config
+          GcsTools.gcs_client(
+            project_id: gcs_config.project_id,
+            credentials: gcs_config.credentials,
+            dry_run: gcs_config.dry_run
+          ).put_object(gcs_config.bucket_name, gcs_config.metrics_file_name, data.to_json)
+          logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to GCS bucket!")
+        rescue StandardError => e
+          logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to GCS: #{e.message}")
+        end
+
+        # Push data to clickhouse
+        #
+        # @param data [Array<Hash>]
+        # @return [void]
+        def push_to_clickhouse(data)
+          return logger.debug("ClickHouse configuration missing, skipping ClickHouse export!") unless config.clickhouse_config
+
+          clickhouse_config = config.clickhouse_config
+          ClickHouse::Client.new(
+            url: clickhouse_config.url,
+            database: clickhouse_config.database,
+            username: clickhouse_config.username,
+            password: clickhouse_config.password,
+            logger: logger
+          ).insert_json_data(clickhouse_config.table_name, data)
+          logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to ClickHouse!")
+        rescue StandardError => e
+          logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to ClickHouse: #{e.message}")
         end
-        # rubocop:enable Metrics/AbcSize
       end
     end
   end
 end
+# rubocop:enable Metrics/AbcSize

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb

@@ -5,131 +5,138 @@ require 'time'
 module GitlabQuality
   module TestTooling
     module TestMetricsExporter
-      module TestMetrics
-        # Single common timestamp for all exported example metrics to keep data points consistently grouped
-        #
-        # @return [Time]
-        def time
-          return @time if defined?(@time)
-
-          created_at = Time.strptime(env('CI_PIPELINE_CREATED_AT'), '%Y-%m-%dT%H:%M:%S%z') if env('CI_PIPELINE_CREATED_AT')
-          @time = Time.parse((created_at || Time.now).utc.strftime('%Y-%m-%d %H:%M:%S %z'))
+      class TestMetrics
+        def initialize(example, timestamp)
+          @example = example
+          @timestamp = timestamp
         end
 
-        # rubocop:disable Metrics/AbcSize
-        # Metrics tags
+        # Test data hash
         #
-        # @param [RSpec::Core::Example] example
-        # @param [Array<String>] custom_keys
-        # @param [String]
         # @return [Hash]
-        def tags(example, custom_keys, run_type)
+        def data
           {
-            name: example.full_description,
-            file_path: example.metadata[:file_path].sub(/\A./, ''),
-            status: status(example),
-            quarantined: quarantined(example),
-            job_name: job_name,
-            merge_request: merge_request,
-            run_type: run_type,
-            feature_category: example.metadata[:feature_category],
-            product_group: example.metadata[:product_group],
-            exception_class: example.execution_result.exception&.class&.to_s,
-            **custom_metrics(example.metadata, custom_keys)
+            time: timestamp,
+            **rspec_metrics,
+            **ci_metrics,
+            **custom_metrics
           }.compact
         end
-        # rubocop:enable Metrics/AbcSize
 
-        # Metrics fields
+        private
+
+        attr_reader :example, :timestamp
+
+        # Exporter configuration
+        #
+        # @return [Config]
+        def config
+          Config.configuration
+        end
+
+        # Rspec related metrics
         #
-        # @param [RSpec::Core::Example] example
-        # @param [Array<String>] custom_keys
         # @return [Hash]
-        def fields(example, custom_keys)
+        def rspec_metrics # rubocop:disable Metrics/AbcSize
           {
             id: example.id,
+            name: example.full_description,
+            file_path: example.metadata[:file_path].sub(/\A./, ''),
+            status: example.execution_result.status,
             run_time: (example.execution_result.run_time * 1000).round,
-            job_url: Runtime::Env.ci_job_url,
-            pipeline_url: env('CI_PIPELINE_URL'),
-            pipeline_id: env('CI_PIPELINE_ID'),
-            job_id: env('CI_JOB_ID'),
-            merge_request_iid: merge_request_iid,
-            failure_exception: example.execution_result.exception.to_s.delete("\n"),
-            **custom_metrics(example.metadata, custom_keys)
-          }.compact
+            location: example_location,
+            exception_class: exception_class,
+            failure_exception: failure_exception,
+            quarantined: quarantined?,
+            test_retried: config.test_retried_proc.call(example)
+          }
         end
 
-        # Return a more detailed status
-        #
-        # - if test is failed or pending, return rspec status
-        # - if test passed but had more than 1 attempt, consider test flaky
+        # CI related metrics
         #
-        # @param [RSpec::Core::Example] example
-        # @return [Symbol]
-        def status(example)
-          rspec_status = example.execution_result.status
-          return rspec_status if [:pending, :failed].include?(rspec_status)
-
-          retry_attempts(example.metadata).positive? ? :flaky : :passed
+        # @return [Hash]
+        def ci_metrics
+          {
+            ci_project_id: env("CI_PROJECT_ID")&.to_i,
+            ci_project_path: env("CI_PROJECT_PATH"),
+            ci_job_name: ci_job_name,
+            ci_job_id: env('CI_JOB_ID')&.to_i,
+            ci_pipeline_id: env('CI_PIPELINE_ID')&.to_i,
+            ci_merge_request_iid: (env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID'))&.to_i,
+            ci_branch: env("CI_COMMIT_REF_NAME"),
+            ci_target_branch: env("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
+          }
         end
 
-        # Retry attempts
+        # Additional custom metrics
         #
-        # @param [Hash] example
-        # @return [Integer]
-        def retry_attempts(metadata)
-          metadata[:retry_attempts] || 0
+        # @return [Hash]
+        def custom_metrics
+          metrics = example.metadata
+            .slice(*config.extra_rspec_metadata_keys)
+            .merge(config.custom_metrics_proc.call(example))
+
+          metrics.each_with_object({}) do |(k, value), custom_metrics|
+            custom_metrics[k.to_sym] = metrics_value(value)
+          end
         end
 
         # Checks if spec is quarantined
         #
-        # @param [RSpec::Core::Example] example
         # @return [String]
-        def quarantined(example)
-          return "false" unless example.metadata.key?(:quarantine)
+        def quarantined?
+          return false unless example.metadata.key?(:quarantine)
 
           # if quarantine key is present and status is pending, consider it quarantined
-          (example.execution_result.status == :pending).to_s
+          example.execution_result.status == :pending
         end
 
         # Base ci job name
         #
         # @return [String]
-        def job_name
-          @job_name ||= Runtime::Env.ci_job_name&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+        def ci_job_name
+          env("CI_JOB_NAME")&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
         end
 
-        # Check if it is a merge request execution
+        # Example location
         #
         # @return [String]
-        def merge_request
-          (!!merge_request_iid).to_s
+        def example_location
+          # ensures that location will be correct even in case of shared examples
+          file = example
+                 .metadata
+                 .fetch(:shared_group_inclusion_backtrace)
+                 .last
+                 &.formatted_inclusion_location
+
+          return example.location unless file
+
+          file
         end
 
-        # Merge request iid
+        # Failure exception class
         #
         # @return [String]
-        def merge_request_iid
-          env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID')
+        def exception_class
+          example.execution_result.exception&.class&.to_s
         end
 
-        # Custom test metrics
+        # Truncated exception stacktrace
         #
-        # @param [Hash] metadata
-        # @param [Array] array of custom metrics keys
-        # @return [Hash]
-        def custom_metrics(metadata, custom_keys)
-          return {} if custom_keys.nil?
-
-          custom_metrics = {}
-          custom_keys.each do |k|
-            value = metadata[k.to_sym]
-            v = value.is_a?(Numeric) || value.nil? ? value : value.to_s
+        # @return [String]
+        def failure_exception
+          example.execution_result.exception.then do |exception|
+            next unless exception
 
-            custom_metrics[k.to_sym] = v
+            exception.to_s.tr("\n", " ").slice(0, 1000)
           end
+        end
 
-          custom_metrics
+        # Test run type | suite name
+        #
+        # @return [String]
+        def run_type
+          config.run_type || ci_job_name || "unknown"
         end
 
         # Return non empty environment variable value
@@ -141,6 +148,24 @@ module GitlabQuality
 
           ENV.fetch(name)
         end
+
+        # Metrics value cast to a valid type
+        #
+        # @param value [Object]
+        # @return [Object]
+        def metrics_value(value)
+          return value if value.is_a?(Numeric) || value.is_a?(String) || bool?(value) || value.nil?
+
+          value.to_s
+        end
+
+        # Value is a true or false
+        #
+        # @param val [Object]
+        # @return [Boolean]
+        def bool?(val)
+          [true, false].include?(val)
+        end
       end
     end
   end