gitlab_quality-test_tooling 2.16.0 → 2.25.1

data/lib/gitlab_quality/test_tooling/code_coverage/category_owners.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'yaml'
+
+require_relative 'utils'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class CategoryOwners
+        include Utils
+
+        SOURCE_URL  = URI('https://gitlab.com/gitlab-com/www-gitlab-com/raw/master/data/stages.yml')
+        BASE_DELAY  = 1 # seconds
+        MAX_RETRIES = 3
+
+        # @return [Hash] Category ownership hierarchy, section -> stage -> group -> [categories]
+        # @example Return value
+        #   {
+        #     "team_planning" => {             # section
+        #       "project_management" => {      # stage
+        #         "plan" => [                  # group
+        #           "dev",                     # category
+        #           "service_desk"             # category
+        #         ],
+        #         "product_planning" => [      # group
+        #           "portfolio_management",    # category
+        #           ...
+        #         ]
+        #       }
+        #     },
+        #     ...
+        #   }
+        attr_reader :hierarchy
+
+        def initialize
+          @categories_map = {}
+          @hierarchy = {}
+
+          yaml_file = fetch_yaml_file
+          yaml_content = YAML.load(yaml_file)
+          populate_ownership_hierarchy(yaml_content)
+        end
+
+        # @return [Array<Hash>] Flattened category ownership
+        # @example Return value
+        #   [
+        #     { category: "team_planning", group: "project_management", stage: "plan", section: "dev" },
+        #     { category: "service_desk", group: "project_management", stage: "plan", section: "dev" },
+        #     { category: "portfolio_management", group: "product_planning", stage: "plan", section: "dev" }
+        #     ...
+        #   ]
+        def as_db_table
+          hierarchy.each_with_object([]) do |(section, stages), flattened_hierarchy|
+            next unless stages
+
+            stages.each do |stage, groups|
+              next unless groups
+
+              groups.each do |group, categories|
+                Array(categories).each do |category|
+                  flattened_hierarchy << {
+                    category: category,
+                    group: group,
+                    stage: stage,
+                    section: section
+                  }
+                end
+              end
+            end
+          end
+        end
+
+        # @return [Hash] Mapping of categories to teams (i.e., groups, stages, sections)
+        # @example Return value
+        #   {
+        #     "team_planning" => { group: "project_management", stage: "plan", section: "dev" },
+        #     "service_desk" => { group: "project_management", stage: "plan", section: "dev" },
+        #     "portfolio_management" => { group: "product_planning", stage: "plan", section: "dev" },
+        #     ...
+        #   }
+        def categories_to_teams
+          populate_categories_map(hierarchy)
+          @categories_map
+        end
+
+        private
+
+        def fetch_yaml_file
+          retries = 0
+
+          begin
+            response = Net::HTTP.get_response(SOURCE_URL)
+            raise "Failed to fetch file: HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+            response.body
+          rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED, Errno::ETIMEDOUT, IOError, RuntimeError => e
+            retries += 1
+            raise "Failed to fetch YAML after #{MAX_RETRIES} retries: #{e.message}" if retries >= MAX_RETRIES
+
+            delay = exponential_delay_with_jitter(retries)
+            warn "Fetch attempt #{retries} failed: #{e.message}. Retrying in #{delay.round(2)}s..."
+            sleep(delay)
+            retry
+          end
+        end
+
+        def populate_ownership_hierarchy(data)
+          stages = data['stages'] || {}
+
+          stages.each do |stage, stage_data|
+            next unless stage_data.is_a?(Hash)
+
+            section = stage_data['section']
+            groups = stage_data['groups'] || {}
+            next unless section
+
+            groups.each { |group, group_data| add_hierarchy_entry(section, stage, group, group_data['categories']) }
+          end
+        end
+
+        def add_hierarchy_entry(section, stage, group, categories)
+          @hierarchy[section] ||= {}
+          @hierarchy[section][stage] ||= {}
+          @hierarchy[section][stage][group] = categories || []
+        end
+
+        def populate_categories_map(data, current_section = nil, current_stage = nil, current_group = nil)
+          case data
+          when Hash # Sections / Stages / Groups
+            data.each do |key, value|
+              if current_section.nil? # Sections
+                populate_categories_map(value, key, nil, nil)
+              elsif current_stage.nil? # Stages
+                populate_categories_map(value, current_section, key, nil)
+              elsif current_group.nil? # Groups
+                populate_categories_map(value, current_section, current_stage, key)
+              else                     # Categories
+                populate_categories_map(value, current_section, current_stage, current_group)
+              end
+            end
+          when Array # Categories array
+            data.each do |category|
+              next unless category.is_a?(String)
+
+              @categories_map[category] = {
+                section: current_section,
+                stage: current_stage,
+                group: current_group
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class CategoryOwnersTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
+          TABLE_NAME = "category_owners"
+
+          # Creates the ClickHouse table, if it doesn't exist already
+          # @return [nil]
+          def create
+            logger.debug("#{LOG_PREFIX} Creating category_owners table if it doesn't exist ...")
+
+            client.query(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{table_name} (
+                timestamp DateTime64(6, 'UTC') DEFAULT now64(),
+                category String,
+                group String,
+                stage String,
+                section String,
+                INDEX idx_group group TYPE set(360) GRANULARITY 1,
+                INDEX idx_stage stage TYPE set(360) GRANULARITY 1,
+                INDEX idx_section section TYPE set(360) GRANULARITY 1
+              ) ENGINE = MergeTree()
+              ORDER BY (category, timestamp)
+              SETTINGS index_granularity = 8192;
+            SQL
+
+            logger.info("#{LOG_PREFIX} Category owners table created/verified successfully")
+          end
+
+          def truncate
+            logger.debug("#{LOG_PREFIX} Truncating table #{full_table_name} ...")
+
+            client.query("TRUNCATE TABLE #{full_table_name}")
+
+            logger.info("#{LOG_PREFIX} Successfully truncated table #{full_table_name}")
+          end
+
+          private
+
+          # @return [Boolean] True if the record is valid, false otherwise
+          def valid_record?(record)
+            required_fields = %i[category group stage section]
+
+            required_fields.each do |field|
+              if record[field].nil?
+                logger.warn("#{LOG_PREFIX} Skipping record with nil #{field}: #{record}")
+                return false
+              end
+            end
+
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class CoverageMetricsTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
+          TABLE_NAME = "coverage_metrics"
+
+          # Creates the ClickHouse table, if it doesn't exist already
+          # @return [nil]
+          def create
+            logger.debug("#{LOG_PREFIX} Creating coverage_metrics table if it doesn't exist ...")
+
+            client.query(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{table_name} (
+                timestamp DateTime64(6, 'UTC'),
+                file String,
+                coverage Float64,
+                category Nullable(String),
+                ci_project_id Nullable(UInt32),
+                ci_project_path Nullable(String),
+                ci_job_name Nullable(String),
+                ci_job_id Nullable(UInt64),
+                ci_pipeline_id Nullable(UInt64),
+                ci_merge_request_iid Nullable(UInt32),
+                ci_branch Nullable(String),
+                ci_target_branch Nullable(String)
+              ) ENGINE = MergeTree()
+              PARTITION BY toYYYYMM(timestamp)
+              ORDER BY (ci_project_path, timestamp, file, ci_pipeline_id)
+              SETTINGS index_granularity = 8192, allow_nullable_key = 1;
+            SQL
+
+            logger.info("#{LOG_PREFIX} Coverage metrics table created/verified successfully")
+          end
+
+          private
+
+          # @return [Boolean] True if the record is valid, false otherwise
+          def valid_record?(record)
+            if record[:file].nil?
+              logger.warn("#{LOG_PREFIX} Skipping record with nil file: #{record}")
+              return false
+            end
+
+            if record[:coverage].nil?
+              logger.warn("#{LOG_PREFIX} Skipping record with nil coverage: #{record}")
+              return false
+            end
+
+            if record[:coverage].nan?
+              logger.warn("#{LOG_PREFIX} Skipping record with NaN coverage: #{record}")
+              return false
+            end
+
+            true
+          end
+
+          # @return [Hash] Transformed coverage data including timestamp and CI metadata
+          def sanitized_data_record(record)
+            {
+              timestamp: time,
+              file: record[:file],
+              coverage: record[:coverage],
+              category: record[:category],
+              **ci_metadata
+            }
+          end
+
+          # @return [Time] Common timestamp for all coverage records
+          def time
+            @time ||= begin
+              ci_created_at = ENV.fetch('CI_PIPELINE_CREATED_AT', nil)
+              ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc
+            end
+          end
+
+          # @return [Hash] CI-related metadata
+          def ci_metadata
+            {
+              ci_project_id: env_to_int('CI_PROJECT_ID'),
+              ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil),
+              ci_job_name: ENV.fetch('CI_JOB_NAME', nil)&.gsub(%r{ \d{1,2}/\d{1,2}}, ''),
+              ci_job_id: env_to_int('CI_JOB_ID'),
+              ci_pipeline_id: env_to_int('CI_PIPELINE_ID'),
+              ci_merge_request_iid: env_to_int('CI_MERGE_REQUEST_IID') || env_to_int('TOP_UPSTREAM_MERGE_REQUEST_IID'),
+              ci_branch: ENV.fetch('CI_COMMIT_REF_NAME', nil),
+              ci_target_branch: ENV.fetch('CI_MERGE_REQUEST_TARGET_BRANCH_NAME', nil)
+            }
+          end
+
+          # @param name [String] Environment variable name
+          # @return [Integer, nil] Environment variable converted to integer or
+          #   nil if not present or empty
+          def env_to_int(name)
+            value = ENV.fetch(name, nil)
+            return nil if value.nil? || value.empty?
+
+            value.to_i
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class Table
+          LOG_PREFIX = "[CodeCoverage]"
+
+          def initialize(url:, database:, username: nil, password: nil, logger: nil)
+            @url = url
+            @database = database
+            @username = username
+            @password = password
+            @logger = logger || Runtime::Logger.logger
+          end
+
+          # @param data [Array<Hash>] Code coverage related data to be pushed to ClickHouse
+          # @return [nil]
+          def push(data) # rubocop:disable Metrics/AbcSize
+            return logger.warn("#{LOG_PREFIX} No data found, skipping ClickHouse export!") if data.empty?
+
+            logger.debug("#{LOG_PREFIX} Starting data export to ClickHouse")
+            sanitized_data = sanitize(data)
+
+            client.insert_json_data(table_name, sanitized_data)
+            logger.info("#{LOG_PREFIX} Successfully pushed #{sanitized_data.size} records to #{full_table_name}!")
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Error occurred while pushing data to #{full_table_name}: #{e.message}")
+            raise
+          end
+
+          private
+
+          attr_reader :url, :database, :username, :password, :logger
+
+          def sanitize(data)
+            data.filter_map { |record| sanitized_data_record(record) if valid_record?(record) }
+          end
+
+          def sanitized_data_record(record)
+            record
+          end
+
+          def full_table_name
+            "#{database}.#{table_name}"
+          end
+
+          def table_name
+            self.class::TABLE_NAME
+          rescue NameError
+            raise NotImplementedError, "#{self.class} must define the TABLE_NAME constant"
+          end
+
+          def valid_record?(_record)
+            raise NotImplementedError, "#{self.class}##{__method__} method must be implemented in a subclass"
+          end
+
+          # @return [GitlabQuality::TestTooling::ClickHouse::Client]
+          def client
+            @client ||= GitlabQuality::TestTooling::ClickHouse::Client.new(
+              url: url,
+              database: database,
+              username: username,
+              password: password,
+              logger: logger
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class CoverageData
+        # @param [Hash<String, Hash>] code_coverage_by_source_file Source file
+        #   mapped to test coverage data
+        # @param [Hash<String, Array<String>>] source_file_to_tests Source files
+        #   mapped to all test files testing them
+        # @param [Hash<String, Array<String>>] tests_to_categories Test files
+        #   mapped to all feature categories they belong to
+        # @param [Hash<String, Hash>] categories_to_teams Mapping of categories
+        #   to teams (i.e., groups, stages, sections)
+        def initialize(code_coverage_by_source_file, source_file_to_tests, tests_to_categories, categories_to_teams)
+          @code_coverage_by_source_file = code_coverage_by_source_file
+          @source_file_to_tests = source_file_to_tests
+          @tests_to_categories = tests_to_categories
+          @categories_to_teams = categories_to_teams
+        end
+
+        # @return [Array<Hash<Symbol, String>>] Mapping of column name to row
+        #   value
+        # @example Return value
+        #   [
+        #     {
+        #       file: "app/channels/application_cable/channel.rb"
+        #       coverage: 100.0
+        #       category: "team_planning"
+        #       group: "project_management"
+        #       stage: "plan"
+        #       section: "dev"
+        #     },
+        #     ...
+        #   ]
+        def as_db_table
+          all_files.flat_map do |file|
+            coverage = @code_coverage_by_source_file[file]&.dig(:percentage)
+            categories = categories_for(file)
+            if categories.empty?
+              { file: file, coverage: coverage }.merge(no_owner_info)
+            else
+              categories.map do |category|
+                { file: file, coverage: coverage }.merge(owner_info(category))
+              end
+            end
+          end
+        end
+
+        private
+
+        def no_owner_info
+          {
+            category: nil,
+            group: nil,
+            stage: nil,
+            section: nil
+          }
+        end
+
+        def owner_info(category)
+          owner_info = @categories_to_teams[category]
+
+          {
+            category: category,
+            group: owner_info&.dig(:group),
+            stage: owner_info&.dig(:stage),
+            section: owner_info&.dig(:section)
+          }
+        end
+
+        def categories_for(file)
+          @source_file_to_tests[file]&.flat_map { |test_file| @tests_to_categories[test_file] || [] }&.uniq || []
+        end
+
+        def all_files
+          @all_files ||= @code_coverage_by_source_file.keys
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class LcovFile
+        # @param [String] lcov_file_content The content of the lcov file
+        def initialize(lcov_file_content)
+          @lcov_file_content = lcov_file_content
+        end
+
+        # @return [Hash<String, Hash>] The parsed content of the lcov file
+        # @example Return value
+        #   {
+        #     "path/to/file1.rb" => {
+        #       line_coverage: { 1 => 1, 2 => 0 },
+        #       branch_coverage: {},
+        #       total_lines: 2,
+        #       covered_lines: 1,
+        #       percentage: 50.0
+        #     },
+        #     ...
+        #  }
+        def parsed_content
+          return @parsed_content if @parsed_content
+
+          @parsed_content = {}
+          @current_file = nil
+
+          @lcov_file_content.each_line do |line|
+            case line
+            when /^SF:(.+)$/
+              register_source_file(::Regexp.last_match(1))
+            when /^DA:(\d+),(\d+)$/
+              register_line_data(::Regexp.last_match(1), ::Regexp.last_match(2))
+            when /^BRDA:(\d+),(\d+),(\d+),(-|\d+)$/
+              register_branch_data(::Regexp.last_match(1), ::Regexp.last_match(4))
+            when /^end_of_record$/
+              @current_file = nil
+            end
+          end
+
+          include_coverage
+          @parsed_content
+        end
+
+        private
+
+        def include_coverage
+          @parsed_content.each_key do |file|
+            # TODO: Support branch coverage too
+            @parsed_content[file].merge!(line_coverage_for(file))
+          end
+        end
+
+        def line_coverage_for(file)
+          data = @parsed_content[file]
+          return unless data
+
+          total_lines = data[:line_coverage].size
+          covered_lines = data[:line_coverage].values.count(&:positive?)
+
+          {
+            total_lines: total_lines,
+            covered_lines: covered_lines,
+            percentage: total_lines.zero? ? 0.0 : (covered_lines.to_f / total_lines * 100).round(2)
+          }
+        end
+
+        def register_source_file(filename)
+          @current_file = filename.gsub(%r{^\./}, '')
+          @parsed_content[@current_file] = { line_coverage: {}, branch_coverage: {} }
+        end
+
+        def register_line_data(line_no, count)
+          return unless @current_file
+
+          @parsed_content[@current_file][:line_coverage][line_no.to_i] = count.to_i
+        end
+
+        def register_branch_data(line_no, taken)
+          return unless @current_file
+
+          taken_count = taken == '-' ? 0 : taken.to_i
+          @parsed_content[@current_file][:branch_coverage][line_no.to_i] ||= []
+          @parsed_content[@current_file][:branch_coverage][line_no.to_i] << taken_count
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class RspecReport
+        # @param [Hash<String, Object>] rspec_report The content of an RSpec
+        #   report
+        def initialize(rspec_report)
+          @rspec_report = rspec_report
+        end
+
+        # @return [Array<Hash<String, String>>] Content of the "examples"
+        #   section of the RSpec report
+        def examples
+          @examples ||= @rspec_report['examples']
+        end
+
+        # @return [Hash<String, Array<String>>] Test files mapped to all feature
+        #   categories they belong to
+        # @example Return value
+        #   {
+        #     "spec/path/to/file_spec.rb" => [
+        #       "feature_category1", "feature_category2"
+        #     ],
+        #     ...
+        #   }
+        def tests_to_categories
+          @tests_to_categories ||= examples.to_a.filter_map do |example|
+            next unless example.is_a?(Hash)
+
+            file_path = example['file_path']
+            next unless file_path.is_a?(String)
+
+            [file_path.gsub('./', ''), Array(example['feature_category']).compact]
+          end.to_h
+        end
+      end
+    end
+  end
+end