gitlab_quality-test_tooling 2.16.0 → 3.0.0

data/lib/gitlab_quality/test_tooling/code_coverage/category_owners.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'yaml'
+
+require_relative 'utils'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class CategoryOwners
+        include Utils
+
+        SOURCE_URL  = URI('https://gitlab.com/gitlab-com/www-gitlab-com/raw/master/data/stages.yml')
+        BASE_DELAY  = 1 # seconds
+        MAX_RETRIES = 3
+
+        # @return [Hash] Category ownership hierarchy, section -> stage -> group -> [categories]
+        # @example Return value
+        #   {
+        #     "team_planning" => {             # section
+        #       "project_management" => {      # stage
+        #         "plan" => [                  # group
+        #           "dev",                     # category
+        #           "service_desk"             # category
+        #         ],
+        #         "product_planning" => [      # group
+        #           "portfolio_management",    # category
+        #           ...
+        #         ]
+        #       }
+        #     },
+        #     ...
+        #   }
+        attr_reader :hierarchy
+
+        def initialize
+          @categories_map = {}
+          @hierarchy = {}
+
+          yaml_file = fetch_yaml_file
+          yaml_content = YAML.load(yaml_file)
+          populate_ownership_hierarchy(yaml_content)
+        end
+
+        # @return [Array<Hash>] Flattened category ownership
+        # @example Return value
+        #   [
+        #     { category: "team_planning", group: "project_management", stage: "plan", section: "dev" },
+        #     { category: "service_desk", group: "project_management", stage: "plan", section: "dev" },
+        #     { category: "portfolio_management", group: "product_planning", stage: "plan", section: "dev" }
+        #     ...
+        #   ]
+        def as_db_table
+          hierarchy.each_with_object([]) do |(section, stages), flattened_hierarchy|
+            next unless stages
+
+            stages.each do |stage, groups|
+              next unless groups
+
+              groups.each do |group, categories|
+                Array(categories).each do |category|
+                  flattened_hierarchy << {
+                    category: category,
+                    group: group,
+                    stage: stage,
+                    section: section
+                  }
+                end
+              end
+            end
+          end
+        end
+
+        # @return [Hash] Mapping of categories to teams (i.e., groups, stages, sections)
+        # @example Return value
+        #   {
+        #     "team_planning" => { group: "project_management", stage: "plan", section: "dev" },
+        #     "service_desk" => { group: "project_management", stage: "plan", section: "dev" },
+        #     "portfolio_management" => { group: "product_planning", stage: "plan", section: "dev" },
+        #     ...
+        #   }
+        def categories_to_teams
+          populate_categories_map(hierarchy)
+          @categories_map
+        end
+
+        private
+
+        def fetch_yaml_file
+          retries = 0
+
+          begin
+            response = Net::HTTP.get_response(SOURCE_URL)
+            raise "Failed to fetch file: HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+            response.body
+          rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED, Errno::ETIMEDOUT, IOError, RuntimeError => e
+            retries += 1
+            raise "Failed to fetch YAML after #{MAX_RETRIES} retries: #{e.message}" if retries >= MAX_RETRIES
+
+            delay = exponential_delay_with_jitter(retries)
+            warn "Fetch attempt #{retries} failed: #{e.message}. Retrying in #{delay.round(2)}s..."
+            sleep(delay)
+            retry
+          end
+        end
+
+        def populate_ownership_hierarchy(data)
+          stages = data['stages'] || {}
+
+          stages.each do |stage, stage_data|
+            next unless stage_data.is_a?(Hash)
+
+            section = stage_data['section']
+            groups = stage_data['groups'] || {}
+            next unless section
+
+            groups.each { |group, group_data| add_hierarchy_entry(section, stage, group, group_data['categories']) }
+          end
+        end
+
+        def add_hierarchy_entry(section, stage, group, categories)
+          @hierarchy[section] ||= {}
+          @hierarchy[section][stage] ||= {}
+          @hierarchy[section][stage][group] = categories || []
+        end
+
+        def populate_categories_map(data, current_section = nil, current_stage = nil, current_group = nil)
+          case data
+          when Hash # Sections / Stages / Groups
+            data.each do |key, value|
+              if current_section.nil? # Sections
+                populate_categories_map(value, key, nil, nil)
+              elsif current_stage.nil? # Stages
+                populate_categories_map(value, current_section, key, nil)
+              elsif current_group.nil? # Groups
+                populate_categories_map(value, current_section, current_stage, key)
+              else                     # Categories
+                populate_categories_map(value, current_section, current_stage, current_group)
+              end
+            end
+          when Array # Categories array
+            data.each do |category|
+              next unless category.is_a?(String)
+
+              @categories_map[category] = {
+                section: current_section,
+                stage: current_stage,
+                group: current_group
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class CategoryOwnersTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
+          TABLE_NAME = "category_owners"
+
+          MissingMappingError = Class.new(StandardError)
+
+          # Creates the ClickHouse table, if it doesn't exist already
+          # @return [nil]
+          def create
+            logger.debug("#{LOG_PREFIX} Creating category_owners table if it doesn't exist ...")
+
+            client.query(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{table_name} (
+                timestamp DateTime64(6, 'UTC') DEFAULT now64(),
+                category String,
+                group String,
+                stage String,
+                section String,
+                INDEX idx_group group TYPE set(360) GRANULARITY 1,
+                INDEX idx_stage stage TYPE set(360) GRANULARITY 1,
+                INDEX idx_section section TYPE set(360) GRANULARITY 1
+              ) ENGINE = MergeTree()
+              ORDER BY (category, timestamp)
+              SETTINGS index_granularity = 8192;
+            SQL
+
+            logger.info("#{LOG_PREFIX} Category owners table created/verified successfully")
+          end
+
+          def truncate
+            logger.debug("#{LOG_PREFIX} Truncating table #{full_table_name} ...")
+
+            client.query("TRUNCATE TABLE #{full_table_name}")
+
+            logger.info("#{LOG_PREFIX} Successfully truncated table #{full_table_name}")
+          end
+
+          # Owners of particular category as group, stage and section
+          #
+          # @param category_name [String]
+          # @return [Hash]
+          def owners(category_name)
+            records.fetch(category_name)
+          rescue KeyError
+            raise(MissingMappingError, "Category '#{category_name}' not found in table '#{table_name}'")
+          end
+
+          private
+
+          def records
+            @records ||= client
+              .query("SELECT category, group, stage, section FROM #{table_name}")
+              .each_with_object({}) { |record, hsh| hsh[record["category"]] = record.slice("group", "stage", "section") }
+          end
+
+          # @return [Boolean] True if the record is valid, false otherwise
+          def valid_record?(record)
+            required_fields = %i[category group stage section]
+
+            required_fields.each do |field|
+              if record[field].nil?
+                logger.warn("#{LOG_PREFIX} Skipping record with nil #{field}: #{record}")
+                return false
+              end
+            end
+
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class CoverageMetricsTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
+          TABLE_NAME = "coverage_metrics"
+
+          # Creates the ClickHouse table, if it doesn't exist already
+          # @return [nil]
+          def create
+            logger.debug("#{LOG_PREFIX} Creating coverage_metrics table if it doesn't exist ...")
+
+            client.query(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{table_name} (
+                timestamp DateTime64(6, 'UTC'),
+                file String,
+                line_coverage Float64,
+                branch_coverage Nullable(Float64),
+                function_coverage Nullable(Float64),
+                source_file_type String,
+                category Nullable(String),
+                ci_project_id Nullable(UInt32),
+                ci_project_path Nullable(String),
+                ci_job_name Nullable(String),
+                ci_job_id Nullable(UInt64),
+                ci_pipeline_id Nullable(UInt64),
+                ci_merge_request_iid Nullable(UInt32),
+                ci_branch Nullable(String),
+                ci_target_branch Nullable(String)
+              ) ENGINE = MergeTree()
+              PARTITION BY toYYYYMM(timestamp)
+              ORDER BY (ci_project_path, timestamp, file, ci_pipeline_id)
+              SETTINGS index_granularity = 8192, allow_nullable_key = 1;
+            SQL
+
+            logger.info("#{LOG_PREFIX} Coverage metrics table created/verified successfully")
+          end
+
+          private
+
+          # @return [Boolean] True if the record is valid, false otherwise
+          def valid_record?(record)
+            valid_file?(record) &&
+              valid_line_coverage?(record) &&
+              valid_branch_coverage?(record) &&
+              valid_function_coverage?(record)
+          end
+
+          # @return [Boolean] True if the file field is present
+          def valid_file?(record)
+            return true unless record[:file].nil?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with nil file: #{record}")
+            false
+          end
+
+          # @return [Boolean] True if line_coverage is present and not NaN
+          def valid_line_coverage?(record)
+            if record[:line_coverage].nil?
+              logger.warn("#{LOG_PREFIX} Skipping record with nil line_coverage: #{record}")
+              return false
+            end
+
+            return true unless record[:line_coverage].nan?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with NaN line_coverage: #{record}")
+            false
+          end
+
+          # @return [Boolean] True if branch_coverage is not NaN (or is nil)
+          def valid_branch_coverage?(record)
+            return true unless record[:branch_coverage]&.nan?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with NaN branch_coverage: #{record}")
+            false
+          end
+
+          # @return [Boolean] True if function_coverage is not NaN (or is nil)
+          def valid_function_coverage?(record)
+            return true unless record[:function_coverage]&.nan?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with NaN function_coverage: #{record}")
+            false
+          end
+
+          # @return [Hash] Transformed coverage data including timestamp and CI metadata
+          def sanitized_data_record(record)
+            {
+              timestamp: time,
+              file: record[:file],
+              line_coverage: record[:line_coverage],
+              branch_coverage: record[:branch_coverage],
+              function_coverage: record[:function_coverage],
+              source_file_type: record[:source_file_type],
+              category: record[:category],
+              **ci_metadata
+            }
+          end
+
+          # @return [Time] Common timestamp for all coverage records
+          def time
+            @time ||= begin
+              ci_created_at = ENV.fetch('CI_PIPELINE_CREATED_AT', nil)
+              ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc
+            end
+          end
+
+          # @return [Hash] CI-related metadata
+          def ci_metadata
+            {
+              ci_project_id: env_to_int('CI_PROJECT_ID'),
+              ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil),
+              ci_job_name: ENV.fetch('CI_JOB_NAME', nil)&.gsub(%r{ \d{1,2}/\d{1,2}}, ''),
+              ci_job_id: env_to_int('CI_JOB_ID'),
+              ci_pipeline_id: env_to_int('CI_PIPELINE_ID'),
+              ci_merge_request_iid: env_to_int('CI_MERGE_REQUEST_IID') || env_to_int('TOP_UPSTREAM_MERGE_REQUEST_IID'),
+              ci_branch: ENV.fetch('CI_COMMIT_REF_NAME', nil),
+              ci_target_branch: ENV.fetch('CI_MERGE_REQUEST_TARGET_BRANCH_NAME', nil)
+            }
+          end
+
+          # @param name [String] Environment variable name
+          # @return [Integer, nil] Environment variable converted to integer or
+          #   nil if not present or empty
+          def env_to_int(name)
+            value = ENV.fetch(name, nil)
+            return nil if value.nil? || value.empty?
+
+            value.to_i
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class Table
+          LOG_PREFIX = "[CodeCoverage]"
+
+          def initialize(url:, database:, username: nil, password: nil, logger: nil)
+            @url = url
+            @database = database
+            @username = username
+            @password = password
+            @logger = logger || Runtime::Logger.logger
+          end
+
+          # @param data [Array<Hash>] Code coverage related data to be pushed to ClickHouse
+          # @return [nil]
+          def push(data) # rubocop:disable Metrics/AbcSize
+            return logger.warn("#{LOG_PREFIX} No data found, skipping ClickHouse export!") if data.empty?
+
+            logger.debug("#{LOG_PREFIX} Starting data export to ClickHouse")
+            sanitized_data = sanitize(data)
+
+            return logger.warn("#{LOG_PREFIX} No valid data found after sanitization, skipping ClickHouse export!") if sanitized_data.empty?
+
+            client.insert_json_data(table_name, sanitized_data)
+            logger.info("#{LOG_PREFIX} Successfully pushed #{sanitized_data.size} records to #{full_table_name}!")
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Error occurred while pushing data to #{full_table_name}: #{e.message}")
+            raise
+          end
+
+          private
+
+          attr_reader :url, :database, :username, :password, :logger
+
+          def sanitize(data)
+            data.filter_map { |record| sanitized_data_record(record) if valid_record?(record) }
+          end
+
+          def sanitized_data_record(record)
+            record
+          end
+
+          def full_table_name
+            "#{database}.#{table_name}"
+          end
+
+          def table_name
+            self.class::TABLE_NAME
+          rescue NameError
+            raise NotImplementedError, "#{self.class} must define the TABLE_NAME constant"
+          end
+
+          def valid_record?(_record)
+            raise NotImplementedError, "#{self.class}##{__method__} method must be implemented in a subclass"
+          end
+
+          # @return [GitlabQuality::TestTooling::ClickHouse::Client]
+          def client
+            @client ||= GitlabQuality::TestTooling::ClickHouse::Client.new(
+              url: url,
+              database: database,
+              username: username,
+              password: password,
+              logger: logger
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class CoverageData
+        # @param [Hash<String, Hash>] code_coverage_by_source_file Source file
+        #   mapped to test coverage data
+        # @param [Hash<String, Array<String>>] source_file_to_tests Source files
+        #   mapped to all test files testing them
+        # @param [Hash<String, Array<String>>] tests_to_categories Test files
+        #   mapped to all feature categories they belong to
+        # @param [Hash<String, Hash>] categories_to_teams Mapping of categories
+        #   to teams (i.e., groups, stages, sections)
+        # @param [Hash<String, String>] source_file_types Mapping of source files
+        #   to their types (frontend, backend, etc.)
+        def initialize(code_coverage_by_source_file, source_file_to_tests, tests_to_categories, categories_to_teams, source_file_types = {})
+          @code_coverage_by_source_file = code_coverage_by_source_file
+          @source_file_to_tests = source_file_to_tests
+          @tests_to_categories = tests_to_categories
+          @categories_to_teams = categories_to_teams
+          @source_file_types = source_file_types
+        end
+
+        # @return [Array<Hash<Symbol, String>>] Mapping of column name to row
+        #   value
+        # @example Return value
+        #   [
+        #     {
+        #       file: "app/channels/application_cable/channel.rb"
+        #       line_coverage: 100.0
+        #       branch_coverage: 95.0
+        #       function_coverage: 100.0
+        #       source_file_type: "backend"
+        #       category: "team_planning"
+        #       group: "project_management"
+        #       stage: "plan"
+        #       section: "dev"
+        #     },
+        #     ...
+        #   ]
+        def as_db_table
+          all_files.flat_map do |file|
+            coverage_data = @code_coverage_by_source_file[file]
+            line_coverage = coverage_data&.dig(:percentage)
+            branch_coverage = coverage_data&.dig(:branch_percentage)
+            function_coverage = coverage_data&.dig(:function_percentage)
+
+            categories = categories_for(file)
+            base_data = {
+              file: file,
+              line_coverage: line_coverage,
+              branch_coverage: branch_coverage,
+              function_coverage: function_coverage,
+              source_file_type: @source_file_types[file] || 'other'
+            }
+
+            if categories.empty?
+              base_data.merge(no_owner_info)
+            else
+              categories.map do |category|
+                base_data.merge(owner_info(category))
+              end
+            end
+          end
+        end
+
+        private
+
+        def no_owner_info
+          {
+            category: nil,
+            group: nil,
+            stage: nil,
+            section: nil
+          }
+        end
+
+        def owner_info(category)
+          owner_info = @categories_to_teams[category]
+
+          {
+            category: category,
+            group: owner_info&.dig(:group),
+            stage: owner_info&.dig(:stage),
+            section: owner_info&.dig(:section)
+          }
+        end
+
+        def categories_for(file)
+          @source_file_to_tests[file]&.flat_map { |test_file| @tests_to_categories[test_file] || [] }&.uniq || []
+        end
+
+        def all_files
+          @all_files ||= @code_coverage_by_source_file.keys
+        end
+      end
+    end
+  end
+end