gitlab_quality-test_tooling 2.22.0 → 2.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9bc3f79d2fd78c6d8fd1ee931c50f126e8aa2c91a0608b6e3675605fffa5dd5e
-  data.tar.gz: a80d4658c0781ec47d880b6dbf0166eb97f64742ce4e468ef86f9bc3e1e7615c
+  metadata.gz: 53c1f715f9a4375070f34f4e06697da5e71fd3bcbf052fbdb7367c52db0c8532
+  data.tar.gz: 95512423771bf43470d38d5867407f2b6850d46dbedfdb19314f4d65ef23858e
 SHA512:
-  metadata.gz: bb3d6fde7d83295c03059558525e6e4722416cd87dcf5e8a72c0b8f673dc384e08012d18aaed88877febd22cd400f039a24fb977ebbba985d0774e940b12f24e
-  data.tar.gz: 4cd6cee22b30f7cc1bd6c6291f55c0ba6026d4688b04c00b054cbcb3598054e8a5f696758b7e23adad1853ae9222802ebe22179baea96027be0b88a4598d5b5b
+  metadata.gz: 13ee76309735ad6662abaefc630c949be5b138abedf140c0e78096e8705faf9c148547d8d7ccb6059588c0bb805ae05a03dd8082a621510df09c7a44b39ed8e2
+  data.tar.gz: 8c9123729316f4e535ff958052b39e2f8e36e34bc8cb937a728cf87dd70954b3db7cb42f5e3c0fabcdd5131b534dd3bbab5bec35fbd73836570de1e4458c53c8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (2.22.0)
+    gitlab_quality-test_tooling (2.24.0)
       activesupport (>= 7.0, < 7.3)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/README.md

@@ -324,7 +324,7 @@ See an [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92580) fo
 
 [Automated gem release process](https://gitlab.com/gitlab-org/quality/pipeline-common#release-process) is used to release new version of `gitlab_quality-test_tooling` through pipelines, and this will:
 
-- Publish the gem: https://rubygems.org/gems/gitlab_quality-test_tooling
+- Publish the gem: https://rubygems.org/gems/gitlab_quality-test_tooling (once the version bump is done below in [Steps to release](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling#steps-to-release))
 - Add a release in the `gitlab_quality-test_tooling` project: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/releases
 - Populate the release log with the API contents. For example: https://gitlab.com/api/v4/projects/19861191/repository/changelog?version=3.4.4
 

data/exe/post-to-slack

@@ -77,6 +77,10 @@ options = OptionParser.new do |opts|
     params[:icon_emoji] = icon_emoji
   end
 
+  opts.on('-e', '--environment-issues-file FILE', String, 'Add environment issues alert based on grouped failure data in a file') do |file|
+    params[:environment_issues_file] = file
+  end
+
   opts.on_tail('-v', '--version', 'Show the version') do
     require_relative "../lib/gitlab_quality/test_tooling/version"
     puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"

data/exe/test-coverage

@@ -0,0 +1,123 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "optparse"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/category_owners'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/coverage_data'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/lcov_file'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/artifacts'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/rspec_report'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_map'
+
+params = {}
+required_params = [:working_dir, :rspec_reports_jobs, :rspec_reports_glob, :coverage_report_path, :test_map_url]
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('--working_dir PATH', 'Working directory path') do |path|
+    params[:working_dir] = path
+  end
+
+  opts.on('--rspec-reports-jobs JOB1,...', Array, 'Comma-separated names of RSpec JSON jobs') do |jobs|
+    params[:rspec_reports_jobs] = jobs
+  end
+
+  opts.on('--rspec-reports-glob GLOB', 'Glob pattern for RSpec JSON reports') do |pattern|
+    params[:rspec_reports_glob] = pattern
+  end
+
+  opts.on('--coverage-report-path PATH', 'Path to LCOV coverage report') do |path|
+    params[:coverage_report_path] = path
+  end
+
+  opts.on('--test-map-url URL', 'URL for the test map') do |url|
+    params[:test_map_url] = url
+  end
+
+  opts.on('-h', '--help', 'Show the usage') do
+    puts opts
+    puts "\nExamples:"
+    puts "  #{$PROGRAM_NAME}"
+    exit
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+if params.any? && (required_params - params.keys).none?
+  artifacts = GitlabQuality::TestTooling::CodeCoverage::Artifacts.new(
+    working_dir: params[:working_dir],
+    rspec_reports_jobs: params[:rspec_reports_jobs],
+    rspec_reports_glob: params[:rspec_reports_glob],
+    coverage_report_path: params[:coverage_report_path],
+    test_map_url: params[:test_map_url]
+  )
+
+  coverage_report = artifacts.coverage_report
+  rspec_reports = artifacts.rspec_reports
+  test_map = artifacts.test_map
+
+  code_coverage_by_source_file = GitlabQuality::TestTooling::CodeCoverage::LcovFile.new(coverage_report).parsed_content
+
+  source_file_to_tests = GitlabQuality::TestTooling::CodeCoverage::TestMap.new(test_map).source_to_tests
+
+  tests_to_categories = rspec_reports.reduce({}) do |combined_hash, rspec_report_file|
+    file_categories = GitlabQuality::TestTooling::CodeCoverage::RspecReport.new(rspec_report_file).tests_to_categories
+    combined_hash.merge(file_categories) { |_, old_val, new_val| (old_val + new_val).uniq }
+  end
+
+  category_owners = GitlabQuality::TestTooling::CodeCoverage::CategoryOwners.new
+
+  coverage_data = GitlabQuality::TestTooling::CodeCoverage::CoverageData.new(
+    code_coverage_by_source_file,
+    source_file_to_tests,
+    tests_to_categories,
+    category_owners.categories_to_teams
+  )
+
+  if ENV.fetch('CLICKHOUSE_URL', nil) &&
+      ENV.fetch('CLICKHOUSE_DATABASE', nil) &&
+      ENV.fetch('CLICKHOUSE_USERNAME', nil) &&
+      ENV.fetch('CLICKHOUSE_PASSWORD', nil)
+
+    clickhouse_data = {
+      url: ENV.fetch('CLICKHOUSE_URL', nil),
+      database: ENV.fetch('CLICKHOUSE_DATABASE', nil),
+      username: ENV.fetch('CLICKHOUSE_USERNAME', nil),
+      password: ENV.fetch('CLICKHOUSE_PASSWORD', nil)
+    }
+
+    category_owners_table  = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CategoryOwnersTable.new(**clickhouse_data)
+    coverage_metrics_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CoverageMetricsTable.new(**clickhouse_data)
+
+    category_owners_table.create  if ENV['CLICKHOUSE_CREATE_CATEGORY_OWNERS_TABLE']  == 'true'
+    coverage_metrics_table.create if ENV['CLICKHOUSE_CREATE_COVERAGE_METRICS_TABLE'] == 'true'
+
+    if ENV['CLICKHOUSE_PUSH_CATEGORY_DATA'] == 'true'
+      category_owners_table.truncate
+      category_owners_table.push(category_owners.as_db_table)
+    end
+
+    coverage_metrics_table.push(coverage_data.as_db_table)
+  else
+    puts "ClickHouse configuration not found.\n" \
+      'Set CLICKHOUSE_URL, CLICKHOUSE_DATABASE, CLICKHOUSE_USERNAME, ' \
+      'CLICKHOUSE_PASSWORD environment variables to enable ClickHouse export.'
+  end
+else
+  puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
+  puts options
+  exit 1
+end

data/lib/gitlab_quality/test_tooling/code_coverage/artifacts.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'stringio'
+require 'uri'
+require 'zlib'
+
+require_relative 'utils'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class Artifacts
+        include Utils
+
+        MAX_RETRIES = 7 # retries with exponential backoff: 1s, 2s, 4s, 8s, 16s, 32s, 64s (+ 0-1s jitter)
+
+        def initialize(working_dir:, coverage_report_path:, rspec_reports_jobs:, rspec_reports_glob:, test_map_url:)
+          @working_dir = working_dir
+          @rspec_reports_jobs = rspec_reports_jobs
+          @rspec_reports_glob = rspec_reports_glob
+          @coverage_report_path = coverage_report_path
+          @test_map_uri = URI.parse(test_map_url)
+        end
+
+        def rspec_reports
+          @rspec_report_files ||= rspec_reports_paths.map do |report_path|
+            JSON.parse(File.read(report_path))
+          rescue JSON::ParserError => e
+            raise "Invalid JSON in RSpec report file #{report_path}: #{e.message}"
+          end
+        end
+
+        def coverage_report
+          @coverage_report ||= read_coverage_reports
+        end
+
+        def test_map
+          @test_map ||= fetch_test_map_from_url
+        end
+
+        private
+
+        def rspec_reports_paths
+          @rspec_reports_paths ||= begin
+            paths = @rspec_reports_jobs.flat_map do |job_name|
+              Dir.glob(File.join(@working_dir, job_name, @rspec_reports_glob))
+            end.compact
+
+            if paths.empty? || paths.none? { |path| File.exist?(path) } # rubocop:disable Style/IfUnlessModifier
+              raise "No RSpec reports found in #{@working_dir}/<job-name>/ for these jobs: #{@rspec_reports_jobs}."
+            end
+
+            paths
+          end
+        end
+
+        def read_coverage_reports
+          raise "Coverage report not found in: #{@coverage_report_path}" unless File.exist?(@coverage_report_path)
+
+          File.read(@coverage_report_path)
+        end
+
+        def fetch_test_map_from_url
+          attempt = 0
+
+          begin
+            attempt += 1
+            response = http_get(@test_map_uri)
+            raise "HTTP #{response.code}: #{response.message}" unless response.is_a?(Net::HTTPSuccess)
+
+            JSON.parse(decompressed_gzip(response.body))
+          rescue StandardError => e
+            if attempt <= MAX_RETRIES
+              sleep_duration = exponential_delay_with_jitter(attempt)
+              warn "Attempt #{attempt}/#{MAX_RETRIES} failed: #{e.message}. Retrying in #{sleep_duration.round(2)}s..."
+              sleep(sleep_duration)
+              retry
+            end
+
+            raise "Failed to fetch test map from #{@test_map_uri} after #{MAX_RETRIES} attempts: #{e.message}"
+          end
+        end
+
+        def http_get(uri)
+          Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+            http.get(uri.request_uri)
+          end
+        end
+
+        def decompressed_gzip(gzipped_data)
+          Zlib::GzipReader.new(StringIO.new(gzipped_data)).read
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/category_owners.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'yaml'
+
+require_relative 'utils'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class CategoryOwners
+        include Utils
+
+        SOURCE_URL  = URI('https://gitlab.com/gitlab-com/www-gitlab-com/raw/master/data/stages.yml')
+        BASE_DELAY  = 1 # seconds
+        MAX_RETRIES = 3
+
+        # @return [Hash] Category ownership hierarchy, section -> stage -> group -> [categories]
+        # @example Return value
+        #   {
+        #     "team_planning" => {             # section
+        #       "project_management" => {      # stage
+        #         "plan" => [                  # group
+        #           "dev",                     # category
+        #           "service_desk"             # category
+        #         ],
+        #         "product_planning" => [      # group
+        #           "portfolio_management",    # category
+        #           ...
+        #         ]
+        #       }
+        #     },
+        #     ...
+        #   }
+        attr_reader :hierarchy
+
+        def initialize
+          @categories_map = {}
+          @hierarchy = {}
+
+          yaml_file = fetch_yaml_file
+          yaml_content = YAML.load(yaml_file)
+          populate_ownership_hierarchy(yaml_content)
+        end
+
+        # @return [Array<Hash>] Flattened category ownership
+        # @example Return value
+        #   [
+        #     { category: "team_planning", group: "project_management", stage: "plan", section: "dev" },
+        #     { category: "service_desk", group: "project_management", stage: "plan", section: "dev" },
+        #     { category: "portfolio_management", group: "product_planning", stage: "plan", section: "dev" }
+        #     ...
+        #   ]
+        def as_db_table
+          hierarchy.each_with_object([]) do |(section, stages), flattened_hierarchy|
+            next unless stages
+
+            stages.each do |stage, groups|
+              next unless groups
+
+              groups.each do |group, categories|
+                Array(categories).each do |category|
+                  flattened_hierarchy << {
+                    category: category,
+                    group: group,
+                    stage: stage,
+                    section: section
+                  }
+                end
+              end
+            end
+          end
+        end
+
+        # @return [Hash] Mapping of categories to teams (i.e., groups, stages, sections)
+        # @example Return value
+        #   {
+        #     "team_planning" => { group: "project_management", stage: "plan", section: "dev" },
+        #     "service_desk" => { group: "project_management", stage: "plan", section: "dev" },
+        #     "portfolio_management" => { group: "product_planning", stage: "plan", section: "dev" },
+        #     ...
+        #   }
+        def categories_to_teams
+          populate_categories_map(hierarchy)
+          @categories_map
+        end
+
+        private
+
+        def fetch_yaml_file
+          retries = 0
+
+          begin
+            response = Net::HTTP.get_response(SOURCE_URL)
+            raise "Failed to fetch file: HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+            response.body
+          rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED, Errno::ETIMEDOUT, IOError, RuntimeError => e
+            retries += 1
+            raise "Failed to fetch YAML after #{MAX_RETRIES} retries: #{e.message}" if retries >= MAX_RETRIES
+
+            delay = exponential_delay_with_jitter(retries)
+            warn "Fetch attempt #{retries} failed: #{e.message}. Retrying in #{delay.round(2)}s..."
+            sleep(delay)
+            retry
+          end
+        end
+
+        def populate_ownership_hierarchy(data)
+          stages = data['stages'] || {}
+
+          stages.each do |stage, stage_data|
+            next unless stage_data.is_a?(Hash)
+
+            section = stage_data['section']
+            groups = stage_data['groups'] || {}
+            next unless section
+
+            groups.each { |group, group_data| add_hierarchy_entry(section, stage, group, group_data['categories']) }
+          end
+        end
+
+        def add_hierarchy_entry(section, stage, group, categories)
+          @hierarchy[section] ||= {}
+          @hierarchy[section][stage] ||= {}
+          @hierarchy[section][stage][group] = categories || []
+        end
+
+        def populate_categories_map(data, current_section = nil, current_stage = nil, current_group = nil)
+          case data
+          when Hash # Sections / Stages / Groups
+            data.each do |key, value|
+              if current_section.nil? # Sections
+                populate_categories_map(value, key, nil, nil)
+              elsif current_stage.nil? # Stages
+                populate_categories_map(value, current_section, key, nil)
+              elsif current_group.nil? # Groups
+                populate_categories_map(value, current_section, current_stage, key)
+              else                     # Categories
+                populate_categories_map(value, current_section, current_stage, current_group)
+              end
+            end
+          when Array # Categories array
+            data.each do |category|
+              next unless category.is_a?(String)
+
+              @categories_map[category] = {
+                section: current_section,
+                stage: current_stage,
+                group: current_group
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class CategoryOwnersTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
+          TABLE_NAME = "category_owners"
+
+          # Creates the ClickHouse table, if it doesn't exist already
+          # @return [nil]
+          def create
+            logger.debug("#{LOG_PREFIX} Creating category_owners table if it doesn't exist ...")
+
+            client.query(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{table_name} (
+                timestamp DateTime64(6, 'UTC') DEFAULT now64(),
+                category String,
+                group String,
+                stage String,
+                section String,
+                INDEX idx_group group TYPE set(360) GRANULARITY 1,
+                INDEX idx_stage stage TYPE set(360) GRANULARITY 1,
+                INDEX idx_section section TYPE set(360) GRANULARITY 1
+              ) ENGINE = MergeTree()
+              ORDER BY (category, timestamp)
+              SETTINGS index_granularity = 8192;
+            SQL
+
+            logger.info("#{LOG_PREFIX} Category owners table created/verified successfully")
+          end
+
+          def truncate
+            logger.debug("#{LOG_PREFIX} Truncating table #{full_table_name} ...")
+
+            client.query("TRUNCATE TABLE #{full_table_name}")
+
+            logger.info("#{LOG_PREFIX} Successfully truncated table #{full_table_name}")
+          end
+
+          private
+
+          # @return [Boolean] True if the record is valid, false otherwise
+          def valid_record?(record)
+            required_fields = %i[category group stage section]
+
+            required_fields.each do |field|
+              if record[field].nil?
+                logger.warn("#{LOG_PREFIX} Skipping record with nil #{field}: #{record}")
+                return false
+              end
+            end
+
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class CoverageMetricsTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
+          TABLE_NAME = "coverage_metrics"
+
+          # Creates the ClickHouse table, if it doesn't exist already
+          # @return [nil]
+          def create
+            logger.debug("#{LOG_PREFIX} Creating coverage_metrics table if it doesn't exist ...")
+
+            client.query(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{table_name} (
+                timestamp DateTime64(6, 'UTC'),
+                file String,
+                coverage Float64,
+                category Nullable(String),
+                ci_project_id Nullable(UInt32),
+                ci_project_path Nullable(String),
+                ci_job_name Nullable(String),
+                ci_job_id Nullable(UInt64),
+                ci_pipeline_id Nullable(UInt64),
+                ci_merge_request_iid Nullable(UInt32),
+                ci_branch Nullable(String),
+                ci_target_branch Nullable(String)
+              ) ENGINE = MergeTree()
+              PARTITION BY toYYYYMM(timestamp)
+              ORDER BY (ci_project_path, timestamp, file, ci_pipeline_id)
+              SETTINGS index_granularity = 8192, allow_nullable_key = 1;
+            SQL
+
+            logger.info("#{LOG_PREFIX} Coverage metrics table created/verified successfully")
+          end
+
+          private
+
+          # @return [Boolean] True if the record is valid, false otherwise
+          def valid_record?(record)
+            if record[:file].nil?
+              logger.warn("#{LOG_PREFIX} Skipping record with nil file: #{record}")
+              return false
+            end
+
+            if record[:coverage].nil?
+              logger.warn("#{LOG_PREFIX} Skipping record with nil coverage: #{record}")
+              return false
+            end
+
+            if record[:coverage].nan?
+              logger.warn("#{LOG_PREFIX} Skipping record with NaN coverage: #{record}")
+              return false
+            end
+
+            true
+          end
+
+          # @return [Hash] Transformed coverage data including timestamp and CI metadata
+          def sanitized_data_record(record)
+            {
+              timestamp: time,
+              file: record[:file],
+              coverage: record[:coverage],
+              category: record[:category],
+              **ci_metadata
+            }
+          end
+
+          # @return [Time] Common timestamp for all coverage records
+          def time
+            @time ||= begin
+              ci_created_at = ENV.fetch('CI_PIPELINE_CREATED_AT', nil)
+              ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc
+            end
+          end
+
+          # @return [Hash] CI-related metadata
+          def ci_metadata
+            {
+              ci_project_id: env_to_int('CI_PROJECT_ID'),
+              ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil),
+              ci_job_name: ENV.fetch('CI_JOB_NAME', nil)&.gsub(%r{ \d{1,2}/\d{1,2}}, ''),
+              ci_job_id: env_to_int('CI_JOB_ID'),
+              ci_pipeline_id: env_to_int('CI_PIPELINE_ID'),
+              ci_merge_request_iid: env_to_int('CI_MERGE_REQUEST_IID') || env_to_int('TOP_UPSTREAM_MERGE_REQUEST_IID'),
+              ci_branch: ENV.fetch('CI_COMMIT_REF_NAME', nil),
+              ci_target_branch: ENV.fetch('CI_MERGE_REQUEST_TARGET_BRANCH_NAME', nil)
+            }
+          end
+
+          # @param name [String] Environment variable name
+          # @return [Integer, nil] Environment variable converted to integer or
+          #   nil if not present or empty
+          def env_to_int(name)
+            value = ENV.fetch(name, nil)
+            return nil if value.nil? || value.empty?
+
+            value.to_i
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class Table
+          LOG_PREFIX = "[CodeCoverage]"
+
+          def initialize(url:, database:, username: nil, password: nil, logger: nil)
+            @url = url
+            @database = database
+            @username = username
+            @password = password
+            @logger = logger || Runtime::Logger.logger
+          end
+
+          # @param data [Array<Hash>] Code coverage related data to be pushed to ClickHouse
+          # @return [nil]
+          def push(data) # rubocop:disable Metrics/AbcSize
+            return logger.warn("#{LOG_PREFIX} No data found, skipping ClickHouse export!") if data.empty?
+
+            logger.debug("#{LOG_PREFIX} Starting data export to ClickHouse")
+            sanitized_data = sanitize(data)
+
+            client.insert_json_data(table_name, sanitized_data)
+            logger.info("#{LOG_PREFIX} Successfully pushed #{sanitized_data.size} records to #{full_table_name}!")
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Error occurred while pushing data to #{full_table_name}: #{e.message}")
+            raise
+          end
+
+          private
+
+          attr_reader :url, :database, :username, :password, :logger
+
+          def sanitize(data)
+            data.filter_map { |record| sanitized_data_record(record) if valid_record?(record) }
+          end
+
+          def sanitized_data_record(record)
+            record
+          end
+
+          def full_table_name
+            "#{database}.#{table_name}"
+          end
+
+          def table_name
+            self.class::TABLE_NAME
+          rescue NameError
+            raise NotImplementedError, "#{self.class} must define the TABLE_NAME constant"
+          end
+
+          def valid_record?(_record)
+            raise NotImplementedError, "#{self.class}##{__method__} method must be implemented in a subclass"
+          end
+
+          # @return [GitlabQuality::TestTooling::ClickHouse::Client]
+          def client
+            @client ||= GitlabQuality::TestTooling::ClickHouse::Client.new(
+              url: url,
+              database: database,
+              username: username,
+              password: password,
+              logger: logger
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class CoverageData
+        # @param [Hash<String, Hash>] code_coverage_by_source_file Source file
+        #   mapped to test coverage data
+        # @param [Hash<String, Array<String>>] source_file_to_tests Source files
+        #   mapped to all test files testing them
+        # @param [Hash<String, Array<String>>] tests_to_categories Test files
+        #   mapped to all feature categories they belong to
+        # @param [Hash<String, Hash>] categories_to_teams Mapping of categories
+        #   to teams (i.e., groups, stages, sections)
+        def initialize(code_coverage_by_source_file, source_file_to_tests, tests_to_categories, categories_to_teams)
+          @code_coverage_by_source_file = code_coverage_by_source_file
+          @source_file_to_tests = source_file_to_tests
+          @tests_to_categories = tests_to_categories
+          @categories_to_teams = categories_to_teams
+        end
+
+        # @return [Array<Hash<Symbol, String>>] Mapping of column name to row
+        #   value
+        # @example Return value
+        #   [
+        #     {
+        #       file: "app/channels/application_cable/channel.rb"
+        #       coverage: 100.0
+        #       category: "team_planning"
+        #       group: "project_management"
+        #       stage: "plan"
+        #       section: "dev"
+        #     },
+        #     ...
+        #   ]
+        def as_db_table
+          all_files.flat_map do |file|
+            coverage = @code_coverage_by_source_file[file]&.dig(:percentage)
+            categories = categories_for(file)
+            if categories.empty?
+              { file: file, coverage: coverage }.merge(no_owner_info)
+            else
+              categories.map do |category|
+                { file: file, coverage: coverage }.merge(owner_info(category))
+              end
+            end
+          end
+        end
+
+        private
+
+        def no_owner_info
+          {
+            category: nil,
+            group: nil,
+            stage: nil,
+            section: nil
+          }
+        end
+
+        def owner_info(category)
+          owner_info = @categories_to_teams[category]
+
+          {
+            category: category,
+            group: owner_info&.dig(:group),
+            stage: owner_info&.dig(:stage),
+            section: owner_info&.dig(:section)
+          }
+        end
+
+        def categories_for(file)
+          @source_file_to_tests[file]&.flat_map { |test_file| @tests_to_categories[test_file] || [] }&.uniq || []
+        end
+
+        def all_files
+          @all_files ||= @code_coverage_by_source_file.keys
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class LcovFile
+        # @param [String] lcov_file_content The content of the lcov file
+        def initialize(lcov_file_content)
+          @lcov_file_content = lcov_file_content
+        end
+
+        # @return [Hash<String, Hash>] The parsed content of the lcov file
+        # @example Return value
+        #   {
+        #     "path/to/file1.rb" => {
+        #       line_coverage: { 1 => 1, 2 => 0 },
+        #       branch_coverage: {},
+        #       total_lines: 2,
+        #       covered_lines: 1,
+        #       percentage: 50.0
+        #     },
+        #     ...
+        #  }
+        def parsed_content
+          return @parsed_content if @parsed_content
+
+          @parsed_content = {}
+          @current_file = nil
+
+          @lcov_file_content.each_line do |line|
+            case line
+            when /^SF:(.+)$/
+              register_source_file(::Regexp.last_match(1))
+            when /^DA:(\d+),(\d+)$/
+              register_line_data(::Regexp.last_match(1), ::Regexp.last_match(2))
+            when /^BRDA:(\d+),(\d+),(\d+),(-|\d+)$/
+              register_branch_data(::Regexp.last_match(1), ::Regexp.last_match(4))
+            when /^end_of_record$/
+              @current_file = nil
+            end
+          end
+
+          include_coverage
+          @parsed_content
+        end
+
+        private
+
+        def include_coverage
+          @parsed_content.each_key do |file|
+            # TODO: Support branch coverage too
+            @parsed_content[file].merge!(line_coverage_for(file))
+          end
+        end
+
+        def line_coverage_for(file)
+          data = @parsed_content[file]
+          return unless data
+
+          total_lines = data[:line_coverage].size
+          covered_lines = data[:line_coverage].values.count(&:positive?)
+
+          {
+            total_lines: total_lines,
+            covered_lines: covered_lines,
+            percentage: total_lines.zero? ? 0.0 : (covered_lines.to_f / total_lines * 100).round(2)
+          }
+        end
+
+        def register_source_file(filename)
+          @current_file = filename.gsub(%r{^\./}, '')
+          @parsed_content[@current_file] = { line_coverage: {}, branch_coverage: {} }
+        end
+
+        def register_line_data(line_no, count)
+          return unless @current_file
+
+          @parsed_content[@current_file][:line_coverage][line_no.to_i] = count.to_i
+        end
+
+        def register_branch_data(line_no, taken)
+          return unless @current_file
+
+          taken_count = taken == '-' ? 0 : taken.to_i
+          @parsed_content[@current_file][:branch_coverage][line_no.to_i] ||= []
+          @parsed_content[@current_file][:branch_coverage][line_no.to_i] << taken_count
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class RspecReport
+        # @param [Hash<String, Object>] rspec_report The content of an RSpec
+        #   report
+        def initialize(rspec_report)
+          @rspec_report = rspec_report
+        end
+
+        # @return [Array<Hash<String, String>>] Content of the "examples"
+        #   section of the RSpec report
+        def examples
+          @examples ||= @rspec_report['examples']
+        end
+
+        # @return [Hash<String, Array<String>>] Test files mapped to all feature
+        #   categories they belong to
+        # @example Return value
+        #   {
+        #     "spec/path/to/file_spec.rb" => [
+        #       "feature_category1", "feature_category2"
+        #     ],
+        #     ...
+        #   }
+        def tests_to_categories
+          @tests_to_categories ||= examples.to_a.filter_map do |example|
+            next unless example.is_a?(Hash)
+
+            file_path = example['file_path']
+            next unless file_path.is_a?(String)
+
+            [file_path.gsub('./', ''), Array(example['feature_category']).compact]
+          end.to_h
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/test_map.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class TestMap
+        SEPARATOR = '/'
+        MARKER = 1
+
+        # @param [Hash] compact_map A nested hash structure where keys are
+        #   source files and values are tree structures with test file paths
+        # @example Example of compact_map for a file tested by 3 spec files
+        #   {
+        #     "app/models/user.rb" => {
+        #       "spec" => {
+        #         "models" => {
+        #           "user_spec.rb" => 1  # MARKER (1) indicates a leaf node
+        #         }
+        #       },
+        #       "ee" => {
+        #         "spec" => {
+        #           "lib" => {
+        #              "ee" => {
+        #                "gitlab" => {
+        #                  "background_migration" => {
+        #                    "delete_invalid_epic_issues_spec.rb"=>1,
+        #                    "backfill_security_policies_spec.rb"=>1
+        #                  }
+        #                }
+        #              }
+        #           }
+        #         }
+        #       }
+        #     }
+        #   }
+        def initialize(compact_map)
+          @compact_map = compact_map
+        end
+
+        # @return [Hash<String, Array<String>>] Source files mapped to all test
+        #   files testing them
+        # @example Return value
+        #   {
+        #     "path/to/file1.rb" => [
+        #       "spec/path/to/file1_spec.rb",
+        #       "spec/path/to/another/file1_spec.rb"
+        #     ],
+        #     ...
+        #   }
+        def source_to_tests
+          @source_to_tests ||= @compact_map.transform_values { |tree| traverse(tree).to_a.uniq }
+        end
+
+        # @return [Hash<String, Array<String>>] Test files mapped to all source
+        #   files tested by them
+        # @example Return value
+        #   {
+        #     "spec/path/to/file1_spec.rb" => [
+        #       "path/to/file1.rb",
+        #       "path/to/file2.rb"
+        #     ],
+        #     ...
+        #   }
+        def test_to_sources
+          @test_to_sources ||= begin
+            test_to_sources = Hash.new { |hash, key| hash[key] = [] }
+
+            @compact_map.each do |source_file, tree|
+              traverse(tree).to_a.each do |test_file|
+                test_to_sources[test_file] << source_file
+              end
+            end
+
+            test_to_sources
+          end
+        end
+
+        private
+
+        def traverse(tree, segments = [], &block)
+          return to_enum(__method__, tree, segments) unless block
+          return yield segments.join(SEPARATOR) if tree == MARKER && !segments.empty?
+
+          tree.each do |key, value|
+            traverse(value, segments + [key], &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/utils.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'yaml'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module Utils
+        def exponential_delay_with_jitter(attempt)
+          exponential_delay = (2**(attempt - 1))
+          jitter = rand # 0-1 seconds
+          exponential_delay + jitter
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb

@@ -36,7 +36,7 @@ module GitlabQuality
           end
 
           def new_issue_labels(_test)
-            %w[E2E status::automated]
+            %w[E2E status::automated suppress-contributor-links]
           end
 
           def up_to_date_labels(test:, issue: nil, new_labels: Set.new)

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -71,6 +71,7 @@ module GitlabQuality
           exclude_labels_for_search: nil,
           metrics_files: [],
           group_similar: false,
+          environment_issues_output_file: nil,
           **kwargs)
           super
           @max_diff_ratio = max_diff_ratio.to_f
@@ -81,11 +82,13 @@ module GitlabQuality
           @commented_issue_list = Set.new
           @metrics_files = Array(metrics_files)
           @group_similar = group_similar
+          @environment_issues_output_file = environment_issues_output_file
         end
 
         private
 
-        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client, :group_similar
+        attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client, :group_similar,
+          :environment_issues_output_file
 
         def run!
           puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
@@ -123,6 +126,8 @@ module GitlabQuality
 
           grouper.process_failures(failure_data)
           grouper.process_issues
+
+          export_environment_issues_for_slack(environment_issues_output_file) if environment_issues_output_file
         end
 
         def collect_all_test_results
@@ -690,6 +695,43 @@ module GitlabQuality
 
           "|| [Screenshot](#{ci_job_url}/artifacts/file/#{screenshot_path})"
         end
+
+        def export_environment_issues_for_slack(output_file)
+          return unless similar_issues_grouped?
+
+          File.write(output_file, build_environment_issues_data.to_json)
+        rescue StandardError => e
+          puts "Warning: Failed to export environment issues for Slack: #{e.message}"
+        end
+
+        def build_environment_issues_data
+          {
+            grouped_failures: format_grouped_failures,
+            summary: grouper.summary
+          }
+        end
+
+        def format_grouped_failures
+          grouper.grouped_failures.map do |_fingerprint, grouped_failure|
+            {
+              fingerprint: grouped_failure[:fingerprint],
+              pattern_name: grouped_failure[:pattern_name],
+              normalized_message: grouped_failure[:normalized_message],
+              failure_count: grouped_failure[:failures].size,
+              failures: format_individual_failures(grouped_failure[:failures])
+            }
+          end
+        end
+
+        def format_individual_failures(failures)
+          failures.map do |failure|
+            {
+              description: failure[:description],
+              file_path: failure[:file_path],
+              ci_job_url: failure[:ci_job_url]
+            }
+          end
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/slack/post_to_slack.rb

@@ -1,15 +1,21 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module GitlabQuality
   module TestTooling
     module Slack
       class PostToSlack
-        def initialize(slack_webhook_url:, channel:, message:, username:, icon_emoji:)
+        MAX_PATTERN_MESSAGE_LENGTH = 150
+        MAX_GROUPED_FAILURES_TO_DISPLAY = 10
+
+        def initialize(slack_webhook_url:, channel:, message:, username:, icon_emoji:, environment_issues_file: nil)
           @slack_webhook_url = slack_webhook_url
           @channel = channel
           @message = message
           @username = username
           @icon_emoji = icon_emoji
+          @environment_issues_file = environment_issues_file
         end
 
         def invoke!
@@ -17,7 +23,7 @@ module GitlabQuality
           params['channel'] = channel
           params['username'] = username
           params['icon_emoji'] = icon_emoji
-          params['text'] = message
+          params['text'] = build_message
 
           Support::HttpRequest.make_http_request(
             method: 'post',
@@ -29,7 +35,101 @@ module GitlabQuality
 
         private
 
-        attr_reader :slack_webhook_url, :channel, :message, :username, :icon_emoji
+        attr_reader :slack_webhook_url, :channel, :message, :username, :icon_emoji, :environment_issues_file
+
+        def build_message
+          messages = []
+          messages << message if message && !message.empty?
+          messages << format_environment_issues if environment_issues_file && File.exist?(environment_issues_file)
+
+          messages.join("\n\n")
+        end
+
+        def format_environment_issues
+          issues_data = JSON.parse(File.read(environment_issues_file))
+          return nil if issues_data.nil? || issues_data['grouped_failures'].empty?
+
+          build_slack_message(issues_data)
+        rescue JSON::ParserError => e
+          ":x: Error parsing environment issues file: #{e.message}"
+        rescue StandardError => e
+          ":x: Error formatting environment issues: #{e.message}"
+        end
+
+        def format_single_environment_issue(failure)
+          pattern_title = pattern_display_name(failure['pattern_name'])
+
+          issue_text = build_issue_header(pattern_title, failure)
+          issue_text + build_job_info(failure)
+        end
+
+        def pattern_display_name(pattern_name)
+          case pattern_name&.downcase
+          when /http_500/
+            "HTTP 500 Internal Server Errors"
+          when /http_400/
+            "HTTP 400 Bad Request Errors"
+          when /http_503/
+            "HTTP 503 Service Unavailable"
+          when /timeout/
+            "Timeout Errors"
+          when /git_rpc|repository/
+            "Git/Repository Errors"
+          else
+            "#{pattern_name&.humanize || 'Unknown'} Errors"
+          end
+        end
+
+        def build_slack_message(issues_data)
+          header = ":warning: *Environment Issues Detected*\n"
+
+          issue_messages = format_issue_messages(issues_data['grouped_failures'])
+          truncation_note = build_truncation_note(issues_data['grouped_failures'].size)
+          summary = build_summary_text(issues_data['summary'])
+
+          header + issue_messages + truncation_note + summary
+        end
+
+        def format_issue_messages(grouped_failures)
+          failures_to_show = grouped_failures.first(MAX_GROUPED_FAILURES_TO_DISPLAY)
+          failures_to_show.map { |failure| format_single_environment_issue(failure) }.join("\n\n")
+        end
+
+        def build_truncation_note(total_failures)
+          return "" unless total_failures > MAX_GROUPED_FAILURES_TO_DISPLAY
+
+          "\n_... and #{total_failures - MAX_GROUPED_FAILURES_TO_DISPLAY} more environment issue(s)_"
+        end
+
+        def build_issue_header(pattern_title, failure)
+          <<~TEXT
+            *#{pattern_title}*
+            • Affected tests: #{failure['failure_count']}
+            • Pattern: `#{truncate_message(failure['normalized_message'])}`
+          TEXT
+        end
+
+        def build_job_info(failure)
+          return "" unless failure['failures']&.any?
+
+          job_urls = failure['failures'].filter_map { |f| f['ci_job_url'] }.uniq
+          job_urls.any? ? "• Jobs affected: #{job_urls.size}\n" : ""
+        end
+
+        def build_summary_text(summary)
+          <<~TEXT
+
+            *Summary:* #{summary['grouped_issues']} environment issue(s) affecting #{summary['total_grouped_failures']} test(s)
+
+            _Note: Future improvements will include direct GitLab issue links and enhanced filtering._
+            _Track progress: https://gitlab.com/groups/gitlab-org/quality/quality-engineering/-/epics/168_
+          TEXT
+        end
+
+        def truncate_message(message)
+          text = message.to_s
+          text.length > MAX_PATTERN_MESSAGE_LENGTH ? "#{text[0..MAX_PATTERN_MESSAGE_LENGTH]}..." : text
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/utils.rb

@@ -78,7 +78,7 @@ module GitlabQuality
             )
             ENGINE = MergeTree()
             PARTITION BY toYYYYMM(timestamp)
-            ORDER BY (ci_project_path, timestamp, run_type, status, feature_category, file_path, ci_pipeline_id)
+            ORDER BY (ci_project_path, status, run_type, feature_category, file_path, timestamp, ci_pipeline_id)
             SETTINGS index_granularity = 8192;
           SQL
           return if config.extra_metadata_columns.empty?

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "2.22.0"
+    VERSION = "2.24.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 2.22.0
+  version: 2.24.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-10-15 00:00:00.000000000 Z
+date: 2025-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -449,6 +449,7 @@ executables:
 - report-results
 - slow-test-issues
 - slow-test-merge-request-report-note
+- test-coverage
 - update-screenshot-paths
 - update-test-meta
 extensions: []
@@ -483,11 +484,22 @@ files:
 - exe/report-results
 - exe/slow-test-issues
 - exe/slow-test-merge-request-report-note
+- exe/test-coverage
 - exe/update-screenshot-paths
 - exe/update-test-meta
 - lefthook.yml
 - lib/gitlab_quality/test_tooling.rb
 - lib/gitlab_quality/test_tooling/click_house/client.rb
+- lib/gitlab_quality/test_tooling/code_coverage/artifacts.rb
+- lib/gitlab_quality/test_tooling/code_coverage/category_owners.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb
+- lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb
+- lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb
+- lib/gitlab_quality/test_tooling/code_coverage/test_map.rb
+- lib/gitlab_quality/test_tooling/code_coverage/utils.rb
 - lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb
 - lib/gitlab_quality/test_tooling/failed_jobs_table.rb
 - lib/gitlab_quality/test_tooling/feature_readiness/analyzed_items/analyzed_epic.rb