gitlab_quality-test_tooling 3.5.0 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0afeda440d33c3cb9e1e81adf43ce462d12b604cffdf96b04dbe83a9a4d1f853
-  data.tar.gz: 1a638b0a0b409ed28cb94da821a59f73967b6ae55cad7e87552a096b9c5c525e
+  metadata.gz: 4a734c368f480307b04c5cb1fb3e91059e0a8c357da88eb10e42b06071b7a528
+  data.tar.gz: 032f619282b68ef105086a34e1a67cb29bf9e620e3031d30fe941ddb9e36abc8
 SHA512:
-  metadata.gz: 5bece6881a2ab2506f4d22b7da49b976af027a0f7d93c276d704f37fb52091ddde319232db0ccca1b717141267bbd5e283f30796a8bc06b14dbbbd38115327ca
-  data.tar.gz: 7136b103704d811fcdaee9a44ee815775d2eb0411934654131d6478a8d2b62c3225a94a5602de169284cd3121a1b464bcbcf65570a5f7f923add0cdb0b5b386a
+  metadata.gz: 409cff4f9f3afd710b9b8e06d15e8b90cc550d59d15237ce8cbd11b813cdee938bba0873cbefcca8df55beb817c5a0fa4507d880d7124e520e74e549c48fdbe6
+  data.tar.gz: 9d1d9bcee36c44006824cee0f9cfafe80c1a8da902975dc6f53d9c7c64286775513b7e1830dee811487c9784a6223b31d853e3252a20488ceb98e012c8e4c8b7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.5.0)
+    gitlab_quality-test_tooling (3.6.0)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/README.md

@@ -276,6 +276,53 @@ Usage: exe/feature-readiness-evaluation [options]
     -h, --help                       Show the usage
 ```
 
+### `exe/test-coverage`
+
+```shell
+Purpose: Export test coverage metrics to ClickHouse
+Usage: exe/test-coverage [options]
+
+Options:
+        --test-reports GLOB          Glob pattern for test JSON reports (RSpec or Jest) (e.g., "reports/**/*.json")
+        --coverage-report PATH       Path to the LCOV coverage report (e.g., "coverage/lcov/gitlab.lcov")
+        --test-map PATH              Path to the test map file (e.g., "crystalball/packed-mapping.json.gz")
+        --clickhouse-url URL         ClickHouse server URL
+        --clickhouse-database DATABASE
+                                     ClickHouse database name
+        --clickhouse-username USERNAME
+                                     ClickHouse username
+        --clickhouse-shared-database DATABASE
+                                     ClickHouse shared database name (default: shared)
+        --responsibility-patterns PATH
+                                     Path to YAML file with responsibility classification patterns
+
+Environment variables:
+  GLCI_CLICKHOUSE_METRICS_PASSWORD   ClickHouse password (required, not passed via CLI for security)
+
+    -h, --help                       Show the usage
+    -v, --version                    Show the version
+```
+
+### `exe/sync-category-owners`
+
+```shell
+Purpose: Sync feature category ownership data from stages.yml to ClickHouse
+Usage: exe/sync-category-owners [options]
+
+Options:
+        --clickhouse-url URL         ClickHouse server URL
+        --clickhouse-database DATABASE
+                                     ClickHouse database name (default: shared)
+        --clickhouse-username USERNAME
+                                     ClickHouse username
+
+Environment variables:
+  GLCI_CLICKHOUSE_METRICS_PASSWORD   ClickHouse password (required)
+
+    -h, --help                       Show the usage
+    -v, --version                    Show the version
+```
+
 ## Development
 
 ### Initial setup

data/exe/sync-category-owners

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "optparse"
+require "uri"
+
+require_relative "../lib/gitlab_quality/test_tooling"
+
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/category_owners'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table'
+
+params = {}
+required_params = [:clickhouse_url, :clickhouse_database, :clickhouse_username]
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.separator ""
+  opts.separator "Syncs feature category ownership data from stages.yml to ClickHouse."
+  opts.separator ""
+  opts.separator "Options:"
+
+  opts.on('--clickhouse-url URL', 'ClickHouse server URL') do |url|
+    params[:clickhouse_url] = url
+  end
+
+  opts.on('--clickhouse-database DATABASE', 'ClickHouse database name (default: shared)') do |database|
+    params[:clickhouse_database] = database
+  end
+
+  opts.on('--clickhouse-username USERNAME', 'ClickHouse username') do |username|
+    params[:clickhouse_username] = username
+  end
+
+  opts.separator ""
+  opts.separator "Environment variables:"
+  opts.separator "  GLCI_CLICKHOUSE_METRICS_PASSWORD   ClickHouse password (required)"
+  opts.separator ""
+
+  opts.on('-h', '--help', 'Show the usage') do
+    puts opts
+    exit
+  end
+
+  opts.on_tail('-v', '--version', 'Show the version') do
+    require_relative "../lib/gitlab_quality/test_tooling/version"
+    puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
+    exit
+  end
+
+  opts.parse(ARGV)
+end
+
+# Default database to 'shared' if not specified
+params[:clickhouse_database] ||= 'shared'
+
+if params.any? && (required_params - params.keys).none?
+  clickhouse_password = ENV.fetch('GLCI_CLICKHOUSE_METRICS_PASSWORD', nil)
+  if clickhouse_password.to_s.strip.empty?
+    puts "Error: GLCI_CLICKHOUSE_METRICS_PASSWORD environment variable must be set and not empty"
+    exit 1
+  end
+
+  [:clickhouse_url, :clickhouse_database, :clickhouse_username].each do |param|
+    if params[param].to_s.strip.empty?
+      puts "Error: --#{param.to_s.tr('_', '-')} cannot be empty"
+      exit 1
+    end
+  end
+
+  begin
+    uri = URI.parse(params[:clickhouse_url])
+    unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+      puts "Error: --clickhouse-url must be a valid HTTP or HTTPS URL"
+      exit 1
+    end
+  rescue URI::InvalidURIError
+    puts "Error: --clickhouse-url is not a valid URL format"
+    exit 1
+  end
+
+  category_owners = GitlabQuality::TestTooling::CodeCoverage::CategoryOwners.new
+
+  category_owners_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CategoryOwnersTable.new(
+    url: params[:clickhouse_url],
+    database: params[:clickhouse_database],
+    username: params[:clickhouse_username],
+    password: clickhouse_password
+  )
+
+  category_owners_table.truncate
+  category_owners_table.push(category_owners.as_db_table)
+
+  puts "Successfully synced #{category_owners.as_db_table.length} feature categories to ClickHouse"
+else
+  puts "Missing argument(s). Required arguments are: #{required_params.map { |p| "--#{p.to_s.tr('_', '-')}" }.join(', ')}"
+  puts options
+  exit 1
+end

data/exe/test-coverage

@@ -178,17 +178,18 @@ if params.any? && (required_params - params.keys).none?
   }
 
   category_owners_table  = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CategoryOwnersTable.new(**shared_clickhouse_data)
-  coverage_metrics_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CoverageMetricsTable.new(**clickhouse_data)
-
-  if ENV['CLICKHOUSE_PUSH_CATEGORY_DATA'] == 'true'
-    category_owners_table.truncate
-    category_owners_table.push(category_owners.as_db_table)
-  end
-
+  coverage_metrics_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CoverageMetricsTable.new(
+    category_owners_table: category_owners_table,
+    **clickhouse_data
+  )
   coverage_metrics_table.push(coverage_data.as_db_table)
 
   # Export test-to-file mappings
-  test_file_mapping_data = GitlabQuality::TestTooling::CodeCoverage::TestFileMappingData.new(test_to_sources)
+  test_file_mapping_data = GitlabQuality::TestTooling::CodeCoverage::TestFileMappingData.new(
+    test_to_sources,
+    tests_to_categories: tests_to_categories,
+    feature_categories_to_teams: category_owners.feature_categories_to_teams
+  )
   test_file_mappings_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestFileMappingsTable.new(**shared_clickhouse_data)
   test_file_mappings_table.push(test_file_mapping_data.as_db_table)
 else

data/lib/gitlab_quality/test_tooling/code_coverage/category_owners.rb

@@ -116,14 +116,18 @@ module GitlabQuality
             groups = stage_data['groups'] || {}
             next unless section
 
-            groups.each { |group, group_data| add_hierarchy_entry(section, stage, group, group_data['categories']) }
+            groups.each do |group, group_data|
+              add_hierarchy_entry(section, stage, group, group_data['categories'])
+              add_hierarchy_entry(section, stage, group, group_data['maintained_categories'])
+            end
           end
         end
 
         def add_hierarchy_entry(section, stage, group, categories)
           @hierarchy[section] ||= {}
           @hierarchy[section][stage] ||= {}
-          @hierarchy[section][stage][group] = categories || []
+          @hierarchy[section][stage][group] ||= []
+          @hierarchy[section][stage][group].concat(Array(categories))
         end
 
         def populate_feature_categories_map(data, current_section = nil, current_stage = nil, current_group = nil)

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'table'
+require_relative 'category_owners_table'
 
 module GitlabQuality
   module TestTooling
@@ -9,8 +10,15 @@ module GitlabQuality
         class CoverageMetricsTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
           TABLE_NAME = "coverage_metrics"
 
+          def initialize(category_owners_table: nil, **args)
+            super(**args)
+            @category_owners_table = category_owners_table
+          end
+
           private
 
+          attr_reader :category_owners_table
+
           # @return [Boolean] True if the record is valid, false otherwise
           def valid_record?(record)
             valid_file?(record) &&
@@ -68,10 +76,47 @@ module GitlabQuality
               is_responsible: record[:is_responsible],
               is_dependent: record[:is_dependent],
               category: record[:feature_category],
+              **coverage_counts(record),
+              **org_data(record[:feature_category]),
               **ci_metadata
             }
           end
 
+          # @return [Hash] Raw coverage counts from the record
+          def coverage_counts(record)
+            {
+              total_lines: record[:total_lines] || 0,
+              covered_lines: record[:covered_lines] || 0,
+              total_branches: record[:total_branches] || 0,
+              covered_branches: record[:covered_branches] || 0,
+              total_functions: record[:total_functions] || 0,
+              covered_functions: record[:covered_functions] || 0
+            }
+          end
+
+          # @param category [String, nil] Feature category name
+          # @return [Hash] Organization data (group, stage, section) for the category
+          def org_data(category)
+            return { group: '', stage: '', section: '' } if category.nil? || category_owners_table.nil?
+
+            @org_data_cache ||= {}
+            @org_data_cache[category] ||= fetch_org_data(category)
+          end
+
+          # @param category [String] Feature category name
+          # @return [Hash] Organization data fetched from category_owners_table
+          def fetch_org_data(category)
+            owners = category_owners_table.owners(category)
+            {
+              group: owners['group'] || '',
+              stage: owners['stage'] || '',
+              section: owners['section'] || ''
+            }
+          rescue CategoryOwnersTable::MissingMappingError
+            logger.warn("#{LOG_PREFIX} No org data found for category '#{category}', using empty values")
+            { group: '', stage: '', section: '' }
+          end
+
           # @return [Hash] CI-related metadata
           def ci_metadata
             {

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table.rb

@@ -38,7 +38,11 @@ module GitlabQuality
               timestamp: time,
               test_file: record[:test_file],
               source_file: record[:source_file],
-              ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil)
+              ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil),
+              category: record[:category] || '',
+              group: record[:group] || '',
+              stage: record[:stage] || '',
+              section: record[:section] || ''
             }
           end
         end

data/lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb

@@ -80,7 +80,13 @@ module GitlabQuality
             line_coverage: coverage_data&.dig(:percentage),
             branch_coverage: coverage_data&.dig(:branch_percentage),
             function_coverage: coverage_data&.dig(:function_percentage),
-            source_file_type: @source_file_types[file] || 'other'
+            source_file_type: @source_file_types[file] || 'other',
+            total_lines: coverage_data&.dig(:total_lines) || 0,
+            covered_lines: coverage_data&.dig(:covered_lines) || 0,
+            total_branches: coverage_data&.dig(:total_branches) || 0,
+            covered_branches: coverage_data&.dig(:covered_branches) || 0,
+            total_functions: coverage_data&.dig(:total_functions) || 0,
+            covered_functions: coverage_data&.dig(:covered_functions) || 0
           }
         end
 

data/lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data.rb

@@ -6,23 +6,36 @@ module GitlabQuality
       class TestFileMappingData
         # @param [Hash<String, Array<String>>] test_to_sources Test files
         #   mapped to all source files they cover
-        def initialize(test_to_sources)
+        # @param [Hash<String, Array<String>>] tests_to_categories Test files
+        #   mapped to their feature categories
+        # @param [Hash<String, Hash>] feature_categories_to_teams Feature categories
+        #   mapped to their org hierarchy (group, stage, section)
+        def initialize(test_to_sources, tests_to_categories: {}, feature_categories_to_teams: {})
           @test_to_sources = test_to_sources
+          @tests_to_categories = tests_to_categories
+          @feature_categories_to_teams = feature_categories_to_teams
         end
 
         # @return [Array<Hash<Symbol, String>>] Mapping data formatted for database insertion
         # @example Return value
         #   [
-        #     { test_file: "spec/models/user_spec.rb", source_file: "app/models/user.rb" },
-        #     { test_file: "spec/models/user_spec.rb", source_file: "lib/utils.rb" },
+        #     { test_file: "spec/models/user_spec.rb", source_file: "app/models/user.rb",
+        #       category: "team_planning", group: "project_management", stage: "plan", section: "dev" },
         #     ...
         #   ]
         def as_db_table
           @test_to_sources.flat_map do |test_file, source_files|
+            category = @tests_to_categories[test_file]&.first || ''
+            team = @feature_categories_to_teams[category] || {}
+
             source_files.map do |source_file|
               {
                 test_file: test_file,
-                source_file: source_file
+                source_file: source_file,
+                category: category,
+                group: team[:group] || '',
+                stage: team[:stage] || '',
+                section: team[:section] || ''
               }
             end
           end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.5.0"
+    VERSION = "3.6.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.5.0
+  version: 3.6.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-01-23 00:00:00.000000000 Z
+date: 2026-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -427,6 +427,7 @@ executables:
 - report-results
 - slow-test-issues
 - slow-test-merge-request-report-note
+- sync-category-owners
 - test-coverage
 - update-screenshot-paths
 - update-test-meta
@@ -460,6 +461,7 @@ files:
 - exe/report-results
 - exe/slow-test-issues
 - exe/slow-test-merge-request-report-note
+- exe/sync-category-owners
 - exe/test-coverage
 - exe/update-screenshot-paths
 - exe/update-test-meta