gitlab_quality-test_tooling 3.17.0 → 3.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3a2a28593ef844ad32b920eae54b731a9981de4821ed9f9eba2776e8cad1056
-  data.tar.gz: f50c339ba813f7f96bc60de1b7607ef8da9667734fc1e263794a5e6e37434470
+  metadata.gz: 3770b676c5ff745323a47749eb04f305d435cff9769a9b567a1cc76289d5b2bb
+  data.tar.gz: 9cab37f5980a19ce80980dec4f97d867ca7bfe6e5524b383c71ea8d01509bbc8
 SHA512:
-  metadata.gz: 1a4ce06d8ba98f5799aee426090cebe56eae29053febf14029300d2b9cfe9788549b0e4041e0c6f217f4db02775ce0340f836507da117ee80240354508d42204
-  data.tar.gz: 01ff166aa1fa3c6bcd1a8e5da11b0205c9747814c5fa8e33265cb18c37afe6366274c56f5032c32c0864b2e201269ead352d90e72aae13c6bd7e8f36aafabed0
+  metadata.gz: 42a46bd31d9d7bdf985ad8d430c02e0d7037e116b426ff6358f089ae619f08680cf091758cbe6b5b229612be29e691a5ee760a57fb87b0da91324860b3939590
+  data.tar.gz: 71e1eda39f86d10c71d4d14770a925eaad6dc09d019c1cbcf7d769c6e23a73de79cce7fab02412b53ddfee8d4f6e0fa56fa55ae9880636491ffa9aa1024121ae

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.17.0)
+    gitlab_quality-test_tooling (3.18.0)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/README.md

@@ -264,6 +264,12 @@ Usage: exe/feature-readiness-evaluation [options]
 
 ### `exe/test-coverage`
 
+Runs in one of three modes:
+
+- **Full coverage-metrics export** (default, no `--per-test-coverage` / `--run-aggregation`): reads the LCOV report, Crystalball test map, and test reports, then exports the coverage-metrics and test-file-mapping tables. Requires the full argument set.
+- **Per-test coverage** (`--per-test-coverage GLOB`): inserts per-test rows into `code_coverage.test_coverage_per_file` and runs the daily `test_health_risk` aggregation unless `--skip-aggregation`. Needs only the `--clickhouse-*` options; `--test-reports` and `--jest-quarantine-file` are optional enrichment.
+- **Aggregation only** (`--run-aggregation`): runs the daily `test_health_risk` aggregation and exits. Needs only the `--clickhouse-*` options. Pair with `--skip-aggregation` on batched per-test inserts to aggregate once at the end instead of once per batch.
+
 ```shell
 Purpose: Export test coverage metrics to ClickHouse
 Usage: exe/test-coverage [options]
@@ -281,6 +287,10 @@ Options:
                                      ClickHouse shared database name
         --responsibility-patterns PATH
                                      Path to YAML file with responsibility classification patterns
+        --per-test-coverage GLOB     Per-test coverage mode (see above). Glob for per-test coverage JSON/NDJSON files.
+        --jest-quarantine-file PATH  Optional. Path to quarantined_vue3_specs.txt, ingested before the aggregation.
+        --skip-aggregation           Optional. With --per-test-coverage, insert rows without running the aggregation.
+        --run-aggregation            Run only the daily test_health_risk aggregation, then exit.
 
 Environment variables:
   GLCI_CLICKHOUSE_METRICS_PASSWORD   ClickHouse password (required, not passed via CLI for security)

data/exe/test-coverage

@@ -21,12 +21,12 @@ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_report'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_map'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config'
 
 params = {}
-required_params = [:test_reports, :coverage_report, :test_map, :clickhouse_url, :clickhouse_database, :clickhouse_username, :clickhouse_shared_database, :responsibility_patterns]
 
 options = OptionParser.new do |opts|
   opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
@@ -68,19 +68,35 @@ options = OptionParser.new do |opts|
   end
 
   opts.on('--per-test-coverage GLOB',
-    'Optional. Glob pattern for per-test coverage JSON files. ' \
-      'When provided, populates code_coverage.test_coverage_per_file and runs the ' \
-      'daily test_health_risk aggregation. (e.g., "tmp/per-test-coverage-*.json")') do |pattern|
+    'Per-test coverage mode. Glob for per-test coverage JSON/NDJSON files. ' \
+      'Inserts rows into code_coverage.test_coverage_per_file and runs the daily ' \
+      'test_health_risk aggregation unless --skip-aggregation. Needs only the ' \
+      '--clickhouse-* options; --test-reports and --jest-quarantine-file are ' \
+      'optional enrichment. (e.g., "tmp/per-test-coverage-*.ndjson")') do |pattern|
     params[:per_test_coverage] = pattern
   end
 
   opts.on('--jest-quarantine-file PATH',
     'Optional. Path to quarantined_vue3_specs.txt. When provided alongside ' \
       '--per-test-coverage, populates code_coverage.jest_quarantined_tests_today ' \
-      'before the nightly aggregation so Jest tests count toward is_quarantined.') do |path|
+      'before the aggregation so Jest tests count toward is_quarantined.') do |path|
     params[:jest_quarantine_file] = path
   end
 
+  opts.on('--skip-aggregation',
+    'Optional. With --per-test-coverage, insert rows (and ingest the jest ' \
+      'quarantine file) without running the test_health_risk aggregation. Use ' \
+      'when streaming many batches, then run --run-aggregation once at the end.') do
+    params[:skip_aggregation] = true
+  end
+
+  opts.on('--run-aggregation',
+    'Run only the daily test_health_risk aggregation over ' \
+      'code_coverage.test_coverage_per_file, then exit. Needs only the ' \
+      '--clickhouse-* options.') do
+    params[:run_aggregation] = true
+  end
+
   opts.separator ""
   opts.separator "Environment variables:"
   opts.separator "  GLCI_CLICKHOUSE_METRICS_PASSWORD   ClickHouse password (required, not passed via CLI for security)"
@@ -102,31 +118,74 @@ options = OptionParser.new do |opts|
   opts.parse(ARGV)
 end
 
-if params.any? && (required_params - params.keys).none?
-  clickhouse_password = ENV.fetch('GLCI_CLICKHOUSE_METRICS_PASSWORD', nil)
-  if clickhouse_password.to_s.strip.empty?
-    puts "Error: GLCI_CLICKHOUSE_METRICS_PASSWORD environment variable must be set and not empty"
-    exit 1
+mode =
+  if params[:run_aggregation]
+    puts "Warning: --per-test-coverage is ignored in --run-aggregation mode." if params[:per_test_coverage]
+    :aggregation
+  elsif params[:per_test_coverage]
+    :per_test
+  else
+    :full_export
   end
 
-  [:clickhouse_url, :clickhouse_database, :clickhouse_username, :clickhouse_shared_database].each do |param|
-    if params[param].to_s.strip.empty?
-      puts "Error: --#{param.to_s.tr('_', '-')} cannot be empty"
-      exit 1
-    end
-  end
+# Per-test and aggregation-only modes are self-contained: they touch only the
+# ClickHouse connection, not the LCOV report, Crystalball test map, or
+# responsibility patterns the full coverage-metrics export needs.
+clickhouse_required = [:clickhouse_url, :clickhouse_database, :clickhouse_username]
+full_export_required =
+  [:test_reports, :coverage_report, :test_map, :responsibility_patterns, :clickhouse_shared_database] + clickhouse_required
+required_params = mode == :full_export ? full_export_required : clickhouse_required
 
-  begin
-    uri = URI.parse(params[:clickhouse_url])
-    unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
-      puts "Error: --clickhouse-url must be a valid HTTP or HTTPS URL"
-      exit 1
-    end
-  rescue URI::InvalidURIError
-    puts "Error: --clickhouse-url is not a valid URL format"
+unless params.any? && (required_params - params.keys).none?
+  puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
+  puts options
+  exit 1
+end
+
+clickhouse_password = ENV.fetch('GLCI_CLICKHOUSE_METRICS_PASSWORD', nil)
+if clickhouse_password.to_s.strip.empty?
+  puts "Error: GLCI_CLICKHOUSE_METRICS_PASSWORD environment variable must be set and not empty"
+  exit 1
+end
+
+non_empty_params = mode == :full_export ? clickhouse_required + [:clickhouse_shared_database] : clickhouse_required
+non_empty_params.each do |param|
+  next unless params[param].to_s.strip.empty?
+
+  puts "Error: --#{param.to_s.tr('_', '-')} cannot be empty"
+  exit 1
+end
+
+begin
+  uri = URI.parse(params[:clickhouse_url])
+  unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    puts "Error: --clickhouse-url must be a valid HTTP or HTTPS URL"
     exit 1
   end
+rescue URI::InvalidURIError
+  puts "Error: --clickhouse-url is not a valid URL format"
+  exit 1
+end
 
+clickhouse_data = {
+  url: params[:clickhouse_url],
+  database: params[:clickhouse_database],
+  username: params[:clickhouse_username],
+  password: clickhouse_password
+}
+
+case mode
+when :aggregation
+  GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestHealthRiskAggregator.new(**clickhouse_data).run
+when :per_test
+  GitlabQuality::TestTooling::CodeCoverage::PerTestCoverageExporter.new(
+    coverage_glob: params[:per_test_coverage],
+    clickhouse: clickhouse_data,
+    test_reports: params[:test_reports],
+    jest_quarantine_file: params[:jest_quarantine_file],
+    skip_aggregation: params.fetch(:skip_aggregation, false)
+  ).run
+when :full_export
   artifacts = GitlabQuality::TestTooling::CodeCoverage::Artifacts.new(
     coverage_report: params[:coverage_report],
     test_map: params[:test_map],
@@ -181,13 +240,6 @@ if params.any? && (required_params - params.keys).none?
     test_classifications
   )
 
-  clickhouse_data = {
-    url: params[:clickhouse_url],
-    database: params[:clickhouse_database],
-    username: params[:clickhouse_username],
-    password: clickhouse_password
-  }
-
   shared_clickhouse_data = {
     url: params[:clickhouse_url],
     database: params[:clickhouse_shared_database],
@@ -210,36 +262,4 @@ if params.any? && (required_params - params.keys).none?
   )
   test_file_mappings_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestFileMappingsTable.new(**shared_clickhouse_data)
   test_file_mappings_table.push(test_file_mapping_data.as_db_table)
-
-  # Per-test coverage export (optional). Only runs when --per-test-coverage
-  # was provided AND at least one matching artifact exists.
-  if params[:per_test_coverage]
-    per_test_files = Dir.glob(params[:per_test_coverage])
-    if per_test_files.any?
-      per_test_data = GitlabQuality::TestTooling::CodeCoverage::PerTestCoverageData.new(
-        per_test_files,
-        tests_to_categories: tests_to_categories,
-        feature_categories_to_teams: category_owners.feature_categories_to_teams,
-        captured_sha: ENV.fetch('CI_COMMIT_SHA', '')
-      )
-      per_test_coverage_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::PerTestCoverageTable.new(**clickhouse_data)
-      per_test_coverage_table.push(per_test_data.as_db_table)
-
-      if params[:jest_quarantine_file] && File.exist?(params[:jest_quarantine_file])
-        jest_qt = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::JestQuarantinedTestsTable.new(**clickhouse_data)
-        jest_qt.populate(quarantine_file: params[:jest_quarantine_file])
-      elsif params[:jest_quarantine_file]
-        puts "Jest quarantine file not found at #{params[:jest_quarantine_file]}; skipping jest quarantine ingestion."
-      end
-
-      aggregator = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestHealthRiskAggregator.new(**clickhouse_data)
-      aggregator.run
-    else
-      puts "No per-test coverage artifacts matched #{params[:per_test_coverage]}; skipping per-test export and aggregation."
-    end
-  end
-else
-  puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
-  puts options
-  exit 1
 end

data/lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'logger'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      # Standalone per-test coverage export: inserts per-test, per-source-file
+      # rows into `code_coverage.test_coverage_per_file` and, unless skipped,
+      # runs the daily `test_health_risk` aggregation.
+      #
+      # Unlike the full coverage-metrics export, this path needs only ClickHouse
+      # credentials and the per-test coverage glob, not the LCOV report, the
+      # Crystalball test map, or the responsibility patterns. Category enrichment
+      # (feature_category/group/stage/section per row) is optional: pass
+      # `test_reports` to map test files to feature categories via the test
+      # report JSON, otherwise those columns stay blank.
+      #
+      # `skip_aggregation` lets a batched caller insert without re-running the
+      # table-wide aggregation on every batch. The streaming export invokes this
+      # once per artifact batch with `skip_aggregation: true`, then runs the
+      # aggregation once at the end.
+      class PerTestCoverageExporter
+        # @param clickhouse [Hash] connection params (:url, :database, :username, :password)
+        def initialize(
+          coverage_glob:, clickhouse:,
+          test_reports: nil, jest_quarantine_file: nil,
+          captured_sha: ENV.fetch('CI_COMMIT_SHA', ''), skip_aggregation: false, logger: nil)
+          @coverage_glob = coverage_glob
+          @clickhouse = clickhouse
+          @test_reports = test_reports
+          @jest_quarantine_file = jest_quarantine_file
+          @captured_sha = captured_sha.to_s
+          @skip_aggregation = skip_aggregation
+          @logger = logger || ::Logger.new($stdout)
+        end
+
+        # @return [void]
+        def run
+          coverage_files = Dir.glob(coverage_glob)
+          if coverage_files.empty?
+            logger.info(
+              "#{ClickHouse::LOG_PREFIX} No per-test coverage artifacts matched #{coverage_glob}; nothing to export."
+            )
+            return
+          end
+
+          insert(coverage_files)
+          ingest_jest_quarantine
+          aggregate unless skip_aggregation
+        end
+
+        private
+
+        attr_reader :coverage_glob, :clickhouse, :test_reports, :jest_quarantine_file,
+          :captured_sha, :skip_aggregation, :logger
+
+        def insert(coverage_files)
+          tests_to_categories, feature_categories_to_teams = resolve_categories
+
+          data = PerTestCoverageData.new(
+            coverage_files,
+            tests_to_categories: tests_to_categories,
+            feature_categories_to_teams: feature_categories_to_teams,
+            captured_sha: captured_sha
+          )
+
+          ClickHouse::PerTestCoverageTable.new(**clickhouse_credentials).push(data.as_db_table)
+        end
+
+        def ingest_jest_quarantine
+          return unless jest_quarantine_file
+
+          unless File.exist?(jest_quarantine_file)
+            logger.info(
+              "#{ClickHouse::LOG_PREFIX} Jest quarantine file not found at #{jest_quarantine_file}; " \
+                "skipping jest quarantine ingestion."
+            )
+            return
+          end
+
+          ClickHouse::JestQuarantinedTestsTable.new(**clickhouse_credentials)
+            .populate(quarantine_file: jest_quarantine_file)
+        end
+
+        def aggregate
+          ClickHouse::TestHealthRiskAggregator.new(**clickhouse_credentials).run
+        end
+
+        # `tests_to_categories` comes from the test report JSON; without it, rows
+        # carry blank categories. `CategoryOwners` fetches stages.yml over HTTP,
+        # so only build it when there are reports to enrich.
+        def resolve_categories
+          return [{}, {}] unless test_reports
+
+          # Comma-separated patterns, mirroring Artifacts#test_reports.
+          patterns = test_reports.split(',').map(&:strip).reject(&:empty?)
+          report_files = Dir.glob(patterns)
+          return [{}, {}] if report_files.empty?
+
+          [tests_to_categories_from(report_files), CategoryOwners.new.feature_categories_to_teams]
+        end
+
+        def tests_to_categories_from(report_files)
+          report_files.each_with_object({}) do |file, combined|
+            # Category enrichment is optional, so a malformed or vanished report
+            # shouldn't abort the per-test insert (the rows are the point). Warn
+            # and skip the file, leaving those tests with blank categories.
+            begin
+              report = TestReport.new(JSON.parse(File.read(file)))
+            rescue JSON::ParserError, Errno::ENOENT => e
+              logger.warn("#{ClickHouse::LOG_PREFIX} Skipping unreadable test report #{file}: #{e.message}")
+              next
+            end
+
+            combined.merge!(report.tests_to_categories) { |_, old_val, new_val| (old_val + new_val).uniq }
+          end
+        end
+
+        def clickhouse_credentials
+          clickhouse.merge(logger: logger)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.17.0"
+    VERSION = "3.18.0"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.17.0
+  version: 3.18.0
 platform: ruby
 authors:
 - GitLab Quality
@@ -502,6 +502,7 @@ files:
 - lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb
 - lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb
 - lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data.rb
+- lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config.rb
 - lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb