gitlab_quality-test_tooling 3.14.0 → 3.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 473fb18a44195986ea17fec58d277b67fd7589fca55a08250704687e2b70aa80
-  data.tar.gz: 4c18c0a11106a2a6103c1e5ba139dd68d1879ccb94360bddd77e1cf3beeafd34
+  metadata.gz: c8b4293811154f61f07fd8e35523a2406ba9832c6fe3e5eb3cf12164688df97a
+  data.tar.gz: c31082d6308fabe29ec51b4be09ebfd5d4c1d84e61ae8b3ce3741ffadcace9a6
 SHA512:
-  metadata.gz: ba2aac53d5ce9f33bba7791ddc5dfaae10dff68f118d0de6ab33ca9569b64e1a555f29ee9bb9d3d1b04d9f182e374dac27a3d75bd561531ceaa6c728f0a63545
-  data.tar.gz: 999efaaa3f6067aeb29b7c0cb00bf3ba8877f61fb108f8ef4213b1946e739a22249b77cf0a54feed33396ca6d5677920a34070d5b861a49535070f1cbc4bf464
+  metadata.gz: 1e522f9897eeddf1857caa10470ab1e11ae1ad7bad5062af80dbabdfb80e0fd11170f2011a2c28e9fcf5bb59394ca5124d9a88bedca637fbaec2e2fbf462503c
+  data.tar.gz: 1bb96de8d84feb07694145507df76707d88906b0671887b8104b79c3105859dda2c1d9ca6b68e0344478b955cdf91ec5f459ad6f1e48ca3612653b233c214531

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.14.0)
+    gitlab_quality-test_tooling (3.15.0)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/exe/test-coverage

@@ -10,13 +10,16 @@ require_relative "../lib/gitlab_quality/test_tooling"
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/category_owners'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregator'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/coverage_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/lcov_file'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/artifacts'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_report'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_map'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config'
@@ -63,6 +66,13 @@ options = OptionParser.new do |opts|
     params[:responsibility_patterns] = path
   end
 
+  opts.on('--per-test-coverage GLOB',
+    'Optional. Glob pattern for per-test coverage JSON files. ' \
+      'When provided, populates code_coverage.test_coverage_per_file and runs the ' \
+      'daily test_health_risk aggregation. (e.g., "tmp/per-test-coverage-*.json")') do |pattern|
+    params[:per_test_coverage] = pattern
+  end
+
   opts.separator ""
   opts.separator "Environment variables:"
   opts.separator "  GLCI_CLICKHOUSE_METRICS_PASSWORD   ClickHouse password (required, not passed via CLI for security)"
@@ -192,6 +202,27 @@ if params.any? && (required_params - params.keys).none?
   )
   test_file_mappings_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestFileMappingsTable.new(**shared_clickhouse_data)
   test_file_mappings_table.push(test_file_mapping_data.as_db_table)
+
+  # Per-test coverage export (optional). Only runs when --per-test-coverage
+  # was provided AND at least one matching artifact exists.
+  if params[:per_test_coverage]
+    per_test_files = Dir.glob(params[:per_test_coverage])
+    if per_test_files.any?
+      per_test_data = GitlabQuality::TestTooling::CodeCoverage::PerTestCoverageData.new(
+        per_test_files,
+        tests_to_categories: tests_to_categories,
+        feature_categories_to_teams: category_owners.feature_categories_to_teams,
+        captured_sha: ENV.fetch('CI_COMMIT_SHA', '')
+      )
+      per_test_coverage_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::PerTestCoverageTable.new(**clickhouse_data)
+      per_test_coverage_table.push(per_test_data.as_db_table)
+
+      aggregator = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestHealthRiskAggregator.new(**clickhouse_data)
+      aggregator.run
+    else
+      puts "No per-test coverage artifacts matched #{params[:per_test_coverage]}; skipping per-test export and aggregation."
+    end
+  end
 else
   puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
   puts options

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/client.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        # Memoized ClickHouse client accessor shared by `Table` and
+        # `TestHealthRiskAggregator`. Both classes need the same client
+        # construction from `@url` / `@database` / `@username` / `@password` /
+        # `@logger` instance variables; this module factors out the duplicated
+        # accessor without forcing one class to inherit from the other.
+        module Client
+          private
+
+          # @return [GitlabQuality::TestTooling::ClickHouse::Client]
+          def client
+            @client ||= GitlabQuality::TestTooling::ClickHouse::Client.new(
+              url: url,
+              database: database,
+              username: username,
+              password: password,
+              logger: logger
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        # Inserts per-test, per-source-file line-set coverage rows into
+        # `code_coverage.test_coverage_per_file`. The `covered_lines` column is
+        # `AggregateFunction(groupBitmap, UInt32)` so JSONEachRow can't carry it;
+        # rows go in via raw `INSERT ... VALUES` statements wrapping
+        # `bitmapBuild(CAST([line, ...] AS Array(UInt32)))` per row.
+        #
+        # Dedup across runs is handled by the table's
+        # `SharedReplacingMergeTree(version)` engine on
+        # `(ci_project_path, test_file, source_file)` ORDER BY. Within a single
+        # run, callers must pre-aggregate at the (test_file, source_file) grain
+        # before pushing: multiple examples within the same test_file should be
+        # unioned into one row by the loader, not handed in as duplicates.
+        class PerTestCoverageTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
+          TABLE_NAME = "test_coverage_per_file"
+          BATCH_SIZE = 500
+          # Intentionally generous ceiling on line numbers. Real source files
+          # are thousands of lines; generated artifacts (large GraphQL schemas,
+          # bundled JS, JSON manifests) can run past 100k. The cap is set to
+          # flag clearly bogus values (negative, garbage casts, anything past
+          # ~1M) without rejecting realistic generated files. ClickHouse's
+          # UInt32 ceiling is ~4.3B, so we still have orders of magnitude of
+          # headroom above this. Tighten only with evidence.
+          MAX_LINE_NUMBER = 1_000_000
+
+          # @param data [Array<Hash>] one entry per (test_file, source_file). Each entry needs:
+          #   :test_file [String]
+          #   :source_file [String]
+          #   :covered_lines [Array<Integer>] line numbers covered by this test on this file
+          #   :total_lines [Integer] executable lines in the source file
+          #   :feature_category, :group, :stage, :section [String, optional]
+          # @return [void]
+          def push(data) # rubocop:disable Metrics/AbcSize
+            return logger.warn("#{LOG_PREFIX} No data found, skipping ClickHouse export!") if data.empty?
+
+            logger.debug("#{LOG_PREFIX} Starting per-test coverage export to ClickHouse")
+            sanitized_data = sanitize(data)
+
+            return logger.warn("#{LOG_PREFIX} No valid data found after sanitization, skipping ClickHouse export!") if sanitized_data.empty?
+
+            total_batches = (sanitized_data.size.to_f / BATCH_SIZE).ceil
+            sanitized_data.each_slice(BATCH_SIZE).with_index do |batch, index|
+              logger.debug("#{LOG_PREFIX} Pushing batch #{index + 1} of #{total_batches} (#{batch.size} rows)")
+              client.query(build_insert_sql(batch), format: "TabSeparated")
+            end
+            logger.info("#{LOG_PREFIX} Successfully pushed #{sanitized_data.size} records to #{full_table_name}")
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Error occurred while pushing data to #{full_table_name}: #{e.message}")
+            raise
+          end
+
+          private
+
+          def valid_record?(record)
+            valid_test_file?(record) && valid_source_file?(record) && valid_covered_lines?(record)
+          end
+
+          def valid_test_file?(record)
+            return true unless record[:test_file].blank?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with nil/empty test_file: #{record}")
+            false
+          end
+
+          def valid_source_file?(record)
+            return true unless record[:source_file].blank?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with nil/empty source_file: #{record}")
+            false
+          end
+
+          def valid_covered_lines?(record)
+            covered = record[:covered_lines]
+            return true if covered.is_a?(Array) && !covered.empty?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with empty/invalid covered_lines: #{record[:test_file]} → #{record[:source_file]}")
+            false
+          end
+
+          def sanitized_data_record(record) # rubocop:disable Metrics/AbcSize
+            sanitized_lines = sanitize_lines(record[:covered_lines])
+
+            # `valid_covered_lines?` only checks the raw input is a non-empty
+            # Array. Post-sanitisation, every entry could still be filtered
+            # out (negatives, zeros, values past MAX_LINE_NUMBER). An empty
+            # `bitmapBuild([])` row carries no useful signal for the
+            # aggregation and just wastes a tuple, so drop the record here.
+            if sanitized_lines.empty?
+              logger.warn(
+                "#{LOG_PREFIX} Skipping record whose covered_lines emptied after sanitisation: " \
+                  "#{record[:test_file]} → #{record[:source_file]}"
+              )
+              return nil
+            end
+
+            {
+              timestamp: time,
+              ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil),
+              test_file: record[:test_file],
+              source_file: record[:source_file],
+              covered_lines: sanitized_lines,
+              total_lines: record[:total_lines].to_i,
+              feature_category: record[:feature_category] || '',
+              group: record[:group] || '',
+              stage: record[:stage] || '',
+              section: record[:section] || '',
+              captured_sha: record[:captured_sha].to_s
+            }
+          end
+
+          def build_insert_sql(batch)
+            rows_sql = batch.map { |record| build_row_sql(record) }.join(",\n")
+            <<~SQL
+              INSERT INTO #{full_table_name}
+                (timestamp, ci_project_path, test_file, source_file, covered_lines, total_lines, feature_category, `group`, stage, section, captured_sha)
+              VALUES
+              #{rows_sql}
+            SQL
+          end
+
+          # Precondition: `record[:covered_lines]` is the sanitised integer
+          # array produced upstream by `sanitize_lines` (via
+          # `sanitized_data_record`). Values are positive integers within
+          # MAX_LINE_NUMBER; no defensive validation is repeated here because
+          # this method is on the hot path (every row in every batch).
+          def build_row_sql(record) # rubocop:disable Metrics/AbcSize
+            line_array = "[#{record[:covered_lines].join(',')}]"
+            timestamp_str = record[:timestamp].iso8601(3)
+            "(" \
+              "'#{timestamp_str}', " \
+              "'#{escape(record[:ci_project_path])}', " \
+              "'#{escape(record[:test_file])}', " \
+              "'#{escape(record[:source_file])}', " \
+              "bitmapBuild(CAST(#{line_array} AS Array(UInt32))), " \
+              "#{record[:total_lines]}, " \
+              "'#{escape(record[:feature_category])}', " \
+              "'#{escape(record[:group])}', " \
+              "'#{escape(record[:stage])}', " \
+              "'#{escape(record[:section])}', " \
+              "'#{escape(record[:captured_sha])}'" \
+              ")"
+          end
+
+          # Filter line numbers down to positive integers within MAX_LINE_NUMBER.
+          # Drops anything that isn't a valid line number; doesn't raise so a
+          # single bad row doesn't fail the whole batch.
+          def sanitize_lines(lines)
+            Array(lines).filter_map do |line|
+              n = line.to_i
+              n if n.positive? && n <= MAX_LINE_NUMBER
+            end
+          end
+
+          # ClickHouse string escape: backslash and single quote.
+          def escape(str)
+            str.to_s.gsub(/\\/, '\\\\\\\\').gsub("'", "''") # rubocop:disable Style/RedundantRegexpArgument
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb

@@ -2,12 +2,18 @@
 
 require 'time'
 
+require_relative 'client'
+
 module GitlabQuality
   module TestTooling
     module CodeCoverage
       module ClickHouse
+        # Shared log prefix for all classes in this namespace. Hoisted up from
+        # individual classes so the prefix can change in one place.
+        LOG_PREFIX = "[CodeCoverage]" unless defined?(LOG_PREFIX)
+
         class Table
-          LOG_PREFIX = "[CodeCoverage]"
+          include Client
 
           def initialize(url:, database:, username: nil, password: nil, logger: nil)
             @url = url
@@ -74,17 +80,6 @@ module GitlabQuality
             logger.warn("#{LOG_PREFIX} Invalid CI_PIPELINE_CREATED_AT format: #{ci_created_at}, using current time")
             Time.now.utc
           end
-
-          # @return [GitlabQuality::TestTooling::ClickHouse::Client]
-          def client
-            @client ||= GitlabQuality::TestTooling::ClickHouse::Client.new(
-              url: url,
-              database: database,
-              username: username,
-              password: password,
-              logger: logger
-            )
-          end
         end
       end
     end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregation.sql

@@ -0,0 +1,123 @@
+-- Daily aggregation of per-test coverage data into per-group risk summary.
+--
+-- Reads the bitmap line sets from `code_coverage.test_coverage_per_file`,
+-- joins per-test quarantine and flaky status from `test_metrics`, then
+-- computes for each source file:
+--   quarantined_union        = union of line sets covered by *quarantined* tests
+--   flaky_union              = union of line sets covered by *flaky* tests
+--   quarantined_or_flaky_union = union of line sets covered by *quarantined OR flaky* tests
+--   healthy_union            = union of line sets covered by tests that are neither
+--
+-- The "at risk" line counts are:
+--   at_risk_lines_quarantine = lines uniquely covered by quarantined tests
+--                              (not covered by any healthy test)
+--   at_risk_lines_flaky      = lines uniquely covered by flaky tests
+--   at_risk_lines_combined   = lines uniquely covered by quarantined OR flaky
+--
+-- Healthy union excludes flaky tests too, by design: a flaky test isn't a
+-- reliable coverage safety net, so its lines shouldn't count toward the
+-- baseline against which quarantine-only risk is measured.
+--
+-- Result: one row per (snapshot_date, group, stage, section) tuple inserted
+-- into `code_coverage.test_health_risk_per_group`.
+--
+-- Parameter substitution: this template uses `{name}` braces (not %{name} or
+-- :name). They're replaced by literal text via `gsub` in
+-- TestHealthRiskAggregator#build_sql, after each parameter passes a regex
+-- check. A typoed placeholder name will silently fall through gsub as a
+-- literal — match the names exactly.
+--
+--   {snapshot_date}     : the date stamp for this run, e.g. '2026-05-07'
+--   {coverage_window}   : interval to look back for fresh per-test rows,
+--                         default '2 DAY' (see Resilience note below)
+--   {risk_window}       : interval to consider quarantine/flaky status from
+--                         the test_metrics summaries, default '30 DAY'
+
+-- Idempotency: this INSERT is unconditional. The target table must use
+-- SharedReplacingMergeTree(version) (or ReplacingMergeTree on non-Cloud
+-- ClickHouse; Cloud silently rewrites to Shared...) keyed by
+-- (snapshot_date, group, stage, section), with
+-- `version UInt64 MATERIALIZED toUnixTimestamp64Milli(now64(3))`, so re-running
+-- for the same snapshot_date produces a higher version that replaces the
+-- previous row on merge. Without that engine, re-runs would duplicate rows.
+--
+-- Resilience: `{coverage_window}` defaults to 2 DAY (see
+-- TestHealthRiskAggregator::DEFAULT_COVERAGE_WINDOW). One missed nightly
+-- run is recovered on the next night because the aggregation still sees
+-- the previous day's per-test rows. Cross-run race: if two jobs land
+-- inserts overlapping with the aggregation, the later aggregation wins
+-- because of the version-based replacement.
+INSERT INTO code_coverage.test_health_risk_per_group
+WITH
+  quarantine_status AS (
+    -- `date` is the daily aggregation timestamp on the test_metrics summary
+    -- view (when the per-test counts were rolled up), not the date a test
+    -- entered quarantine. We treat any test marked quarantined within the
+    -- risk window as currently quarantined.
+    SELECT
+      test_file,
+      uniqIfMerge(quarantined_cases) > 0 AS is_quarantined
+    FROM test_metrics.test_file_quarantine_summary
+    WHERE date >= now() - INTERVAL {risk_window}
+    GROUP BY test_file
+    HAVING is_quarantined
+  ),
+  flaky_status AS (
+    SELECT
+      test_file,
+      uniqIfMerge(flaky_cases) > 0 AS is_flaky
+    FROM test_metrics.test_file_flaky_summary
+    WHERE date >= now() - INTERVAL {risk_window}
+    GROUP BY test_file
+    HAVING is_flaky
+  ),
+  per_test_status AS (
+    SELECT
+      tc.source_file,
+      tc.`group`,
+      tc.stage,
+      tc.section,
+      tc.total_lines,
+      tc.covered_lines,
+      coalesce(qs.is_quarantined, FALSE) AS is_quarantined,
+      coalesce(fs.is_flaky, FALSE)       AS is_flaky
+    FROM code_coverage.test_coverage_per_file tc FINAL
+    LEFT JOIN quarantine_status qs ON qs.test_file = tc.test_file
+    LEFT JOIN flaky_status      fs ON fs.test_file = tc.test_file
+    WHERE tc.timestamp >= now() - INTERVAL {coverage_window}
+  ),
+  per_file AS (
+    SELECT
+      source_file,
+      `group`,
+      stage,
+      section,
+      max(total_lines) AS total_lines,
+      groupBitmapMergeStateIf(covered_lines, is_quarantined)                      AS quarantined_union,
+      groupBitmapMergeStateIf(covered_lines, is_flaky)                            AS flaky_union,
+      groupBitmapMergeStateIf(covered_lines, is_quarantined OR is_flaky)          AS quarantined_or_flaky_union,
+      groupBitmapMergeStateIf(covered_lines, NOT is_quarantined AND NOT is_flaky) AS healthy_union
+    FROM per_test_status
+    GROUP BY source_file, `group`, stage, section
+  )
+SELECT
+  toDate('{snapshot_date}') AS snapshot_date,
+  `group`,
+  stage,
+  section,
+  count(*) AS source_file_count,
+  -- at_risk_lines_combined ≤ at_risk_lines_quarantine + at_risk_lines_flaky.
+  -- A line covered by both a quarantined and a flaky test is single-counted
+  -- in quarantined_or_flaky_union, so the combined column is the *unique*
+  -- exclusive-line loss across both risk sources, not their sum.
+  sum(bitmapCardinality(bitmapAndnot(quarantined_union,          healthy_union))) AS at_risk_lines_quarantine,
+  sum(bitmapCardinality(bitmapAndnot(flaky_union,                healthy_union))) AS at_risk_lines_flaky,
+  sum(bitmapCardinality(bitmapAndnot(quarantined_or_flaky_union, healthy_union))) AS at_risk_lines_combined,
+  -- team_executable_lines is the per-team executable-line denominator across
+  -- every (source_file, team) row owned by the team. A source file owned by
+  -- tests in two different (group, stage, section) tuples contributes to each
+  -- team's total separately, so summing this column across teams is NOT the
+  -- codebase-wide line count.
+  sum(total_lines) AS team_executable_lines
+FROM per_file
+GROUP BY `group`, stage, section;

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregator.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative 'client'
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        # Runs the daily aggregation that turns `code_coverage.test_coverage_per_file`
+        # rows into a small `code_coverage.test_health_risk_per_group` summary
+        # the dashboard reads.
+        #
+        # Hybrid model: this Ruby class is the orchestrator (schedule, error
+        # handling, parameter substitution); ClickHouse runs the bitmap math
+        # via `INSERT ... SELECT` from the SQL file shipped alongside.
+        class TestHealthRiskAggregator
+          include Client
+
+          SQL_FILE = File.expand_path('test_health_risk_aggregation.sql', __dir__)
+
+          # 2 DAY rather than 1 DAY makes the aggregation self-healing across
+          # a single missed nightly run: if last night's export failed, this
+          # night's run still sees yesterday's per-test rows and produces a
+          # current snapshot. ReplacingMergeTree FINAL on the source table
+          # ensures we read only the latest version per (test_file, source_file).
+          # ClickHouse accepts both '2 DAY' and '2 DAYS'; we use the singular
+          # form for consistency with `30 DAY` below.
+          DEFAULT_COVERAGE_WINDOW = '2 DAY'
+          DEFAULT_RISK_WINDOW = '30 DAY'
+
+          # `snapshot_date` is YYYY-MM-DD; intervals are `<integer> <unit>`
+          # (singular or plural).
+          DATE_PATTERN = /\A\d{4}-\d{2}-\d{2}\z/
+          INTERVAL_PATTERN = /\A\d+\s+(SECOND|MINUTE|HOUR|DAY|WEEK|MONTH|QUARTER|YEAR)S?\z/i
+
+          def initialize(
+            url:, database:, username: nil, password: nil, logger: nil,
+            coverage_window: DEFAULT_COVERAGE_WINDOW, risk_window: DEFAULT_RISK_WINDOW)
+            @url = url
+            @database = database
+            @username = username
+            @password = password
+            @logger = logger || ::Logger.new($stdout, level: 1)
+            @coverage_window = coverage_window
+            @risk_window = risk_window
+          end
+
+          # @param snapshot_date [Date, String] date stamp for this run; defaults to today.
+          # @return [void]
+          def run(snapshot_date: Date.today) # rubocop:disable Metrics/AbcSize
+            sql = build_sql(snapshot_date: snapshot_date)
+            logger.info(
+              "#{LOG_PREFIX} Running test_health_risk aggregation snapshot_date=#{snapshot_date} " \
+                "coverage_window=#{coverage_window} risk_window=#{risk_window}"
+            )
+            client.query(sql, format: "TabSeparated")
+            inserted = fetch_row_count(snapshot_date)
+            if inserted.is_a?(Integer) && inserted.zero?
+              logger.warn(
+                "#{LOG_PREFIX} Aggregation wrote 0 rows for snapshot_date=#{snapshot_date}. " \
+                  "This is valid if no per-test data is in the coverage_window, but worth checking " \
+                  "test_coverage_per_file directly if a recent export ran."
+              )
+            else
+              logger.info("#{LOG_PREFIX} Aggregation wrote #{inserted} rows for snapshot_date=#{snapshot_date}")
+            end
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Aggregation failed for #{snapshot_date}: #{e.message}")
+            raise
+          end
+
+          private
+
+          attr_reader :url, :database, :username, :password, :logger, :coverage_window, :risk_window
+
+          def build_sql(snapshot_date:)
+            template = File.read(SQL_FILE)
+            template
+              .gsub('{snapshot_date}', validate_date(snapshot_date))
+              .gsub('{coverage_window}', validate_interval(coverage_window))
+              .gsub('{risk_window}', validate_interval(risk_window))
+          end
+
+          # DateTime/Time `to_s` includes the time portion and is rejected.
+          def validate_date(value)
+            str = value.to_s
+            raise ArgumentError, "Invalid snapshot_date: #{value.inspect}" unless DATE_PATTERN.match?(str)
+
+            str
+          end
+
+          def validate_interval(value)
+            raise ArgumentError, "Invalid interval expression: #{value.inspect}" unless INTERVAL_PATTERN.match?(value.to_s)
+
+            value
+          end
+
+          # Returns 'unknown' on any error so a transient count-query failure
+          # can't mask the success of the actual INSERT.
+          def fetch_row_count(snapshot_date)
+            count_sql = "SELECT count() FROM code_coverage.test_health_risk_per_group FINAL " \
+              "WHERE snapshot_date = toDate('#{validate_date(snapshot_date)}')"
+            result = client.query(count_sql, format: "JSONCompact")
+            result&.dig('data', 0, 0) || 'unknown'
+          rescue StandardError => e
+            logger.debug("#{LOG_PREFIX} Could not fetch post-aggregation row count: #{e.message}")
+            'unknown'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      # Reads per-test coverage files and produces rows for
+      # `PerTestCoverageTable`.
+      #
+      # Two input formats are supported, dispatched by file extension:
+      #
+      # `.json`: one document with the example id as the outer key.
+      #   {
+      #     "spec/path/to/test_spec.rb[1:1]": {
+      #       "app/path/to/source.rb": [null, 1, 0, 5, 1, ...]
+      #     },
+      #     ...
+      #   }
+      #
+      # `.ndjson`: one JSON object per line, with `id` and `files` fields.
+      #   {"id":"spec/path/to/test_spec.rb[1:1]","files":{"app/path/to/source.rb":[null,1,0,5,1]}}
+      #   {"id":"spec/path/to/test_spec.rb[1:2]","files":{"app/path/to/source.rb":[null,0,1,0,1]}}
+      #
+      # The NDJSON form lets the producing formatter stream per-example data
+      # to disk without holding the full suite in memory. Both forms carry
+      # the same per-test data; the parser is symmetric.
+      #
+      # Inner key (in either form) is a source file path. Inner value is a
+      # 0-indexed array of per-line hit counts. `null` means non-executable;
+      # `0` means executable but not hit by this test; positive integer means
+      # executed. This is the standard Ruby `Coverage` module output shape,
+      # also produced by any per-test capture that emits one line-hit array
+      # per (test, file) pair.
+      #
+      # This class:
+      #   - strips `[<example_uid>]` from the example id to get a per-test-file key
+      #   - converts each line-hit array into a (covered_lines, total_lines) pair
+      #   - pre-aggregates within (test_file, source_file): unions covered
+      #     lines across all examples in the same test file, takes the max
+      #     total_lines
+      #   - drops rows with empty bitmaps (file imported but no line hit)
+      #   - enriches with feature_category / group / stage / section when test
+      #     metadata is provided
+      class PerTestCoverageData
+        # Raised when a coverage artifact can't be parsed. Wraps the underlying
+        # `JSON::ParserError` or `Errno::ENOENT` so callers outside the
+        # gitlab-org/gitlab CI context (where upstream `needs:` ordering
+        # guarantees well-formed artifacts) can rescue precisely without
+        # catching unrelated standard exceptions.
+        ParseError = Class.new(StandardError)
+
+        # @param coverage_files [Array<String>] paths to per-test coverage JSON artifacts
+        # @param tests_to_categories [Hash<String, Array<String>>] test_file => [feature_category]
+        # @param feature_categories_to_teams [Hash<String, Hash>] category => {group:, stage:, section:}
+        # @param captured_sha [String] the git SHA the coverage was captured against; attached to
+        #   every emitted row so downstream delta-capture jobs can ask
+        #   `SELECT max(captured_sha) FROM code_coverage.test_coverage_per_file` to find the
+        #   previous successful capture point. Defaults to '' when unknown.
+        # @raise [ParseError] if a coverage file is missing or contains invalid JSON
+        def initialize(coverage_files, tests_to_categories: {}, feature_categories_to_teams: {}, captured_sha: '')
+          @coverage_files = Array(coverage_files)
+          @tests_to_categories = tests_to_categories
+          @feature_categories_to_teams = feature_categories_to_teams
+          @captured_sha = captured_sha.to_s
+        end
+
+        # @return [Array<Hash<Symbol, Object>>] per-test-file, per-source-file rows for PerTestCoverageTable
+        def as_db_table # rubocop:disable Metrics/AbcSize
+          aggregated = {}
+
+          @coverage_files.each do |path|
+            each_example(path) do |example_id, files|
+              test_file = extract_test_file_path(example_id)
+              files.each do |source_file, line_hits|
+                covered, total = parse_line_hits(line_hits)
+                next if covered.empty?
+
+                key = [test_file, source_file]
+                if aggregated.key?(key)
+                  aggregated[key][:covered_lines].merge(covered)
+                  # max rather than picking either side: examples within the
+                  # same test file may report arrays of different lengths if
+                  # the source file was edited mid-run. Pragmatic, not exact.
+                  aggregated[key][:total_lines] = [aggregated[key][:total_lines], total].max
+                else
+                  # dup so the merge above can never alias a Set returned by
+                  # parse_line_hits to a different key later in the loop.
+                  aggregated[key] = { covered_lines: covered.dup, total_lines: total }
+                end
+              end
+            end
+          end
+
+          aggregated.map do |(test_file, source_file), agg|
+            category = @tests_to_categories[test_file]&.first || ''
+            team = @feature_categories_to_teams[category] || {}
+
+            {
+              test_file: test_file,
+              source_file: source_file,
+              covered_lines: agg[:covered_lines].to_a.sort,
+              total_lines: agg[:total_lines],
+              feature_category: category,
+              group: team[:group] || '',
+              stage: team[:stage] || '',
+              section: team[:section] || '',
+              captured_sha: @captured_sha
+            }
+          end
+        end
+
+        private
+
+        # Yield (example_id, files) pairs from one input file. Dispatches
+        # on extension: `.ndjson` is parsed line-by-line so a multi-GB
+        # capture file does not need to fit in memory; everything else is
+        # parsed as a single JSON document with `{example_id => files}` at
+        # the top level. Both forms wrap parse failures in `ParseError` so
+        # callers can rescue without naming the underlying exception classes.
+        def each_example(path)
+          if path.end_with?('.ndjson')
+            File.foreach(path) do |line|
+              line = line.strip
+              next if line.empty?
+
+              entry = JSON.parse(line)
+              yield entry.fetch('id'), entry.fetch('files')
+            end
+          else
+            JSON.parse(File.read(path)).each do |example_id, files|
+              yield example_id, files
+            end
+          end
+        rescue JSON::ParserError, Errno::ENOENT => e
+          raise ParseError, "Failed to parse coverage artifact #{path}: #{e.message}"
+        end
+
+        # Normalise an example id (`<test_file>[<uid>]:<line>`) to the
+        # bare test_file path. Strips a leading `./`, the trailing
+        # `[<uid>]`, and any `:<line>` suffix so multiple examples within
+        # the same spec file collapse to the same test_file key. Path
+        # separator handling assumes Linux paths (everything in CI is
+        # Linux); a Windows-style `C:/foo/spec.rb` would split incorrectly
+        # on the first `:`.
+        def extract_test_file_path(example_id)
+          stripped = example_id.delete_prefix('./')
+          stripped = stripped.sub(/\[.+\]\z/, '')
+          stripped.split(':').first
+        end
+
+        # Walk the per-line hit array. Returns (Set<Integer> covered_lines,
+        # Integer total_executable_lines). Indexes are 0-based; line numbers
+        # are 1-based. `nil` entries are non-executable lines.
+        # A `nil` value at the file level (file loaded but never recorded
+        # under this example) yields an empty result rather than raising,
+        # so a single odd cell doesn't fail the whole export.
+        def parse_line_hits(line_hits)
+          return [Set.new, 0] unless line_hits.is_a?(Array)
+
+          covered = Set.new
+          total = 0
+          line_hits.each_with_index do |hits, index|
+            next if hits.nil?
+
+            total += 1
+            covered.add(index + 1) if hits.is_a?(Numeric) && hits.positive?
+          end
+          [covered, total]
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.14.0"
+    VERSION = "3.15.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.14.0
+  version: 3.15.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-09 00:00:00.000000000 Z
+date: 2026-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -491,11 +491,16 @@ files:
 - lib/gitlab_quality/test_tooling/code_coverage/artifacts.rb
 - lib/gitlab_quality/test_tooling/code_coverage/category_owners.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/client.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregation.sql
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregator.rb
 - lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb
 - lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb
+- lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config.rb
 - lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb