gitlab_quality-test_tooling 3.13.0 → 3.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1284d833aad5c38a45cff835cf3e701193117aa9694293b42fee9ce60345b0d
-  data.tar.gz: e209cf22987a7a19c58a4675ad529a6910ec387117a49fbcef97aadef9230c6f
+  metadata.gz: c8b4293811154f61f07fd8e35523a2406ba9832c6fe3e5eb3cf12164688df97a
+  data.tar.gz: c31082d6308fabe29ec51b4be09ebfd5d4c1d84e61ae8b3ce3741ffadcace9a6
 SHA512:
-  metadata.gz: '09ded437ab350eddf3736017e844865ef0a5ca311c6d62e800b87b272e691eab904ae12c6241f202f76ae1afa1e717880b99984e640fbd9728791e684fbfbbed'
-  data.tar.gz: a9064f95ca94ef8d8d476f097f3ee9412d71dca931a715e9f446a81f8646e75e14ab141b93fe83ebd1bf898d32503b4806179ac2f9b3233fe69c95024c693491
+  metadata.gz: 1e522f9897eeddf1857caa10470ab1e11ae1ad7bad5062af80dbabdfb80e0fd11170f2011a2c28e9fcf5bb59394ca5124d9a88bedca637fbaec2e2fbf462503c
+  data.tar.gz: 1bb96de8d84feb07694145507df76707d88906b0671887b8104b79c3105859dda2c1d9ca6b68e0344478b955cdf91ec5f459ad6f1e48ca3612653b233c214531

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.13.0)
+    gitlab_quality-test_tooling (3.15.0)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/exe/test-coverage

@@ -10,13 +10,16 @@ require_relative "../lib/gitlab_quality/test_tooling"
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/category_owners'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregator'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/coverage_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/lcov_file'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/artifacts'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_report'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_map'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config'
@@ -63,6 +66,13 @@ options = OptionParser.new do |opts|
     params[:responsibility_patterns] = path
   end
 
+  opts.on('--per-test-coverage GLOB',
+    'Optional. Glob pattern for per-test coverage JSON files. ' \
+      'When provided, populates code_coverage.test_coverage_per_file and runs the ' \
+      'daily test_health_risk aggregation. (e.g., "tmp/per-test-coverage-*.json")') do |pattern|
+    params[:per_test_coverage] = pattern
+  end
+
   opts.separator ""
   opts.separator "Environment variables:"
   opts.separator "  GLCI_CLICKHOUSE_METRICS_PASSWORD   ClickHouse password (required, not passed via CLI for security)"
@@ -192,6 +202,27 @@ if params.any? && (required_params - params.keys).none?
   )
   test_file_mappings_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestFileMappingsTable.new(**shared_clickhouse_data)
   test_file_mappings_table.push(test_file_mapping_data.as_db_table)
+
+  # Per-test coverage export (optional). Only runs when --per-test-coverage
+  # was provided AND at least one matching artifact exists.
+  if params[:per_test_coverage]
+    per_test_files = Dir.glob(params[:per_test_coverage])
+    if per_test_files.any?
+      per_test_data = GitlabQuality::TestTooling::CodeCoverage::PerTestCoverageData.new(
+        per_test_files,
+        tests_to_categories: tests_to_categories,
+        feature_categories_to_teams: category_owners.feature_categories_to_teams,
+        captured_sha: ENV.fetch('CI_COMMIT_SHA', '')
+      )
+      per_test_coverage_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::PerTestCoverageTable.new(**clickhouse_data)
+      per_test_coverage_table.push(per_test_data.as_db_table)
+
+      aggregator = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestHealthRiskAggregator.new(**clickhouse_data)
+      aggregator.run
+    else
+      puts "No per-test coverage artifacts matched #{params[:per_test_coverage]}; skipping per-test export and aggregation."
+    end
+  end
 else
   puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
   puts options

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/client.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        # Memoized ClickHouse client accessor shared by `Table` and
+        # `TestHealthRiskAggregator`. Both classes need the same client
+        # construction from `@url` / `@database` / `@username` / `@password` /
+        # `@logger` instance variables; this module factors out the duplicated
+        # accessor without forcing one class to inherit from the other.
+        module Client
+          private
+
+          # @return [GitlabQuality::TestTooling::ClickHouse::Client]
+          def client
+            @client ||= GitlabQuality::TestTooling::ClickHouse::Client.new(
+              url: url,
+              database: database,
+              username: username,
+              password: password,
+              logger: logger
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        # Inserts per-test, per-source-file line-set coverage rows into
+        # `code_coverage.test_coverage_per_file`. The `covered_lines` column is
+        # `AggregateFunction(groupBitmap, UInt32)` so JSONEachRow can't carry it;
+        # rows go in via raw `INSERT ... VALUES` statements wrapping
+        # `bitmapBuild(CAST([line, ...] AS Array(UInt32)))` per row.
+        #
+        # Dedup across runs is handled by the table's
+        # `SharedReplacingMergeTree(version)` engine on
+        # `(ci_project_path, test_file, source_file)` ORDER BY. Within a single
+        # run, callers must pre-aggregate at the (test_file, source_file) grain
+        # before pushing: multiple examples within the same test_file should be
+        # unioned into one row by the loader, not handed in as duplicates.
+        class PerTestCoverageTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
+          TABLE_NAME = "test_coverage_per_file"
+          BATCH_SIZE = 500
+          # Intentionally generous ceiling on line numbers. Real source files
+          # are thousands of lines; generated artifacts (large GraphQL schemas,
+          # bundled JS, JSON manifests) can run past 100k. The cap is set to
+          # flag clearly bogus values (negative, garbage casts, anything past
+          # ~1M) without rejecting realistic generated files. ClickHouse's
+          # UInt32 ceiling is ~4.3B, so we still have orders of magnitude of
+          # headroom above this. Tighten only with evidence.
+          MAX_LINE_NUMBER = 1_000_000
+
+          # @param data [Array<Hash>] one entry per (test_file, source_file). Each entry needs:
+          #   :test_file [String]
+          #   :source_file [String]
+          #   :covered_lines [Array<Integer>] line numbers covered by this test on this file
+          #   :total_lines [Integer] executable lines in the source file
+          #   :feature_category, :group, :stage, :section [String, optional]
+          # @return [void]
+          def push(data) # rubocop:disable Metrics/AbcSize
+            return logger.warn("#{LOG_PREFIX} No data found, skipping ClickHouse export!") if data.empty?
+
+            logger.debug("#{LOG_PREFIX} Starting per-test coverage export to ClickHouse")
+            sanitized_data = sanitize(data)
+
+            return logger.warn("#{LOG_PREFIX} No valid data found after sanitization, skipping ClickHouse export!") if sanitized_data.empty?
+
+            total_batches = (sanitized_data.size.to_f / BATCH_SIZE).ceil
+            sanitized_data.each_slice(BATCH_SIZE).with_index do |batch, index|
+              logger.debug("#{LOG_PREFIX} Pushing batch #{index + 1} of #{total_batches} (#{batch.size} rows)")
+              client.query(build_insert_sql(batch), format: "TabSeparated")
+            end
+            logger.info("#{LOG_PREFIX} Successfully pushed #{sanitized_data.size} records to #{full_table_name}")
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Error occurred while pushing data to #{full_table_name}: #{e.message}")
+            raise
+          end
+
+          private
+
+          def valid_record?(record)
+            valid_test_file?(record) && valid_source_file?(record) && valid_covered_lines?(record)
+          end
+
+          def valid_test_file?(record)
+            return true unless record[:test_file].blank?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with nil/empty test_file: #{record}")
+            false
+          end
+
+          def valid_source_file?(record)
+            return true unless record[:source_file].blank?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with nil/empty source_file: #{record}")
+            false
+          end
+
+          def valid_covered_lines?(record)
+            covered = record[:covered_lines]
+            return true if covered.is_a?(Array) && !covered.empty?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with empty/invalid covered_lines: #{record[:test_file]} → #{record[:source_file]}")
+            false
+          end
+
+          def sanitized_data_record(record) # rubocop:disable Metrics/AbcSize
+            sanitized_lines = sanitize_lines(record[:covered_lines])
+
+            # `valid_covered_lines?` only checks the raw input is a non-empty
+            # Array. Post-sanitisation, every entry could still be filtered
+            # out (negatives, zeros, values past MAX_LINE_NUMBER). An empty
+            # `bitmapBuild([])` row carries no useful signal for the
+            # aggregation and just wastes a tuple, so drop the record here.
+            if sanitized_lines.empty?
+              logger.warn(
+                "#{LOG_PREFIX} Skipping record whose covered_lines emptied after sanitisation: " \
+                  "#{record[:test_file]} → #{record[:source_file]}"
+              )
+              return nil
+            end
+
+            {
+              timestamp: time,
+              ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil),
+              test_file: record[:test_file],
+              source_file: record[:source_file],
+              covered_lines: sanitized_lines,
+              total_lines: record[:total_lines].to_i,
+              feature_category: record[:feature_category] || '',
+              group: record[:group] || '',
+              stage: record[:stage] || '',
+              section: record[:section] || '',
+              captured_sha: record[:captured_sha].to_s
+            }
+          end
+
+          def build_insert_sql(batch)
+            rows_sql = batch.map { |record| build_row_sql(record) }.join(",\n")
+            <<~SQL
+              INSERT INTO #{full_table_name}
+                (timestamp, ci_project_path, test_file, source_file, covered_lines, total_lines, feature_category, `group`, stage, section, captured_sha)
+              VALUES
+              #{rows_sql}
+            SQL
+          end
+
+          # Precondition: `record[:covered_lines]` is the sanitised integer
+          # array produced upstream by `sanitize_lines` (via
+          # `sanitized_data_record`). Values are positive integers within
+          # MAX_LINE_NUMBER; no defensive validation is repeated here because
+          # this method is on the hot path (every row in every batch).
+          def build_row_sql(record) # rubocop:disable Metrics/AbcSize
+            line_array = "[#{record[:covered_lines].join(',')}]"
+            timestamp_str = record[:timestamp].iso8601(3)
+            "(" \
+              "'#{timestamp_str}', " \
+              "'#{escape(record[:ci_project_path])}', " \
+              "'#{escape(record[:test_file])}', " \
+              "'#{escape(record[:source_file])}', " \
+              "bitmapBuild(CAST(#{line_array} AS Array(UInt32))), " \
+              "#{record[:total_lines]}, " \
+              "'#{escape(record[:feature_category])}', " \
+              "'#{escape(record[:group])}', " \
+              "'#{escape(record[:stage])}', " \
+              "'#{escape(record[:section])}', " \
+              "'#{escape(record[:captured_sha])}'" \
+              ")"
+          end
+
+          # Filter line numbers down to positive integers within MAX_LINE_NUMBER.
+          # Drops anything that isn't a valid line number; doesn't raise so a
+          # single bad row doesn't fail the whole batch.
+          def sanitize_lines(lines)
+            Array(lines).filter_map do |line|
+              n = line.to_i
+              n if n.positive? && n <= MAX_LINE_NUMBER
+            end
+          end
+
+          # ClickHouse string escape: backslash and single quote.
+          def escape(str)
+            str.to_s.gsub(/\\/, '\\\\\\\\').gsub("'", "''") # rubocop:disable Style/RedundantRegexpArgument
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb

@@ -2,12 +2,18 @@
 
 require 'time'
 
+require_relative 'client'
+
 module GitlabQuality
   module TestTooling
     module CodeCoverage
       module ClickHouse
+        # Shared log prefix for all classes in this namespace. Hoisted up from
+        # individual classes so the prefix can change in one place.
+        LOG_PREFIX = "[CodeCoverage]" unless defined?(LOG_PREFIX)
+
         class Table
-          LOG_PREFIX = "[CodeCoverage]"
+          include Client
 
           def initialize(url:, database:, username: nil, password: nil, logger: nil)
             @url = url
@@ -74,17 +80,6 @@ module GitlabQuality
             logger.warn("#{LOG_PREFIX} Invalid CI_PIPELINE_CREATED_AT format: #{ci_created_at}, using current time")
             Time.now.utc
           end
-
-          # @return [GitlabQuality::TestTooling::ClickHouse::Client]
-          def client
-            @client ||= GitlabQuality::TestTooling::ClickHouse::Client.new(
-              url: url,
-              database: database,
-              username: username,
-              password: password,
-              logger: logger
-            )
-          end
         end
       end
     end

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregation.sql

@@ -0,0 +1,123 @@
+-- Daily aggregation of per-test coverage data into per-group risk summary.
+--
+-- Reads the bitmap line sets from `code_coverage.test_coverage_per_file`,
+-- joins per-test quarantine and flaky status from `test_metrics`, then
+-- computes for each source file:
+--   quarantined_union        = union of line sets covered by *quarantined* tests
+--   flaky_union              = union of line sets covered by *flaky* tests
+--   quarantined_or_flaky_union = union of line sets covered by *quarantined OR flaky* tests
+--   healthy_union            = union of line sets covered by tests that are neither
+--
+-- The "at risk" line counts are:
+--   at_risk_lines_quarantine = lines uniquely covered by quarantined tests
+--                              (not covered by any healthy test)
+--   at_risk_lines_flaky      = lines uniquely covered by flaky tests
+--   at_risk_lines_combined   = lines uniquely covered by quarantined OR flaky
+--
+-- Healthy union excludes flaky tests too, by design: a flaky test isn't a
+-- reliable coverage safety net, so its lines shouldn't count toward the
+-- baseline against which quarantine-only risk is measured.
+--
+-- Result: one row per (snapshot_date, group, stage, section) tuple inserted
+-- into `code_coverage.test_health_risk_per_group`.
+--
+-- Parameter substitution: this template uses `{name}` braces (not %{name} or
+-- :name). They're replaced by literal text via `gsub` in
+-- TestHealthRiskAggregator#build_sql, after each parameter passes a regex
+-- check. A typoed placeholder name will silently fall through gsub as a
+-- literal — match the names exactly.
+--
+--   {snapshot_date}     : the date stamp for this run, e.g. '2026-05-07'
+--   {coverage_window}   : interval to look back for fresh per-test rows,
+--                         default '2 DAY' (see Resilience note below)
+--   {risk_window}       : interval to consider quarantine/flaky status from
+--                         the test_metrics summaries, default '30 DAY'
+
+-- Idempotency: this INSERT is unconditional. The target table must use
+-- SharedReplacingMergeTree(version) (or ReplacingMergeTree on non-Cloud
+-- ClickHouse; Cloud silently rewrites to Shared...) keyed by
+-- (snapshot_date, group, stage, section), with
+-- `version UInt64 MATERIALIZED toUnixTimestamp64Milli(now64(3))`, so re-running
+-- for the same snapshot_date produces a higher version that replaces the
+-- previous row on merge. Without that engine, re-runs would duplicate rows.
+--
+-- Resilience: `{coverage_window}` defaults to 2 DAY (see
+-- TestHealthRiskAggregator::DEFAULT_COVERAGE_WINDOW). One missed nightly
+-- run is recovered on the next night because the aggregation still sees
+-- the previous day's per-test rows. Cross-run race: if two jobs land
+-- inserts overlapping with the aggregation, the later aggregation wins
+-- because of the version-based replacement.
+INSERT INTO code_coverage.test_health_risk_per_group
+WITH
+  quarantine_status AS (
+    -- `date` is the daily aggregation timestamp on the test_metrics summary
+    -- view (when the per-test counts were rolled up), not the date a test
+    -- entered quarantine. We treat any test marked quarantined within the
+    -- risk window as currently quarantined.
+    SELECT
+      test_file,
+      uniqIfMerge(quarantined_cases) > 0 AS is_quarantined
+    FROM test_metrics.test_file_quarantine_summary
+    WHERE date >= now() - INTERVAL {risk_window}
+    GROUP BY test_file
+    HAVING is_quarantined
+  ),
+  flaky_status AS (
+    SELECT
+      test_file,
+      uniqIfMerge(flaky_cases) > 0 AS is_flaky
+    FROM test_metrics.test_file_flaky_summary
+    WHERE date >= now() - INTERVAL {risk_window}
+    GROUP BY test_file
+    HAVING is_flaky
+  ),
+  per_test_status AS (
+    SELECT
+      tc.source_file,
+      tc.`group`,
+      tc.stage,
+      tc.section,
+      tc.total_lines,
+      tc.covered_lines,
+      coalesce(qs.is_quarantined, FALSE) AS is_quarantined,
+      coalesce(fs.is_flaky, FALSE)       AS is_flaky
+    FROM code_coverage.test_coverage_per_file tc FINAL
+    LEFT JOIN quarantine_status qs ON qs.test_file = tc.test_file
+    LEFT JOIN flaky_status      fs ON fs.test_file = tc.test_file
+    WHERE tc.timestamp >= now() - INTERVAL {coverage_window}
+  ),
+  per_file AS (
+    SELECT
+      source_file,
+      `group`,
+      stage,
+      section,
+      max(total_lines) AS total_lines,
+      groupBitmapMergeStateIf(covered_lines, is_quarantined)                      AS quarantined_union,
+      groupBitmapMergeStateIf(covered_lines, is_flaky)                            AS flaky_union,
+      groupBitmapMergeStateIf(covered_lines, is_quarantined OR is_flaky)          AS quarantined_or_flaky_union,
+      groupBitmapMergeStateIf(covered_lines, NOT is_quarantined AND NOT is_flaky) AS healthy_union
+    FROM per_test_status
+    GROUP BY source_file, `group`, stage, section
+  )
+SELECT
+  toDate('{snapshot_date}') AS snapshot_date,
+  `group`,
+  stage,
+  section,
+  count(*) AS source_file_count,
+  -- at_risk_lines_combined ≤ at_risk_lines_quarantine + at_risk_lines_flaky.
+  -- A line covered by both a quarantined and a flaky test is single-counted
+  -- in quarantined_or_flaky_union, so the combined column is the *unique*
+  -- exclusive-line loss across both risk sources, not their sum.
+  sum(bitmapCardinality(bitmapAndnot(quarantined_union,          healthy_union))) AS at_risk_lines_quarantine,
+  sum(bitmapCardinality(bitmapAndnot(flaky_union,                healthy_union))) AS at_risk_lines_flaky,
+  sum(bitmapCardinality(bitmapAndnot(quarantined_or_flaky_union, healthy_union))) AS at_risk_lines_combined,
+  -- team_executable_lines is the per-team executable-line denominator across
+  -- every (source_file, team) row owned by the team. A source file owned by
+  -- tests in two different (group, stage, section) tuples contributes to each
+  -- team's total separately, so summing this column across teams is NOT the
+  -- codebase-wide line count.
+  sum(total_lines) AS team_executable_lines
+FROM per_file
+GROUP BY `group`, stage, section;

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregator.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative 'client'
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        # Runs the daily aggregation that turns `code_coverage.test_coverage_per_file`
+        # rows into a small `code_coverage.test_health_risk_per_group` summary
+        # the dashboard reads.
+        #
+        # Hybrid model: this Ruby class is the orchestrator (schedule, error
+        # handling, parameter substitution); ClickHouse runs the bitmap math
+        # via `INSERT ... SELECT` from the SQL file shipped alongside.
+        class TestHealthRiskAggregator
+          include Client
+
+          SQL_FILE = File.expand_path('test_health_risk_aggregation.sql', __dir__)
+
+          # 2 DAY rather than 1 DAY makes the aggregation self-healing across
+          # a single missed nightly run: if last night's export failed, this
+          # night's run still sees yesterday's per-test rows and produces a
+          # current snapshot. ReplacingMergeTree FINAL on the source table
+          # ensures we read only the latest version per (test_file, source_file).
+          # ClickHouse accepts both '2 DAY' and '2 DAYS'; we use the singular
+          # form for consistency with `30 DAY` below.
+          DEFAULT_COVERAGE_WINDOW = '2 DAY'
+          DEFAULT_RISK_WINDOW = '30 DAY'
+
+          # `snapshot_date` is YYYY-MM-DD; intervals are `<integer> <unit>`
+          # (singular or plural).
+          DATE_PATTERN = /\A\d{4}-\d{2}-\d{2}\z/
+          INTERVAL_PATTERN = /\A\d+\s+(SECOND|MINUTE|HOUR|DAY|WEEK|MONTH|QUARTER|YEAR)S?\z/i
+
+          def initialize(
+            url:, database:, username: nil, password: nil, logger: nil,
+            coverage_window: DEFAULT_COVERAGE_WINDOW, risk_window: DEFAULT_RISK_WINDOW)
+            @url = url
+            @database = database
+            @username = username
+            @password = password
+            @logger = logger || ::Logger.new($stdout, level: 1)
+            @coverage_window = coverage_window
+            @risk_window = risk_window
+          end
+
+          # @param snapshot_date [Date, String] date stamp for this run; defaults to today.
+          # @return [void]
+          def run(snapshot_date: Date.today) # rubocop:disable Metrics/AbcSize
+            sql = build_sql(snapshot_date: snapshot_date)
+            logger.info(
+              "#{LOG_PREFIX} Running test_health_risk aggregation snapshot_date=#{snapshot_date} " \
+                "coverage_window=#{coverage_window} risk_window=#{risk_window}"
+            )
+            client.query(sql, format: "TabSeparated")
+            inserted = fetch_row_count(snapshot_date)
+            if inserted.is_a?(Integer) && inserted.zero?
+              logger.warn(
+                "#{LOG_PREFIX} Aggregation wrote 0 rows for snapshot_date=#{snapshot_date}. " \
+                  "This is valid if no per-test data is in the coverage_window, but worth checking " \
+                  "test_coverage_per_file directly if a recent export ran."
+              )
+            else
+              logger.info("#{LOG_PREFIX} Aggregation wrote #{inserted} rows for snapshot_date=#{snapshot_date}")
+            end
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Aggregation failed for #{snapshot_date}: #{e.message}")
+            raise
+          end
+
+          private
+
+          attr_reader :url, :database, :username, :password, :logger, :coverage_window, :risk_window
+
+          def build_sql(snapshot_date:)
+            template = File.read(SQL_FILE)
+            template
+              .gsub('{snapshot_date}', validate_date(snapshot_date))
+              .gsub('{coverage_window}', validate_interval(coverage_window))
+              .gsub('{risk_window}', validate_interval(risk_window))
+          end
+
+          # DateTime/Time `to_s` includes the time portion and is rejected.
+          def validate_date(value)
+            str = value.to_s
+            raise ArgumentError, "Invalid snapshot_date: #{value.inspect}" unless DATE_PATTERN.match?(str)
+
+            str
+          end
+
+          def validate_interval(value)
+            raise ArgumentError, "Invalid interval expression: #{value.inspect}" unless INTERVAL_PATTERN.match?(value.to_s)
+
+            value
+          end
+
+          # Returns 'unknown' on any error so a transient count-query failure
+          # can't mask the success of the actual INSERT.
+          def fetch_row_count(snapshot_date)
+            count_sql = "SELECT count() FROM code_coverage.test_health_risk_per_group FINAL " \
+              "WHERE snapshot_date = toDate('#{validate_date(snapshot_date)}')"
+            result = client.query(count_sql, format: "JSONCompact")
+            result&.dig('data', 0, 0) || 'unknown'
+          rescue StandardError => e
+            logger.debug("#{LOG_PREFIX} Could not fetch post-aggregation row count: #{e.message}")
+            'unknown'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      # Reads per-test coverage files and produces rows for
+      # `PerTestCoverageTable`.
+      #
+      # Two input formats are supported, dispatched by file extension:
+      #
+      # `.json`: one document with the example id as the outer key.
+      #   {
+      #     "spec/path/to/test_spec.rb[1:1]": {
+      #       "app/path/to/source.rb": [null, 1, 0, 5, 1, ...]
+      #     },
+      #     ...
+      #   }
+      #
+      # `.ndjson`: one JSON object per line, with `id` and `files` fields.
+      #   {"id":"spec/path/to/test_spec.rb[1:1]","files":{"app/path/to/source.rb":[null,1,0,5,1]}}
+      #   {"id":"spec/path/to/test_spec.rb[1:2]","files":{"app/path/to/source.rb":[null,0,1,0,1]}}
+      #
+      # The NDJSON form lets the producing formatter stream per-example data
+      # to disk without holding the full suite in memory. Both forms carry
+      # the same per-test data; the parser is symmetric.
+      #
+      # Inner key (in either form) is a source file path. Inner value is a
+      # 0-indexed array of per-line hit counts. `null` means non-executable;
+      # `0` means executable but not hit by this test; positive integer means
+      # executed. This is the standard Ruby `Coverage` module output shape,
+      # also produced by any per-test capture that emits one line-hit array
+      # per (test, file) pair.
+      #
+      # This class:
+      #   - strips `[<example_uid>]` from the example id to get a per-test-file key
+      #   - converts each line-hit array into a (covered_lines, total_lines) pair
+      #   - pre-aggregates within (test_file, source_file): unions covered
+      #     lines across all examples in the same test file, takes the max
+      #     total_lines
+      #   - drops rows with empty bitmaps (file imported but no line hit)
+      #   - enriches with feature_category / group / stage / section when test
+      #     metadata is provided
+      class PerTestCoverageData
+        # Raised when a coverage artifact can't be parsed. Wraps the underlying
+        # `JSON::ParserError` or `Errno::ENOENT` so callers outside the
+        # gitlab-org/gitlab CI context (where upstream `needs:` ordering
+        # guarantees well-formed artifacts) can rescue precisely without
+        # catching unrelated standard exceptions.
+        ParseError = Class.new(StandardError)
+
+        # @param coverage_files [Array<String>] paths to per-test coverage JSON artifacts
+        # @param tests_to_categories [Hash<String, Array<String>>] test_file => [feature_category]
+        # @param feature_categories_to_teams [Hash<String, Hash>] category => {group:, stage:, section:}
+        # @param captured_sha [String] the git SHA the coverage was captured against; attached to
+        #   every emitted row so downstream delta-capture jobs can ask
+        #   `SELECT max(captured_sha) FROM code_coverage.test_coverage_per_file` to find the
+        #   previous successful capture point. Defaults to '' when unknown.
+        # @raise [ParseError] if a coverage file is missing or contains invalid JSON
+        def initialize(coverage_files, tests_to_categories: {}, feature_categories_to_teams: {}, captured_sha: '')
+          @coverage_files = Array(coverage_files)
+          @tests_to_categories = tests_to_categories
+          @feature_categories_to_teams = feature_categories_to_teams
+          @captured_sha = captured_sha.to_s
+        end
+
+        # @return [Array<Hash<Symbol, Object>>] per-test-file, per-source-file rows for PerTestCoverageTable
+        def as_db_table # rubocop:disable Metrics/AbcSize
+          aggregated = {}
+
+          @coverage_files.each do |path|
+            each_example(path) do |example_id, files|
+              test_file = extract_test_file_path(example_id)
+              files.each do |source_file, line_hits|
+                covered, total = parse_line_hits(line_hits)
+                next if covered.empty?
+
+                key = [test_file, source_file]
+                if aggregated.key?(key)
+                  aggregated[key][:covered_lines].merge(covered)
+                  # max rather than picking either side: examples within the
+                  # same test file may report arrays of different lengths if
+                  # the source file was edited mid-run. Pragmatic, not exact.
+                  aggregated[key][:total_lines] = [aggregated[key][:total_lines], total].max
+                else
+                  # dup so the merge above can never alias a Set returned by
+                  # parse_line_hits to a different key later in the loop.
+                  aggregated[key] = { covered_lines: covered.dup, total_lines: total }
+                end
+              end
+            end
+          end
+
+          aggregated.map do |(test_file, source_file), agg|
+            category = @tests_to_categories[test_file]&.first || ''
+            team = @feature_categories_to_teams[category] || {}
+
+            {
+              test_file: test_file,
+              source_file: source_file,
+              covered_lines: agg[:covered_lines].to_a.sort,
+              total_lines: agg[:total_lines],
+              feature_category: category,
+              group: team[:group] || '',
+              stage: team[:stage] || '',
+              section: team[:section] || '',
+              captured_sha: @captured_sha
+            }
+          end
+        end
+
+        private
+
+        # Yield (example_id, files) pairs from one input file. Dispatches
+        # on extension: `.ndjson` is parsed line-by-line so a multi-GB
+        # capture file does not need to fit in memory; everything else is
+        # parsed as a single JSON document with `{example_id => files}` at
+        # the top level. Both forms wrap parse failures in `ParseError` so
+        # callers can rescue without naming the underlying exception classes.
+        def each_example(path)
+          if path.end_with?('.ndjson')
+            File.foreach(path) do |line|
+              line = line.strip
+              next if line.empty?
+
+              entry = JSON.parse(line)
+              yield entry.fetch('id'), entry.fetch('files')
+            end
+          else
+            JSON.parse(File.read(path)).each do |example_id, files|
+              yield example_id, files
+            end
+          end
+        rescue JSON::ParserError, Errno::ENOENT => e
+          raise ParseError, "Failed to parse coverage artifact #{path}: #{e.message}"
+        end
+
+        # Normalise an example id (`<test_file>[<uid>]:<line>`) to the
+        # bare test_file path. Strips a leading `./`, the trailing
+        # `[<uid>]`, and any `:<line>` suffix so multiple examples within
+        # the same spec file collapse to the same test_file key. Path
+        # separator handling assumes Linux paths (everything in CI is
+        # Linux); a Windows-style `C:/foo/spec.rb` would split incorrectly
+        # on the first `:`.
+        def extract_test_file_path(example_id)
+          stripped = example_id.delete_prefix('./')
+          stripped = stripped.sub(/\[.+\]\z/, '')
+          stripped.split(':').first
+        end
+
+        # Walk the per-line hit array. Returns (Set<Integer> covered_lines,
+        # Integer total_executable_lines). Indexes are 0-based; line numbers
+        # are 1-based. `nil` entries are non-executable lines.
+        # A `nil` value at the file level (file loaded but never recorded
+        # under this example) yields an empty result rather than raising,
+        # so a single odd cell doesn't fail the whole export.
+        def parse_line_hits(line_hits)
+          return [Set.new, 0] unless line_hits.is_a?(Array)
+
+          covered = Set.new
+          total = 0
+          line_hits.each_with_index do |hits, index|
+            next if hits.nil?
+
+            total += 1
+            covered.add(index + 1) if hits.is_a?(Numeric) && hits.positive?
+          end
+          [covered, total]
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/client.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "httparty"
+require "json"
+
+module GitlabQuality
+  module TestTooling
+    module TestMetricsExporter
+      class Client
+        ResponseError = Class.new(StandardError)
+
+        TESTS_PATH = "/api/v1/tests"
+        # Observer enforces a per-request cap; batch above this size to avoid silent failures.
+        MAX_BATCH_SIZE = 10_000
+
+        def initialize(url:, token:)
+          @url = url
+          @token = token
+        end
+
+        # POST array of test metric records to the observer service.
+        # Wraps each batch as { "tests" => [...] } and splits oversized payloads
+        # into chunks of at most MAX_BATCH_SIZE records.
+        #
+        # @param tests [Array<Hash>]
+        # @return [Boolean] true when every batch succeeds (or input is empty)
+        # @raise [ResponseError] on the first non-2xx batch response
+        def post_tests(tests)
+          tests.each_slice(MAX_BATCH_SIZE) do |batch|
+            response = HTTParty.post(
+              "#{url.to_s.chomp('/')}#{TESTS_PATH}",
+              body: { tests: batch }.to_json,
+              headers: {
+                "X-Gitlab-Token" => token,
+                "Content-Type" => "application/json"
+              }
+            )
+            raise ResponseError, "Observer request failed with status #{response.code}: #{response.body}" unless response.success?
+          end
+
+          true
+        end
+
+        private
+
+        attr_reader :url, :token
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "logger"
 require "singleton"
 
 module GitlabQuality
@@ -18,13 +19,9 @@ module GitlabQuality
           end
         end
 
-        attr_reader :initial_run
         attr_accessor :run_type,
-          :clickhouse_url,
-          :clickhouse_database,
-          :clickhouse_table_name,
-          :clickhouse_username,
-          :clickhouse_password
+          :observer_url,
+          :observer_token
         attr_writer :extra_rspec_metadata_keys,
           :skip_record_proc,
           :test_retried_proc,
@@ -32,44 +29,13 @@ module GitlabQuality
           :spec_file_path_prefix,
           :logger
 
-        # rubocop:disable Style/TrivialAccessors -- allows adding documentation for extra_metadata_columns
-
-        # Additional columns to be created in the table if initial_run setup is used
-        # Columns should be defined in the format used for ALTER TABLE query, example;
-        #   [
-        #     "feature_category LowCardinality(String) DEFAULT ''",
-        #     "level LowCardinality(String) DEFAULT ''"
-        #   ]
-        #
-        # @param columns [Array]
-        # @return [Array]
-        def extra_metadata_columns=(columns)
-          @extra_metadata_columns = columns
-        end
-
-        # rubocop:enable Style/TrivialAccessors
-
-        # Whether ClickHouse export is configured
+        # Whether observer export is configured
         #
         # Export is considered enabled when all required attributes are set
         #
         # @return [Boolean]
-        def clickhouse_configured?
-          [clickhouse_url, clickhouse_database, clickhouse_table_name, clickhouse_username, clickhouse_password].none?(&:blank?)
-        end
-
-        # Marks execution as initial run and performs setup tasks before running tests, like creating database in ClickHouse
-        #
-        # @return [Boolean]
-        def initial_run!
-          @initial_run = true
-        end
-
-        # Additional metadata columns used during initial table creation
-        #
-        # @return [Array]
-        def extra_metadata_columns
-          @extra_metadata_columns ||= []
+        def observer_configured?
+          [observer_url, observer_token].none? { |value| value.nil? || value.to_s.empty? }
         end
 
         # Extra rspec metadata keys to include in exported metrics

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config_helper.rb

@@ -1,35 +1,37 @@
 # frozen_string_literal: true
 
+require 'logger'
 require 'active_support/core_ext/object/blank'
 
+require_relative 'config'
+require_relative 'formatter'
+
 module GitlabQuality
   module TestTooling
     module TestMetricsExporter
       class ConfigHelper
-        REQUIRED_CLICKHOUSE_ENV_VARS = %w[
-          GLCI_DA_CLICKHOUSE_URL
-          GLCI_CLICKHOUSE_METRICS_USERNAME
-          GLCI_CLICKHOUSE_METRICS_PASSWORD
-          GLCI_CLICKHOUSE_METRICS_DB
-          GLCI_CLICKHOUSE_METRICS_TABLE
-          GLCI_CLICKHOUSE_SHARED_DB
+        REQUIRED_OBSERVER_ENV_VARS = %w[
+          GLCI_OBSERVER_URL
+          GLCI_OBSERVER_AUTH_TOKEN
         ].freeze
 
         class << self
           def configure!(run_type = test_run_type)
-            return unless ENV.fetch("CI", nil) && ENV["GLCI_EXPORT_TEST_METRICS"] == "true" && run_type
+            return unless ENV.fetch("CI", nil) && ENV.fetch("GLCI_EXPORT_TEST_METRICS", "true") == "true" && run_type
 
             RSpec.configure do |rspec_config|
               next if rspec_config.dry_run?
 
               Config.configure do |exporter_config|
                 self.logger = exporter_config.logger
-                next warn_missing_clickhouse_variables unless clickhouse_env_vars_present?
+                next warn_missing_observer_variables unless observer_env_vars_present?
 
                 yield(exporter_config) if block_given?
                 configure_exporter!(exporter_config, run_type)
 
                 rspec_config.add_formatter Formatter
+
+                logger.info("Test metrics export is enabled for run type: #{run_type}")
               end
             end
           end
@@ -42,59 +44,30 @@ module GitlabQuality
             @logger ||= Logger.new($stdout)
           end
 
-          def clickhouse_env_vars_present?
-            REQUIRED_CLICKHOUSE_ENV_VARS.all? { |var| ENV.fetch(var, nil) && !ENV[var].empty? }
-          end
-
-          def owner_records
-            @owner_records ||= GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CategoryOwnersTable.new(
-              database: ENV.fetch("GLCI_CLICKHOUSE_SHARED_DB", nil),
-              url: clickhouse_url,
-              username: clickhouse_username,
-              password: clickhouse_password
-            ).owner_records
-          rescue StandardError => e
-            logger.error("Failed to retrieve owner data: #{e}")
-            @owner_records = {}
+          def observer_env_vars_present?
+            REQUIRED_OBSERVER_ENV_VARS.all? { |var| ENV.fetch(var, nil) && !ENV[var].empty? }
           end
 
           def configure_exporter!(config, run_type)
             config.run_type = run_type
             config.custom_metrics_proc = custom_metrics_proc
 
-            configure_clickhouse!(config)
+            configure_observer!(config)
           end
 
-          def configure_clickhouse!(config)
-            config.clickhouse_database = ENV.fetch("GLCI_CLICKHOUSE_METRICS_DB", nil)
-            config.clickhouse_table_name = ENV.fetch("GLCI_CLICKHOUSE_METRICS_TABLE", nil)
-            config.clickhouse_url = clickhouse_url
-            config.clickhouse_username = clickhouse_username
-            config.clickhouse_password = clickhouse_password
+          def configure_observer!(config)
+            config.observer_url = observer_url
+            config.observer_token = observer_token
           end
 
-          def warn_missing_clickhouse_variables
-            missing = REQUIRED_CLICKHOUSE_ENV_VARS.reject { |var| ENV.fetch(var, nil) && !ENV[var].empty? }
+          def warn_missing_observer_variables
+            missing = REQUIRED_OBSERVER_ENV_VARS.reject { |var| ENV.fetch(var, nil) && !ENV[var].empty? }
             logger.warn("Test metrics export is enabled but missing environment variables: #{missing.join(', ')}")
           end
 
           def custom_metrics_proc
-            proc do |example|
-              feature_category = example.metadata[:feature_category]
-
-              owners = if feature_category.blank?
-                         logger.warn("Example '#{example.description}' is missing feature category metadata!")
-                         {}
-                       elsif unowned?(feature_category)
-                         # currently will default to shared or tooling
-                         { group: feature_category, stage: feature_category, section: feature_category }
-                       else
-                         owner_records.fetch(feature_category.to_s, {}).tap do |o|
-                           logger.warn("Feature category '#{feature_category}' has no owner data") if o.empty?
-                         end
-                       end
-
-              { pipeline_type: pipeline_type, ci_pipeline_id: ci_pipeline_id, **owners }
+            proc do |_example|
+              { pipeline_type: pipeline_type, ci_pipeline_id: ci_pipeline_id }
             end
           end
 
@@ -120,26 +93,16 @@ module GitlabQuality
                                end
           end
 
-          def unowned?(feature_category)
-            GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CategoryOwnersTable::KNOWN_UNOWNED.include?(
-              feature_category.to_s
-            )
-          end
-
           def test_run_type
             @run_type ||= ENV.fetch("GLCI_TEST_METRICS_RUN_TYPE", nil)
           end
 
-          def clickhouse_url
-            ENV.fetch("GLCI_DA_CLICKHOUSE_URL", nil)
-          end
-
-          def clickhouse_username
-            ENV.fetch("GLCI_CLICKHOUSE_METRICS_USERNAME", nil)
+          def observer_url
+            ENV.fetch("GLCI_OBSERVER_URL", nil)
           end
 
-          def clickhouse_password
-            ENV.fetch("GLCI_CLICKHOUSE_METRICS_PASSWORD", nil)
+          def observer_token
+            ENV.fetch("GLCI_OBSERVER_AUTH_TOKEN", nil)
           end
 
           def ci_pipeline_id

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb

@@ -2,27 +2,17 @@
 
 require "rspec/core/formatters/base_formatter"
 
+require_relative "test_metrics"
+require_relative "client"
+
 module GitlabQuality
   module TestTooling
     module TestMetricsExporter
       class Formatter < RSpec::Core::Formatters::BaseFormatter
-        include Utils
-
-        RSpec::Core::Formatters.register(self, :start, :stop)
+        RSpec::Core::Formatters.register(self, :stop)
 
         LOG_PREFIX = "[MetricsExporter]"
 
-        def start(_notification)
-          return unless config.initial_run
-
-          logger.info("#{LOG_PREFIX} Running initial setup for metrics export")
-          raise "Initial setup is enabled, but clickhouse configuration is missing!" unless config.clickhouse_configured?
-
-          create_clickhouse_metrics_table
-        rescue StandardError => e
-          logger.error("#{LOG_PREFIX} Error occurred during initial setup: #{e.message}")
-        end
-
         def stop(notification)
           logger.debug("#{LOG_PREFIX} Starting test metrics export")
           data = notification.examples.filter_map do |example|
@@ -32,32 +22,44 @@ module GitlabQuality
           end
           return logger.warn("#{LOG_PREFIX} No test execution records found, metrics will not be exported!") if data.empty?
 
-          push_to_clickhouse(data)
+          push_to_observer(data)
         end
 
         private
 
+        def config
+          Config.configuration
+        end
+
+        def logger
+          config.logger
+        end
+
         # Single common timestamp for all exported example metrics to keep data points consistently grouped
         #
         # @return [String]
         def time
           return @time if @time
 
-          ci_created_at = Runtime::Env.ci_pipeline_created_at
+          ci_created_at = ENV.fetch("CI_PIPELINE_CREATED_AT", nil)
           @time = (ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc).strftime('%Y-%m-%dT%H:%M:%S.%6N')
         end
 
-        # Push data to clickhouse
+        # Push data to observer service
         #
         # @param data [Array<Hash>]
         # @return [void]
-        def push_to_clickhouse(data)
-          return logger.debug("ClickHouse configuration missing, skipping ClickHouse export!") unless config.clickhouse_configured?
+        def push_to_observer(data)
+          return logger.debug("#{LOG_PREFIX} Observer configuration missing, skipping export!") unless config.observer_configured?
 
-          clickhouse_client.insert_json_data(config.clickhouse_table_name, data)
-          logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to ClickHouse!")
+          observer_client.post_tests(data)
+          logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to Observer!")
         rescue StandardError => e
-          logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to ClickHouse: #{e.message}")
+          logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to Observer: #{e.message}")
+        end
+
+        def observer_client
+          Client.new(url: config.observer_url, token: config.observer_token)
         end
       end
     end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.13.0"
+    VERSION = "3.15.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.13.0
+  version: 3.15.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-24 00:00:00.000000000 Z
+date: 2026-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -491,11 +491,16 @@ files:
 - lib/gitlab_quality/test_tooling/code_coverage/artifacts.rb
 - lib/gitlab_quality/test_tooling/code_coverage/category_owners.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/client.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregation.sql
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregator.rb
 - lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb
 - lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb
+- lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config.rb
 - lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb
@@ -588,11 +593,11 @@ files:
 - lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb
 - lib/gitlab_quality/test_tooling/test_metric/json_test_metric.rb
 - lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb
+- lib/gitlab_quality/test_tooling/test_metrics_exporter/client.rb
 - lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb
 - lib/gitlab_quality/test_tooling/test_metrics_exporter/config_helper.rb
 - lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb
 - lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb
-- lib/gitlab_quality/test_tooling/test_metrics_exporter/utils.rb
 - lib/gitlab_quality/test_tooling/test_quarantine/quarantine_formatter.rb
 - lib/gitlab_quality/test_tooling/test_quarantine/quarantine_helper.rb
 - lib/gitlab_quality/test_tooling/test_result/base_test_result.rb

data/lib/gitlab_quality/test_tooling/test_metrics_exporter/utils.rb

@@ -1,87 +0,0 @@
-# frozen_string_literal: true
-
-module GitlabQuality
-  module TestTooling
-    module TestMetricsExporter
-      module Utils
-        # Instance of metrics exporter configuration
-        #
-        # @return [Config]
-        def config
-          Config.configuration
-        end
-
-        # Configured logger instance
-        #
-        # @return [Logger]
-        def logger
-          config.logger
-        end
-
-        # Configured clickhouse client
-        #
-        # @return [ClickHouse::Client]
-        def clickhouse_client
-          ClickHouse::Client.new(
-            url: config.clickhouse_url,
-            database: config.clickhouse_database,
-            username: config.clickhouse_username,
-            password: config.clickhouse_password,
-            logger: logger
-          )
-        end
-
-        # Create table for metrics export using current ClickHouse configuration
-        #
-        # This method is mostly for schema documentation but it can be used together with initial_run! method in
-        #
-        # @return [void]
-        def create_clickhouse_metrics_table
-          table_name = config.clickhouse_table_name
-
-          clickhouse_client.query(<<~SQL)
-            CREATE TABLE IF NOT EXISTS #{table_name}
-            (
-                timestamp DateTime64(6, 'UTC'),
-                id String,
-                name String,
-                hash String,
-                file_path String,
-                status LowCardinality(String),
-                run_time UInt32,
-                location String,
-                quarantined Bool,
-                test_retried Bool,
-                feature_category LowCardinality(String) DEFAULT 'unknown',
-                run_type LowCardinality(String) DEFAULT 'unknown',
-                spec_file_path_prefix LowCardinality(String) DEFAULT '',
-                ci_project_id UInt32,
-                ci_job_name LowCardinality(String),
-                ci_job_id UInt64,
-                ci_pipeline_id UInt64,
-                ci_merge_request_iid UInt32 DEFAULT 0,
-                ci_project_path LowCardinality(String),
-                ci_branch String,
-                ci_target_branch LowCardinality(String),
-                ci_server_url LowCardinality(String) DEFAULT 'https://gitlab.com',
-                exception_class String DEFAULT '',
-                exception_classes Array(String) DEFAULT [],
-                failure_exception String DEFAULT ''
-            )
-            ENGINE = MergeTree()
-            PARTITION BY toYYYYMM(timestamp)
-            ORDER BY (ci_project_path, status, run_type, feature_category, file_path, timestamp, ci_pipeline_id)
-            SETTINGS index_granularity = 8192;
-          SQL
-          return if config.extra_metadata_columns.empty?
-
-          clickhouse_client.query(
-            "ALTER TABLE #{table_name} #{config.extra_metadata_columns.map { |column| "ADD COLUMN IF NOT EXISTS #{column}" }.join(', ')};"
-          )
-        end
-
-        module_function :config, :logger, :clickhouse_client
-      end
-    end
-  end
-end