gitlab_quality-test_tooling 3.2.1 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7463b8e510a9decd2156d3167e28d6109c99928b03cc5738be19eba7ecccfa39
-  data.tar.gz: 2094146a3e6ebcbbcf17ede753e5b1c822b8c33b397cc02550b5c17dac1d19a6
+  metadata.gz: 32b52c2177aaf57f1ccf6a2bb922fd47e267b251ce74921c356f9631252051da
+  data.tar.gz: 8930c0fb427f3a946cd99902a230bcf90c48a051c9ffa1d19fb10a487107d231
 SHA512:
-  metadata.gz: d1fe5d32ed4521023381d238ad1a5bd17d7b36ea32b9bd888697e0126452a36fab14d2884d3d665f55620cd76d7f9da5048b5bb18b9f61746b0b49e49ed291f5
-  data.tar.gz: df998fa7999ae675f4de19227bef692594427c02c30116f9312fbcc07d73312bcccb634efabafe20cea3fc7fa947bb85feab19829b13b1071fd5e5626ed0c5a9
+  metadata.gz: f265a7b7c74d4ef36e06c53a945b96ab066a4304acd0e6c439cf10afee6d0483de9253ca9290b279db7ec3c1d2323146da3aea8cf92aa57d4d1914c9c1479b1f
+  data.tar.gz: b983b542d66e9ac5b5775dd97b80ce6b464a618a5adf573f18613014dcfecc86555fc8b7d20e32b33b41b23ce9be255811cc276a50a1ee7945f0e4062deb15cf

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.2.1)
+    gitlab_quality-test_tooling (3.3.0)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/exe/test-coverage

@@ -10,11 +10,13 @@ require_relative "../lib/gitlab_quality/test_tooling"
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/category_owners'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/coverage_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/lcov_file'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/artifacts'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_report'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_map'
+require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config'
@@ -167,15 +169,17 @@ if params.any? && (required_params - params.keys).none?
   category_owners_table  = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CategoryOwnersTable.new(**clickhouse_data)
   coverage_metrics_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CoverageMetricsTable.new(**clickhouse_data)
 
-  category_owners_table.create  if ENV['CLICKHOUSE_CREATE_CATEGORY_OWNERS_TABLE']  == 'true'
-  coverage_metrics_table.create if ENV['CLICKHOUSE_CREATE_COVERAGE_METRICS_TABLE'] == 'true'
-
   if ENV['CLICKHOUSE_PUSH_CATEGORY_DATA'] == 'true'
     category_owners_table.truncate
     category_owners_table.push(category_owners.as_db_table)
   end
 
   coverage_metrics_table.push(coverage_data.as_db_table)
+
+  # Export test-to-file mappings
+  test_file_mapping_data = GitlabQuality::TestTooling::CodeCoverage::TestFileMappingData.new(test_to_sources)
+  test_file_mappings_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestFileMappingsTable.new(**clickhouse_data)
+  test_file_mappings_table.push(test_file_mapping_data.as_db_table)
 else
   puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
   puts options

data/lib/gitlab_quality/test_tooling/code_coverage/README.md

@@ -4,6 +4,7 @@ Exports test coverage data to ClickHouse, enriched with:
 
 - **Feature category ownership** - group, stage, and section for each covered file
 - **Responsibility classification** - whether coverage comes from unit or integration tests
+- **Test-to-file mappings** - which source files each test covers
 
 ## How It Works
 
@@ -134,6 +135,15 @@ dependent:
   - "^spec/features/"
 ```
 
+## Test-to-File Mappings
+
+When a test map is provided, the module also exports test-to-source-file relationships
+to a separate `test_file_mappings` table. This enables:
+
+- **Coverage context for tests** - see which source files a specific test covers
+- **Impact analysis** - understand which files would lose coverage if a test is quarantined
+- **Flaky test triage** - correlate flaky tests with the source files they cover
+
 ## CLI
 
 Example usage:

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb

@@ -11,29 +11,6 @@ module GitlabQuality
 
           MissingMappingError = Class.new(StandardError)
 
-          # Creates the ClickHouse table, if it doesn't exist already
-          # @return [nil]
-          def create
-            logger.debug("#{LOG_PREFIX} Creating category_owners table if it doesn't exist ...")
-
-            client.query(<<~SQL)
-              CREATE TABLE IF NOT EXISTS #{table_name} (
-                timestamp DateTime64(6, 'UTC') DEFAULT now64(),
-                category String,
-                group String,
-                stage String,
-                section String,
-                INDEX idx_group group TYPE set(360) GRANULARITY 1,
-                INDEX idx_stage stage TYPE set(360) GRANULARITY 1,
-                INDEX idx_section section TYPE set(360) GRANULARITY 1
-              ) ENGINE = MergeTree()
-              ORDER BY (category, timestamp)
-              SETTINGS index_granularity = 8192;
-            SQL
-
-            logger.info("#{LOG_PREFIX} Category owners table created/verified successfully")
-          end
-
           def truncate
             logger.debug("#{LOG_PREFIX} Truncating table #{full_table_name} ...")
 

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'time'
 require_relative 'table'
 
 module GitlabQuality
@@ -10,39 +9,6 @@ module GitlabQuality
         class CoverageMetricsTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
           TABLE_NAME = "coverage_metrics"
 
-          # Creates the ClickHouse table, if it doesn't exist already
-          # @return [nil]
-          def create
-            logger.debug("#{LOG_PREFIX} Creating coverage_metrics table if it doesn't exist ...")
-
-            client.query(<<~SQL)
-              CREATE TABLE IF NOT EXISTS #{table_name} (
-                timestamp DateTime64(6, 'UTC'),
-                file String,
-                line_coverage Float64,
-                branch_coverage Nullable(Float64),
-                function_coverage Nullable(Float64),
-                source_file_type String,
-                is_responsible Nullable(Bool),
-                is_dependent Nullable(Bool),
-                category Nullable(String),
-                ci_project_id Nullable(UInt32),
-                ci_project_path Nullable(String),
-                ci_job_name Nullable(String),
-                ci_job_id Nullable(UInt64),
-                ci_pipeline_id Nullable(UInt64),
-                ci_merge_request_iid Nullable(UInt32),
-                ci_branch Nullable(String),
-                ci_target_branch Nullable(String)
-              ) ENGINE = MergeTree()
-              PARTITION BY toYYYYMM(timestamp)
-              ORDER BY (ci_project_path, timestamp, file, ci_pipeline_id)
-              SETTINGS index_granularity = 8192, allow_nullable_key = 1;
-            SQL
-
-            logger.info("#{LOG_PREFIX} Coverage metrics table created/verified successfully")
-          end
-
           private
 
           # @return [Boolean] True if the record is valid, false otherwise
@@ -106,14 +72,6 @@ module GitlabQuality
             }
           end
 
-          # @return [Time] Common timestamp for all coverage records
-          def time
-            @time ||= begin
-              ci_created_at = ENV.fetch('CI_PIPELINE_CREATED_AT', nil)
-              ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc
-            end
-          end
-
           # @return [Hash] CI-related metadata
           def ci_metadata
             {

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'time'
+
 module GitlabQuality
   module TestTooling
     module CodeCoverage
@@ -58,6 +60,21 @@ module GitlabQuality
             raise NotImplementedError, "#{self.class}##{__method__} method must be implemented in a subclass"
           end
 
+          # @return [Time] Common timestamp for all records, memoized
+          def time
+            @time ||= parse_ci_timestamp
+          end
+
+          def parse_ci_timestamp
+            ci_created_at = ENV.fetch('CI_PIPELINE_CREATED_AT', nil)
+            return Time.now.utc unless ci_created_at
+
+            Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z')
+          rescue ArgumentError
+            logger.warn("#{LOG_PREFIX} Invalid CI_PIPELINE_CREATED_AT format: #{ci_created_at}, using current time")
+            Time.now.utc
+          end
+
           # @return [GitlabQuality::TestTooling::ClickHouse::Client]
           def client
             @client ||= GitlabQuality::TestTooling::ClickHouse::Client.new(

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require_relative 'table'
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      module ClickHouse
+        class TestFileMappingsTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
+          TABLE_NAME = "test_file_mappings"
+
+          private
+
+          # @return [Boolean] True if the record is valid, false otherwise
+          def valid_record?(record)
+            valid_test_file?(record) && valid_source_file?(record)
+          end
+
+          # @return [Boolean] True if the test_file field is present
+          def valid_test_file?(record)
+            return true unless record[:test_file].blank?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with nil/empty test_file: #{record}")
+            false
+          end
+
+          # @return [Boolean] True if the source_file field is present
+          def valid_source_file?(record)
+            return true unless record[:source_file].blank?
+
+            logger.warn("#{LOG_PREFIX} Skipping record with nil/empty source_file: #{record}")
+            false
+          end
+
+          # @return [Hash] Transformed mapping data including timestamp and CI metadata
+          def sanitized_data_record(record)
+            {
+              timestamp: time,
+              test_file: record[:test_file],
+              source_file: record[:source_file],
+              ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil)
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb

@@ -85,7 +85,7 @@ module GitlabQuality
         end
 
         def register_source_file(filename)
-          @current_file = filename.gsub(%r{^\./}, '')
+          @current_file = normalize_path(filename)
           @parsed_content[@current_file] = {
             line_coverage: {},
             branch_coverage: {},
@@ -94,6 +94,16 @@ module GitlabQuality
           }
         end
 
+        def normalize_path(filename)
+          # Remove leading ./ if present
+          path = filename.gsub(%r{^\./}, '')
+
+          # Handle GDK/CI paths like "../../../home/gdk/gitlab-development-kit/gitlab/app/..."
+          # Extract path starting from known root directories
+          match = path.match(%r{((?:ee/)?(?:app|lib|config|db|spec|scripts|tooling|workhorse|vendor)/.+)$})
+          match ? match[1] : path
+        end
+
         def register_line_data(line_no, count)
           return unless @current_file
 

data/lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module GitlabQuality
+  module TestTooling
+    module CodeCoverage
+      class TestFileMappingData
+        # @param [Hash<String, Array<String>>] test_to_sources Test files
+        #   mapped to all source files they cover
+        def initialize(test_to_sources)
+          @test_to_sources = test_to_sources
+        end
+
+        # @return [Array<Hash<Symbol, String>>] Mapping data formatted for database insertion
+        # @example Return value
+        #   [
+        #     { test_file: "spec/models/user_spec.rb", source_file: "app/models/user.rb" },
+        #     { test_file: "spec/models/user_spec.rb", source_file: "lib/utils.rb" },
+        #     ...
+        #   ]
+        def as_db_table
+          @test_to_sources.flat_map do |test_file, source_files|
+            source_files.map do |source_file|
+              {
+                test_file: test_file,
+                source_file: source_file
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.2.1"
+    VERSION = "3.3.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.2.1
+  version: 3.3.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-12-12 00:00:00.000000000 Z
+date: 2025-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -490,12 +490,14 @@ files:
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb
+- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb
 - lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier.rb
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config.rb
 - lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb
 - lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier.rb
+- lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data.rb
 - lib/gitlab_quality/test_tooling/code_coverage/test_map.rb
 - lib/gitlab_quality/test_tooling/code_coverage/test_report.rb
 - lib/gitlab_quality/test_tooling/code_coverage/utils.rb