gitlab_quality-test_tooling 3.23.0 → 3.23.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 635939993ec2f8ac5d2593a070f8494cf9953510c3d2b1f659a86afadd57ff66
-  data.tar.gz: 7a75e161d96b24793411864ab93a467bad63bab716d2b1e2a5d1ab272583a9b9
+  metadata.gz: 49a063bdf7dc0b0055d926d178df7a5274c067f352a5feae0d5f04695702ea01
+  data.tar.gz: 426701ca7b4f8b55c433bfd78b173f1e731c1f55dafd099584acb0366b6f77de
 SHA512:
-  metadata.gz: 47a4fb04a69850d455d1dde34a2fdf3026183590d839301c07bad0685c41a8c244e128a40006dd7b90fcf83175e873b01f258945073d03e1cc9c77fe3ee6f37a
-  data.tar.gz: bf09187090fd25886a6c4a0eb320b1d431db0df8a6eca497f09b178245ffb9c229a8ba0c84107016a036eb5f095b01e709dfadc136a0393864ef1b45445df65d
+  metadata.gz: a3a5eb95bc469a879dacb6db1e3a456b35373391ed034df6a4adec4840533666613367de04510f4a8b9e1c60f755bfc5140b6a32048a597ec822bfc1c10cfd92
+  data.tar.gz: 00136e319209d433a0ff36c7c4a583ea54848653a5b1290244ba8d9108e47c0e3f7aad4a736cd7f3b51d42afe8d3beb16074c655cb41939f0135e555cfd96efb

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.23.0)
+    gitlab_quality-test_tooling (3.23.1)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       csv (~> 3.0)

data/README.md

@@ -248,7 +248,7 @@ Usage: exe/feature-readiness-evaluation [options]
 
 Runs in one of three modes:
 
-- **Full coverage-metrics export** (default, no `--per-test-coverage` / `--run-aggregation`): reads the LCOV report, Crystalball test map, and test reports, then exports the coverage-metrics and test-file-mapping tables. Requires the full argument set.
+- **Full coverage-metrics export** (default, no `--per-test-coverage` / `--run-aggregation`): reads the LCOV report, Crystalball test map, and test reports, then exports the coverage-metrics table. Requires the full argument set.
 - **Per-test coverage** (`--per-test-coverage GLOB`): inserts per-test rows into `code_coverage.test_coverage_per_file` and runs the daily `test_health_risk` aggregation unless `--skip-aggregation`. Needs only the `--clickhouse-*` options; `--test-reports` and `--jest-quarantine-file` are optional enrichment.
 - **Aggregation only** (`--run-aggregation`): runs the daily `test_health_risk` aggregation and exits. Needs only the `--clickhouse-*` options. Pair with `--skip-aggregation` on batched per-test inserts to aggregate once at the end instead of once per batch.
 

data/exe/post-to-slack

@@ -15,11 +15,12 @@ gitlab_api_token = nil
 options = OptionParser.new do |opts|
   opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
 
-  opts.on('-w', '--slack-webhook-url SLACK_WEBHOOK_URL', String, 'Slack webhook URL') do |slack_webhook_url|
-    params[:slack_webhook_url] = slack_webhook_url
+  # No String type on the webhook URL and bot token: OptionParser's String rejects "",
+  # and CI passes an unset variable as "".
+  opts.on('-w', '--slack-webhook-url SLACK_WEBHOOK_URL', 'Slack webhook URL') do |slack_webhook_url|
+    params[:slack_webhook_url] = slack_webhook_url unless slack_webhook_url.empty?
   end
 
-  # No String type: OptionParser's String rejects "", and CI passes an unset variable as "".
   opts.on('-b', '--slack-bot-token TOKEN', 'Slack bot token, used instead of the webhook URL when set') do |token|
     params[:slack_bot_token] = token unless token.empty?
   end

data/exe/test-coverage

@@ -11,7 +11,6 @@ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/category_owne
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table'
-require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/jest_quarantined_tests_table'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregator'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_score_snapshotter'
@@ -20,7 +19,6 @@ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/lcov_file'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/artifacts'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_report'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_map'
-require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter'
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier'
@@ -279,12 +277,4 @@ when :full_export
     **clickhouse_data
   )
   coverage_metrics_table.push(coverage_data.as_db_table)
-
-  # Export test-to-file mappings
-  test_file_mapping_data = GitlabQuality::TestTooling::CodeCoverage::TestFileMappingData.new(
-    test_to_sources,
-    tests_to_categories: tests_to_categories
-  )
-  test_file_mappings_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestFileMappingsTable.new(**shared_clickhouse_data)
-  test_file_mappings_table.push(test_file_mapping_data.as_db_table)
 end

data/lib/gitlab_quality/test_tooling/code_coverage/README.md

@@ -135,15 +135,6 @@ dependent:
   - "^spec/features/"
 ```
 
-## Test-to-File Mappings
-
-When a test map is provided, the module also exports test-to-source-file relationships
-to a separate `test_file_mappings` table. This enables:
-
-- **Coverage context for tests** - see which source files a specific test covers
-- **Impact analysis** - understand which files would lose coverage if a test is quarantined
-- **Flaky test triage** - correlate flaky tests with the source files they cover
-
 ## CLI
 
 Example usage:

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.23.0"
+    VERSION = "3.23.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.23.0
+  version: 3.23.1
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-10-23 00:00:00.000000000 Z
+date: 2026-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -502,7 +502,6 @@ files:
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/jest_quarantined_tests_table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb
-- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregation.sql
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregator.rb
 - lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_score_snapshot.sql
@@ -515,7 +514,6 @@ files:
 - lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config.rb
 - lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb
 - lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier.rb
-- lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data.rb
 - lib/gitlab_quality/test_tooling/code_coverage/test_map.rb
 - lib/gitlab_quality/test_tooling/code_coverage/test_report.rb
 - lib/gitlab_quality/test_tooling/code_coverage/utils.rb

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table.rb

@@ -1,263 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'table'
-
-module GitlabQuality
-  module TestTooling
-    module CodeCoverage
-      module ClickHouse
-        class TestFileMappingsTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
-          TABLE_NAME = "test_file_mappings"
-
-          # Override push to skip mappings already stored with the same values.
-          #
-          # @param data [Array<Hash>] Code coverage related data to be pushed to ClickHouse
-          # @return [nil]
-          def push(data) # rubocop:disable Metrics/AbcSize
-            return logger.warn("#{LOG_PREFIX} No data found, skipping ClickHouse export!") if data.empty?
-
-            logger.debug("#{LOG_PREFIX} Starting data export to ClickHouse")
-            @dropped_source_files = []
-            sanitized_data = sanitize(data)
-            report_dropped_source_files(sanitized_data.size)
-
-            return logger.warn("#{LOG_PREFIX} No valid data found after sanitization, skipping ClickHouse export!") if sanitized_data.empty?
-
-            # Filter out records that already exist with identical values
-            new_or_changed_records = filter_duplicates(sanitized_data)
-
-            if new_or_changed_records.empty?
-              logger.info("#{LOG_PREFIX} None of the #{sanitized_data.size} mappings need writing to #{full_table_name}, skipping insert")
-              return
-            end
-
-            client.insert_json_data(table_name, new_or_changed_records)
-            skipped_count = sanitized_data.size - new_or_changed_records.size
-            logger.info("#{LOG_PREFIX} Successfully pushed #{new_or_changed_records.size} new/changed records " \
-              "to #{full_table_name} (#{skipped_count} not written)")
-          rescue StandardError => e
-            logger.error("#{LOG_PREFIX} Error occurred while pushing data to #{full_table_name}: #{e.message}")
-            raise
-          end
-
-          private
-
-          # @return [Boolean] True if the record is valid, false otherwise
-          def valid_record?(record)
-            valid_test_file?(record) && valid_source_file?(record)
-          end
-
-          # @return [Boolean] True if the test_file field is present
-          def valid_test_file?(record)
-            return true unless record[:test_file].blank?
-
-            logger.warn("#{LOG_PREFIX} Skipping record with nil/empty test_file: #{record}")
-            false
-          end
-
-          # @return [Boolean] True if the source_file field is present
-          def valid_source_file?(record)
-            return true unless record[:source_file].blank?
-
-            logger.warn("#{LOG_PREFIX} Skipping record with nil/empty source_file: #{record}")
-            false
-          end
-
-          # @return [Hash, nil] Transformed mapping data including timestamp and CI metadata,
-          #   or nil when the source path cannot be made repo-relative
-          def sanitized_data_record(record)
-            source_file = relative_source_file(record[:source_file])
-
-            if source_file.nil?
-              dropped_source_files << record[:source_file]
-              return
-            end
-
-            {
-              timestamp: time,
-              test_file: record[:test_file],
-              source_file: source_file,
-              # CI_PROJECT_PATH is set by GitLab CI runner and considered trusted infrastructure input.
-              # In non-CI environments, this may be user-controlled, but sanitize_for_clickhouse
-              # provides SQL injection protection.
-              ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil),
-              category: record[:category] || ''
-            }
-          end
-
-          def dropped_source_files
-            @dropped_source_files ||= []
-          end
-
-          # Reports the records that relative_source_file could not make repo-relative, once
-          # for the whole run, and raises when none survived. A CI_PROJECT_DIR that does not
-          # match the artifact's paths drops every mapping, so warning per record buries the
-          # job trace in one line per mapping, and warning alone would let a wholly empty
-          # export finish green.
-          #
-          # @return [void]
-          def report_dropped_source_files(kept_count)
-            return if dropped_source_files.empty?
-
-            message = "#{LOG_PREFIX} Skipped #{dropped_source_files.size} records with " \
-              "non-relative source_file, samples: #{dropped_source_files.first(3).join(', ')}"
-
-            raise message if kept_count.zero?
-
-            logger.warn(message)
-          end
-
-          # The repo-relative form of a source path, which is what the table's
-          # valid_source_file_format constraint accepts. Returns nil when the path cannot be
-          # made relative or normalizes to nothing, and the caller drops that record, because
-          # ClickHouse rejects the whole insert batch on a constraint violation rather than
-          # the offending row.
-          #
-          # @return [String, nil]
-          def relative_source_file(path)
-            ci_project_dir = ENV.fetch('CI_PROJECT_DIR', nil)
-
-            relative = path
-            relative = relative.delete_prefix("#{ci_project_dir.chomp('/')}/") if ci_project_dir.present?
-            relative = relative.sub(%r{\A(\./)+}, '')
-
-            relative.presence unless relative.start_with?('/')
-          end
-
-          # The records worth writing: mappings this project has not stored yet, plus ones
-          # whose category changed, except that a blank never replaces a stored category.
-          # Re-sending an unchanged mapping costs a row that survives until the table's TTL
-          # expires it, because section and stage are still part of the sort key and the
-          # values written here no longer match the stored ones.
-          #
-          # If the stored categories cannot be read, see #insertable_without_lookup.
-          #
-          # @param records [Array<Hash>] Sanitized records to insert
-          # @return [Array<Hash>] Records that are new or have changed values
-          def filter_duplicates(records) # rubocop:disable Metrics/AbcSize
-            return records if records.empty?
-
-            # Assumes all records belong to the same project (batch from single pipeline run)
-            ci_project_path = records.first[:ci_project_path]
-            return records if ci_project_path.blank? # Can't query without project path
-
-            logger.debug("#{LOG_PREFIX} Checking for duplicate mappings in #{full_table_name}")
-
-            existing_mappings = fetch_existing_mappings(ci_project_path, records)
-            return insertable_without_lookup(records) if existing_mappings.nil?
-
-            blank_suppressed = 0
-
-            new_or_changed = records.select do |record|
-              key = "#{record[:test_file]}|#{record[:source_file]}"
-              existing = existing_mappings[key]
-
-              # New mapping - doesn't exist yet
-              next true if existing.nil?
-
-              # A blank category means this pipeline's rspec reports didn't cover the test,
-              # not that the test has no category: the mappings come from the Crystalball map,
-              # which lists every test file regardless of what ran. Writing the blank erases a
-              # known category, and readers resolving ownership through category get nothing.
-              if record[:category].blank? && existing[:category].present?
-                blank_suppressed += 1
-                next false
-              end
-
-              existing[:category] != record[:category]
-            end
-
-            log_blank_suppressions(blank_suppressed)
-            logger.debug("#{LOG_PREFIX} Found #{new_or_changed.size} new/changed mappings out of #{records.size} total")
-            new_or_changed
-          rescue StandardError => e
-            logger.warn("#{LOG_PREFIX} Failed to check for duplicates: #{e.message}")
-            insertable_without_lookup(records)
-          end
-
-          # What to write when the stored categories could not be read. Records carrying a
-          # category still go in, because a failed read must not silently drop mappings.
-          # Blanks are held back: with no stored value to compare against they look new, and
-          # writing one replaces whatever category is already there. The mapping comes back
-          # on the next export that can read.
-          #
-          # @param records [Array<Hash>] Sanitized records to insert
-          # @return [Array<Hash>] Records that carry a category
-          def insertable_without_lookup(records)
-            with_category = records.reject { |record| record[:category].blank? }
-
-            logger.warn("#{LOG_PREFIX} No duplicate check possible; inserting " \
-              "#{with_category.size} of #{records.size} mappings and holding back " \
-              "#{records.size - with_category.size} with no category")
-
-            with_category
-          end
-
-          # Reported at info because it is the only signal that a pipeline's reports were
-          # incomplete. A steady non-zero count across runs means categories are arriving
-          # from fewer reports than there are test files in the map.
-          #
-          # @param count [Integer] Mappings whose stored category was kept over an incoming blank
-          # @return [nil]
-          def log_blank_suppressions(count)
-            return unless count.positive?
-
-            logger.info("#{LOG_PREFIX} Kept the stored category for #{count} mappings " \
-              "this pipeline's reports did not cover")
-          end
-
-          # The category currently stored against each of this project's mappings. Only
-          # filter_duplicates reads it.
-          #
-          # @param ci_project_path [String] The CI project path to filter by
-          # @param records [Array<Hash>] Records being inserted (used to limit query scope)
-          # @return [Hash, nil] "test_file|source_file" => { category: String }, or nil when
-          #   the read fails. An empty Hash means the table holds nothing for this project.
-          def fetch_existing_mappings(ci_project_path, records) # rubocop:disable Metrics/AbcSize
-            # Query for ALL latest mappings for this project
-            # We filter in Ruby rather than building dynamic SQL to avoid injection risks
-            # Aggregate rather than matching rows against a set of the latest timestamps.
-            # The set form has to materialise one tuple per stored mapping before it can
-            # filter anything, which exceeds the server's memory limit at this table's
-            # size; argMax streams and peaks at roughly a quarter of it.
-            sql = <<~SQL
-              SELECT
-                test_file,
-                source_file,
-                argMax(category, timestamp) AS category
-              FROM #{full_table_name}
-              WHERE ci_project_path = {project_path:String}
-              GROUP BY test_file, source_file
-            SQL
-
-            # Substitute project_path value using manual escaping (client doesn't support parameterized queries)
-            sql_with_params = sql.strip.gsub('{project_path:String}', "'#{sanitize_for_clickhouse(ci_project_path)}'")
-            results = client.query(sql_with_params)
-
-            # Build lookup hash, filtering to only the test_files we're checking if dataset is small
-            test_files_set = records.to_set { |r| r[:test_file] } if records.size <= 10_000
-
-            results.each_with_object({}) do |row, hash|
-              # Skip if we're doing selective filtering and this test_file isn't in our set
-              next if test_files_set && !test_files_set.include?(row['test_file'])
-
-              key = "#{row['test_file']}|#{row['source_file']}"
-              hash[key] = { category: row['category'] }
-            end
-          rescue StandardError => e
-            logger.warn("#{LOG_PREFIX} Failed to fetch existing mappings: #{e.message}")
-            nil # nil, not {}: an empty table and an unreadable one need different handling
-          end
-
-          # Sanitize string for ClickHouse by escaping single quotes and backslashes
-          # This protects against SQL injection when we must use string interpolation
-          # @param str [String] String to sanitize
-          # @return [String] Sanitized string
-          def sanitize_for_clickhouse(str)
-            str.to_s.gsub(/\\/, '\\\\\\\\').gsub("'", "''") # rubocop:disable Style/RedundantRegexpArgument
-          end
-        end
-      end
-    end
-  end
-end

data/lib/gitlab_quality/test_tooling/code_coverage/test_file_mapping_data.rb

@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-module GitlabQuality
-  module TestTooling
-    module CodeCoverage
-      class TestFileMappingData
-        # @param [Hash<String, Array<String>>] test_to_sources Test files
-        #   mapped to all source files they cover
-        # @param [Hash<String, Array<String>>] tests_to_categories Test files
-        #   mapped to their feature categories
-        def initialize(test_to_sources, tests_to_categories: {})
-          @test_to_sources = test_to_sources
-          @tests_to_categories = tests_to_categories
-        end
-
-        # Rows for shared.test_file_mappings, one per test-and-source pair.
-        #
-        # Carries no group, stage or section on purpose. Those describe the category
-        # rather than the mapping and change whenever stages.yml is reorganised, so
-        # storing them on every mapping multiplies rows for no gain.
-        #
-        # Readers resolve ownership from shared.category_owners through category. That
-        # table keeps every historical row, up to 57 for one category, so a plain join
-        # fans out; dedupe it with ORDER BY category, timestamp DESC LIMIT 1 BY category.
-        #
-        # @return [Array<Hash<Symbol, String>>] Mapping data formatted for database insertion
-        # @example Return value
-        #   [
-        #     { test_file: "spec/models/user_spec.rb", source_file: "app/models/user.rb",
-        #       category: "team_planning" },
-        #     ...
-        #   ]
-        def as_db_table
-          @test_to_sources.flat_map do |test_file, source_files|
-            category = @tests_to_categories[test_file]&.first || ''
-
-            source_files.map do |source_file|
-              {
-                test_file: test_file,
-                source_file: source_file,
-                category: category
-              }
-            end
-          end
-        end
-      end
-    end
-  end
-end