gitlab_quality-test_tooling 3.6.1 → 3.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6922c9895f727e5c4ad2c7fd9ecc7ffa2c17f0de9125ae24def5627dc040d2f9
-  data.tar.gz: 89a32f162c38d2098ffad75338adfb3ef3d90b773819ba06b03d03db111cd252
+  metadata.gz: 0eb24e5b8b53d2478c3994cd26ca10425b01e0d7d31b51e125fedeb799d2b420
+  data.tar.gz: 374324661a08dafc46f7b92cd63136878bbeca121f5c4293bec7b4bb22ef7b1e
 SHA512:
-  metadata.gz: 1c216c46fe2a705acba58bee70576cc7e4a71773802bc58ac59f60ef3ca413186f4a3bf5255712986b8943ffe7248797ce1e9d35a078176fa76cad458ea95519
-  data.tar.gz: 69c7c48e2270d86d89b6c12d5d11800079fe20af2c6024f65b8d2d35f89499a63816cb188a3c086520f571a9288cd51330849b115554904c7e802d9e2d2fff42
+  metadata.gz: 5803a5c51bdb252e3623379ef3fe0d791c344463ded48a47e568381a5014282f56d62e328b85cf7746940736130223585ced38a6a68747a47c83ae5fef7265df
+  data.tar.gz: 63df90311f5de1f0906f46402299e156c596eea5f78bc466c258cd9bda8aff961b01fb73f415f9d5ea56320ae950759659e98d46fb8cdd1b8d51c71b44d51478

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.6.1)
+    gitlab_quality-test_tooling (3.7.1)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/README.md

@@ -292,7 +292,7 @@ Options:
         --clickhouse-username USERNAME
                                      ClickHouse username
         --clickhouse-shared-database DATABASE
-                                     ClickHouse shared database name (default: shared)
+                                     ClickHouse shared database name
         --responsibility-patterns PATH
                                      Path to YAML file with responsibility classification patterns
 
@@ -312,7 +312,7 @@ Usage: exe/sync-category-owners [options]
 Options:
         --clickhouse-url URL         ClickHouse server URL
         --clickhouse-database DATABASE
-                                     ClickHouse database name (default: shared)
+                                     ClickHouse database name
         --clickhouse-username USERNAME
                                      ClickHouse username
 

data/exe/sync-category-owners

@@ -24,7 +24,7 @@ options = OptionParser.new do |opts|
     params[:clickhouse_url] = url
   end
 
-  opts.on('--clickhouse-database DATABASE', 'ClickHouse database name (default: shared)') do |database|
+  opts.on('--clickhouse-database DATABASE', 'ClickHouse database name') do |database|
     params[:clickhouse_database] = database
   end
 
@@ -51,9 +51,6 @@ options = OptionParser.new do |opts|
   opts.parse(ARGV)
 end
 
-# Default database to 'shared' if not specified
-params[:clickhouse_database] ||= 'shared'
-
 if params.any? && (required_params - params.keys).none?
   clickhouse_password = ENV.fetch('GLCI_CLICKHOUSE_METRICS_PASSWORD', nil)
   if clickhouse_password.to_s.strip.empty?
@@ -88,10 +85,9 @@ if params.any? && (required_params - params.keys).none?
     password: clickhouse_password
   )
 
-  category_owners_table.truncate
   category_owners_table.push(category_owners.as_db_table)
 
-  puts "Successfully synced #{category_owners.as_db_table.length} feature categories to ClickHouse"
+  puts "Successfully synced feature category ownership data to ClickHouse"
 else
   puts "Missing argument(s). Required arguments are: #{required_params.map { |p| "--#{p.to_s.tr('_', '-')}" }.join(', ')}"
   puts options

data/exe/test-coverage

@@ -22,7 +22,7 @@ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibilit
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config'
 
 params = {}
-required_params = [:test_reports, :coverage_report, :test_map, :clickhouse_url, :clickhouse_database, :clickhouse_username, :responsibility_patterns]
+required_params = [:test_reports, :coverage_report, :test_map, :clickhouse_url, :clickhouse_database, :clickhouse_username, :clickhouse_shared_database, :responsibility_patterns]
 
 options = OptionParser.new do |opts|
   opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
@@ -55,7 +55,7 @@ options = OptionParser.new do |opts|
     params[:clickhouse_username] = username
   end
 
-  opts.on('--clickhouse-shared-database DATABASE', 'ClickHouse shared database name (default: shared)') do |database|
+  opts.on('--clickhouse-shared-database DATABASE', 'ClickHouse shared database name') do |database|
     params[:clickhouse_shared_database] = database
   end
 
@@ -91,7 +91,7 @@ if params.any? && (required_params - params.keys).none?
     exit 1
   end
 
-  [:clickhouse_url, :clickhouse_database, :clickhouse_username].each do |param|
+  [:clickhouse_url, :clickhouse_database, :clickhouse_username, :clickhouse_shared_database].each do |param|
     if params[param].to_s.strip.empty?
       puts "Error: --#{param.to_s.tr('_', '-')} cannot be empty"
       exit 1
@@ -172,7 +172,7 @@ if params.any? && (required_params - params.keys).none?
 
   shared_clickhouse_data = {
     url: params[:clickhouse_url],
-    database: params[:clickhouse_shared_database] || 'shared',
+    database: params[:clickhouse_shared_database],
     username: params[:clickhouse_username],
     password: clickhouse_password
   }

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb

@@ -13,12 +13,33 @@ module GitlabQuality
 
           KNOWN_UNOWNED = %w[shared not_owned tooling].freeze
 
-          def truncate
-            logger.debug("#{LOG_PREFIX} Truncating table #{full_table_name} ...")
+          # SQL query to get the latest ownership record for each unique category+ownership combination
+          # Partitions by the full composite key to handle cases where a category has multiple ownerships
+          LATEST_RECORDS_QUERY = <<~SQL
+            SELECT category, group, stage, section
+            FROM (
+              SELECT category, group, stage, section,
+                     ROW_NUMBER() OVER (PARTITION BY category, group, stage, section ORDER BY timestamp DESC) as rn
+              FROM %{table_name}
+            )
+            WHERE rn = 1
+          SQL
 
-            client.query("TRUNCATE TABLE #{full_table_name}")
+          # Insert only new category ownership records that don't already exist
+          # This avoids needing TRUNCATE permission
+          def push(data)
+            return logger.warn("#{LOG_PREFIX} No data found, skipping insert!") if data.empty?
 
-            logger.info("#{LOG_PREFIX} Successfully truncated table #{full_table_name}")
+            sanitized_data = sanitize_and_filter_data(data)
+            return if sanitized_data.empty?
+
+            new_records = filter_new_records(sanitized_data)
+            return if new_records.empty?
+
+            insert_new_records(new_records, sanitized_data.size)
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Error occurred while pushing data to #{full_table_name}: #{e.message}")
+            raise
           end
 
           # Owners of particular feature category as group, stage and section
@@ -41,14 +62,74 @@ module GitlabQuality
           private
 
           def records
-            @records ||= client
-              .query("SELECT category, group, stage, section FROM #{table_name}")
-              .each_with_object({}) { |record, hsh| hsh[record["category"]] = record.slice("group", "stage", "section") }
+            @records ||= fetch_latest_records.each_with_object({}) do |record, hsh|
+              hsh[record["category"]] = record.slice("group", "stage", "section")
+            end
+          end
+
+          def sanitize_and_filter_data(data)
+            logger.debug("#{LOG_PREFIX} Starting data export to ClickHouse")
+            sanitized_data = sanitize(data)
+
+            logger.warn("#{LOG_PREFIX} No valid data found after sanitization, skipping ClickHouse export!") if sanitized_data.empty?
+
+            sanitized_data
+          end
+
+          def filter_new_records(sanitized_data)
+            existing_records = fetch_existing_records
+            # Deduplicate against latest records per category to prevent inserting duplicate historical records.
+            # This ensures we only insert records with new category+ownership combinations, even if an older
+            # version of the same category+ownership existed previously.
+            new_records = sanitized_data.reject { |record| existing_records.include?(record_key(record)) }
+
+            logger.info("#{LOG_PREFIX} No new records to insert, all data already exists") if new_records.empty?
+
+            new_records
+          end
+
+          def insert_new_records(new_records, total_sanitized_count)
+            client.insert_json_data(table_name, new_records)
+            new_count = new_records.size
+            existing_count = total_sanitized_count - new_count
+            record_word = new_count == 1 ? 'record' : 'records'
+            logger.info("#{LOG_PREFIX} Inserted #{new_count} new #{record_word} (#{existing_count} already existed)")
+          end
+
+          def fetch_existing_records
+            fetch_latest_records.to_set { |record| record_key(record) }
+          end
+
+          def fetch_latest_records
+            query = format(LATEST_RECORDS_QUERY, table_name: table_name)
+            client.query(query)
+          end
+
+          def sanitized_data_record(record)
+            {
+              timestamp: time,
+              category: record[:feature_category],
+              group: record[:group],
+              stage: record[:stage],
+              section: record[:section]
+            }
+          end
+
+          def record_key(record)
+            # Create a unique key for the combination of category + ownership
+            # Normalize to string keys for consistent access
+            normalized = record.transform_keys(&:to_s)
+            [
+              normalized["category"],
+              normalized["group"],
+              normalized["stage"],
+              normalized["section"]
+            ]
           end
 
           # @return [Boolean] True if the record is valid, false otherwise
           def valid_record?(record)
-            required_fields = %i[category group stage section]
+            required_fields = %i[feature_category group stage section]
 
             required_fields.each do |field|
               if record[field].nil?

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table.rb

@@ -9,6 +9,36 @@ module GitlabQuality
         class TestFileMappingsTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
           TABLE_NAME = "test_file_mappings"
 
+          # Override push to filter out duplicate mappings before inserting
+          # This prevents accumulating 36M duplicate rows per day in ReplacingMergeTree
+          #
+          # @param data [Array<Hash>] Code coverage related data to be pushed to ClickHouse
+          # @return [nil]
+          def push(data) # rubocop:disable Metrics/AbcSize
+            return logger.warn("#{LOG_PREFIX} No data found, skipping ClickHouse export!") if data.empty?
+
+            logger.debug("#{LOG_PREFIX} Starting data export to ClickHouse")
+            sanitized_data = sanitize(data)
+
+            return logger.warn("#{LOG_PREFIX} No valid data found after sanitization, skipping ClickHouse export!") if sanitized_data.empty?
+
+            # Filter out records that already exist with identical values
+            new_or_changed_records = filter_duplicates(sanitized_data)
+
+            if new_or_changed_records.empty?
+              logger.info("#{LOG_PREFIX} All #{sanitized_data.size} mappings already exist with same values, skipping insert to #{full_table_name}")
+              return
+            end
+
+            client.insert_json_data(table_name, new_or_changed_records)
+            skipped_count = sanitized_data.size - new_or_changed_records.size
+            logger.info("#{LOG_PREFIX} Successfully pushed #{new_or_changed_records.size} new/changed records " \
+              "to #{full_table_name} (skipped #{skipped_count} duplicates)")
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Error occurred while pushing data to #{full_table_name}: #{e.message}")
+            raise
+          end
+
           private
 
           # @return [Boolean] True if the record is valid, false otherwise
@@ -38,6 +68,9 @@ module GitlabQuality
               timestamp: time,
               test_file: record[:test_file],
               source_file: record[:source_file],
+              # CI_PROJECT_PATH is set by GitLab CI runner and considered trusted infrastructure input.
+              # In non-CI environments, this may be user-controlled, but sanitize_for_clickhouse
+              # provides SQL injection protection.
               ci_project_path: ENV.fetch('CI_PROJECT_PATH', nil),
               category: record[:category] || '',
               group: record[:group] || '',
@@ -45,6 +78,115 @@ module GitlabQuality
               section: record[:section] || ''
             }
           end
+
+          # Filter out mappings that already exist in ClickHouse with identical enriched column values
+          # Only returns records that are:
+          # - New mappings (test_file + source_file combination doesn't exist), OR
+          # - Changed mappings (enriched columns category/group/stage/section differ from existing)
+          #
+          # This prevents inserting 36M duplicate rows per day while still allowing updates when
+          # ownership (category/group/stage/section) changes over time.
+          #
+          # @param records [Array<Hash>] Sanitized records to insert
+          # @return [Array<Hash>] Records that are new or have changed values
+          def filter_duplicates(records) # rubocop:disable Metrics/AbcSize
+            return records if records.empty?
+
+            # Assumes all records belong to the same project (batch from single pipeline run)
+            ci_project_path = records.first[:ci_project_path]
+            return records if ci_project_path.blank? # Can't query without project path
+
+            logger.debug("#{LOG_PREFIX} Checking for duplicate mappings in #{full_table_name}")
+
+            # Build a hash of existing mappings with their current enriched column values
+            # Key: "test_file|source_file", Value: {category, group, stage, section}
+            existing_mappings = fetch_existing_mappings(ci_project_path, records)
+
+            # Filter records: keep only new or changed mappings
+            new_or_changed = records.select do |record|
+              key = "#{record[:test_file]}|#{record[:source_file]}"
+              existing = existing_mappings[key]
+
+              # New mapping - doesn't exist yet
+              next true if existing.nil?
+
+              # Existing mapping - check if enriched columns changed
+              existing[:category] != record[:category] ||
+                existing[:group] != record[:group] ||
+                existing[:stage] != record[:stage] ||
+                existing[:section] != record[:section]
+            end
+
+            logger.debug("#{LOG_PREFIX} Found #{new_or_changed.size} new/changed mappings out of #{records.size} total")
+            new_or_changed
+          rescue StandardError => e
+            # If duplicate detection fails, fall back to inserting all records (safer than failing completely)
+            logger.warn("#{LOG_PREFIX} Failed to check for duplicates: #{e.message}. Inserting all records.")
+            records
+          end
+
+          # Fetch existing mappings from ClickHouse for the given project
+          # Returns a hash mapping "test_file|source_file" => {category, group, stage, section}
+          #
+          # @param ci_project_path [String] The CI project path to filter by
+          # @param records [Array<Hash>] Records being inserted (used to limit query scope)
+          # @return [Hash] Existing mappings with their enriched column values
+          def fetch_existing_mappings(ci_project_path, records) # rubocop:disable Metrics/AbcSize
+            # Query for ALL latest mappings for this project
+            # We filter in Ruby rather than building dynamic SQL to avoid injection risks
+            sql = <<~SQL
+              SELECT
+                test_file,
+                source_file,
+                category,
+                `group`,
+                stage,
+                section
+              FROM #{full_table_name}
+              WHERE ci_project_path = {project_path:String}
+                AND (ci_project_path, test_file, source_file, timestamp) IN (
+                  SELECT
+                    ci_project_path,
+                    test_file,
+                    source_file,
+                    MAX(timestamp) AS timestamp
+                  FROM #{full_table_name}
+                  WHERE ci_project_path = {project_path:String}
+                  GROUP BY ci_project_path, test_file, source_file
+                )
+            SQL
+
+            # Substitute project_path value using manual escaping (client doesn't support parameterized queries)
+            sql_with_params = sql.strip.gsub('{project_path:String}', "'#{sanitize_for_clickhouse(ci_project_path)}'")
+            results = client.query(sql_with_params)
+
+            # Build lookup hash, filtering to only the test_files we're checking if dataset is small
+            test_files_set = records.to_set { |r| r[:test_file] } if records.size <= 10_000
+
+            results.each_with_object({}) do |row, hash|
+              # Skip if we're doing selective filtering and this test_file isn't in our set
+              next if test_files_set && !test_files_set.include?(row['test_file'])
+
+              key = "#{row['test_file']}|#{row['source_file']}"
+              hash[key] = {
+                category: row['category'],
+                group: row['group'],
+                stage: row['stage'],
+                section: row['section']
+              }
+            end
+          rescue StandardError => e
+            logger.warn("#{LOG_PREFIX} Failed to fetch existing mappings: #{e.message}")
+            {} # Return empty hash on error (will treat all records as new)
+          end
+
+          # Sanitize string for ClickHouse by escaping single quotes and backslashes
+          # This protects against SQL injection when we must use string interpolation
+          # @param str [String] String to sanitize
+          # @return [String] Sanitized string
+          def sanitize_for_clickhouse(str)
+            str.to_s.gsub(/\\/, '\\\\\\\\').gsub("'", "''") # rubocop:disable Style/RedundantRegexpArgument
+          end
         end
       end
     end

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -671,7 +671,7 @@ module GitlabQuality
         # @return [String] the reason to ignore the failures, or `nil` if any failures should not be ignored.
         def ignored_failure_reason(failures, failure_to_ignore)
           failures_to_ignore = compute_ignored_failures(failures, failure_to_ignore)
-          return if failures_to_ignore.empty?
+          return if failures_to_ignore.empty? || failures_to_ignore.size < failures.size
 
           "the errors included: #{failures_to_ignore.map { |e| "`#{e}`" }.join(', ')}"
         end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.6.1"
+    VERSION = "3.7.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.6.1
+  version: 3.7.1
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-02-03 00:00:00.000000000 Z
+date: 2026-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control