gitlab_quality-test_tooling 3.6.1 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6922c9895f727e5c4ad2c7fd9ecc7ffa2c17f0de9125ae24def5627dc040d2f9
-  data.tar.gz: 89a32f162c38d2098ffad75338adfb3ef3d90b773819ba06b03d03db111cd252
+  metadata.gz: 3cb2e6580b8b15e78116b52ffd4fed91e0ab7db14ce914eb90f34ace0ca84741
+  data.tar.gz: 14c55d5e6048c27a891dded80da8d23c3c3b71df15b04c8ff1c98404643bb19e
 SHA512:
-  metadata.gz: 1c216c46fe2a705acba58bee70576cc7e4a71773802bc58ac59f60ef3ca413186f4a3bf5255712986b8943ffe7248797ce1e9d35a078176fa76cad458ea95519
-  data.tar.gz: 69c7c48e2270d86d89b6c12d5d11800079fe20af2c6024f65b8d2d35f89499a63816cb188a3c086520f571a9288cd51330849b115554904c7e802d9e2d2fff42
+  metadata.gz: 607a792c6df20d566ab3ec3dba96b80ed8ae8c08c6c6fe30ab9cbc39ccc2201c0934b86641ff137a2b9726d2b3386ef0ed22cb76f160faeacd74aef91c4007a8
+  data.tar.gz: 39d357544ea50bbc8968a224b699110f6957774140e7c8074fa9f28a5cd1cf03b1cdb6f0f0e22fd9a2b97ad06b0d54d183d06db043ff52a048a30758e928294f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.6.1)
+    gitlab_quality-test_tooling (3.7.0)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/README.md

@@ -292,7 +292,7 @@ Options:
         --clickhouse-username USERNAME
                                      ClickHouse username
         --clickhouse-shared-database DATABASE
-                                     ClickHouse shared database name (default: shared)
+                                     ClickHouse shared database name
         --responsibility-patterns PATH
                                      Path to YAML file with responsibility classification patterns
 
@@ -312,7 +312,7 @@ Usage: exe/sync-category-owners [options]
 Options:
         --clickhouse-url URL         ClickHouse server URL
         --clickhouse-database DATABASE
-                                     ClickHouse database name (default: shared)
+                                     ClickHouse database name
         --clickhouse-username USERNAME
                                      ClickHouse username
 

data/exe/sync-category-owners

@@ -24,7 +24,7 @@ options = OptionParser.new do |opts|
     params[:clickhouse_url] = url
   end
 
-  opts.on('--clickhouse-database DATABASE', 'ClickHouse database name (default: shared)') do |database|
+  opts.on('--clickhouse-database DATABASE', 'ClickHouse database name') do |database|
     params[:clickhouse_database] = database
   end
 
@@ -51,9 +51,6 @@ options = OptionParser.new do |opts|
   opts.parse(ARGV)
 end
 
-# Default database to 'shared' if not specified
-params[:clickhouse_database] ||= 'shared'
-
 if params.any? && (required_params - params.keys).none?
   clickhouse_password = ENV.fetch('GLCI_CLICKHOUSE_METRICS_PASSWORD', nil)
   if clickhouse_password.to_s.strip.empty?
@@ -88,10 +85,9 @@ if params.any? && (required_params - params.keys).none?
     password: clickhouse_password
   )
 
-  category_owners_table.truncate
   category_owners_table.push(category_owners.as_db_table)
 
-  puts "Successfully synced #{category_owners.as_db_table.length} feature categories to ClickHouse"
+  puts "Successfully synced feature category ownership data to ClickHouse"
 else
   puts "Missing argument(s). Required arguments are: #{required_params.map { |p| "--#{p.to_s.tr('_', '-')}" }.join(', ')}"
   puts options

data/exe/test-coverage

@@ -22,7 +22,7 @@ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibilit
 require_relative '../lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config'
 
 params = {}
-required_params = [:test_reports, :coverage_report, :test_map, :clickhouse_url, :clickhouse_database, :clickhouse_username, :responsibility_patterns]
+required_params = [:test_reports, :coverage_report, :test_map, :clickhouse_url, :clickhouse_database, :clickhouse_username, :clickhouse_shared_database, :responsibility_patterns]
 
 options = OptionParser.new do |opts|
   opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
@@ -55,7 +55,7 @@ options = OptionParser.new do |opts|
     params[:clickhouse_username] = username
   end
 
-  opts.on('--clickhouse-shared-database DATABASE', 'ClickHouse shared database name (default: shared)') do |database|
+  opts.on('--clickhouse-shared-database DATABASE', 'ClickHouse shared database name') do |database|
     params[:clickhouse_shared_database] = database
   end
 
@@ -91,7 +91,7 @@ if params.any? && (required_params - params.keys).none?
     exit 1
   end
 
-  [:clickhouse_url, :clickhouse_database, :clickhouse_username].each do |param|
+  [:clickhouse_url, :clickhouse_database, :clickhouse_username, :clickhouse_shared_database].each do |param|
     if params[param].to_s.strip.empty?
       puts "Error: --#{param.to_s.tr('_', '-')} cannot be empty"
       exit 1
@@ -172,7 +172,7 @@ if params.any? && (required_params - params.keys).none?
 
   shared_clickhouse_data = {
     url: params[:clickhouse_url],
-    database: params[:clickhouse_shared_database] || 'shared',
+    database: params[:clickhouse_shared_database],
     username: params[:clickhouse_username],
     password: clickhouse_password
   }

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb

@@ -13,12 +13,33 @@ module GitlabQuality
 
           KNOWN_UNOWNED = %w[shared not_owned tooling].freeze
 
-          def truncate
-            logger.debug("#{LOG_PREFIX} Truncating table #{full_table_name} ...")
+          # SQL query to get the latest ownership record for each category
+          # Uses window function to avoid loading entire table history
+          LATEST_RECORDS_QUERY = <<~SQL
+            SELECT category, group, stage, section
+            FROM (
+              SELECT category, group, stage, section,
+                     ROW_NUMBER() OVER (PARTITION BY category ORDER BY timestamp DESC) as rn
+              FROM %{table_name}
+            )
+            WHERE rn = 1
+          SQL
 
-            client.query("TRUNCATE TABLE #{full_table_name}")
+          # Insert only new category ownership records that don't already exist
+          # This avoids needing TRUNCATE permission
+          def push(data)
+            return logger.warn("#{LOG_PREFIX} No data found, skipping insert!") if data.empty?
 
-            logger.info("#{LOG_PREFIX} Successfully truncated table #{full_table_name}")
+            sanitized_data = sanitize_and_filter_data(data)
+            return if sanitized_data.empty?
+
+            new_records = filter_new_records(sanitized_data)
+            return if new_records.empty?
+
+            insert_new_records(new_records, sanitized_data.size)
+          rescue StandardError => e
+            logger.error("#{LOG_PREFIX} Error occurred while pushing data to #{full_table_name}: #{e.message}")
+            raise
           end
 
           # Owners of particular feature category as group, stage and section
@@ -41,14 +62,74 @@ module GitlabQuality
           private
 
           def records
-            @records ||= client
-              .query("SELECT category, group, stage, section FROM #{table_name}")
-              .each_with_object({}) { |record, hsh| hsh[record["category"]] = record.slice("group", "stage", "section") }
+            @records ||= fetch_latest_records.each_with_object({}) do |record, hsh|
+              hsh[record["category"]] = record.slice("group", "stage", "section")
+            end
+          end
+
+          def sanitize_and_filter_data(data)
+            logger.debug("#{LOG_PREFIX} Starting data export to ClickHouse")
+            sanitized_data = sanitize(data)
+
+            logger.warn("#{LOG_PREFIX} No valid data found after sanitization, skipping ClickHouse export!") if sanitized_data.empty?
+
+            sanitized_data
+          end
+
+          def filter_new_records(sanitized_data)
+            existing_records = fetch_existing_records
+            # Deduplicate against latest records per category to prevent inserting duplicate historical records.
+            # This ensures we only insert records with new category+ownership combinations, even if an older
+            # version of the same category+ownership existed previously.
+            new_records = sanitized_data.reject { |record| existing_records.include?(record_key(record)) }
+
+            logger.info("#{LOG_PREFIX} No new records to insert, all data already exists") if new_records.empty?
+
+            new_records
+          end
+
+          def insert_new_records(new_records, total_sanitized_count)
+            client.insert_json_data(table_name, new_records)
+            new_count = new_records.size
+            existing_count = total_sanitized_count - new_count
+            record_word = new_count == 1 ? 'record' : 'records'
+            logger.info("#{LOG_PREFIX} Inserted #{new_count} new #{record_word} (#{existing_count} already existed)")
+          end
+
+          def fetch_existing_records
+            fetch_latest_records.to_set { |record| record_key(record) }
+          end
+
+          def fetch_latest_records
+            query = format(LATEST_RECORDS_QUERY, table_name: table_name)
+            client.query(query)
+          end
+
+          def sanitized_data_record(record)
+            {
+              timestamp: time,
+              category: record[:feature_category],
+              group: record[:group],
+              stage: record[:stage],
+              section: record[:section]
+            }
+          end
+
+          def record_key(record)
+            # Create a unique key for the combination of category + ownership
+            # Normalize to string keys for consistent access
+            normalized = record.transform_keys(&:to_s)
+            [
+              normalized["category"],
+              normalized["group"],
+              normalized["stage"],
+              normalized["section"]
+            ]
           end
 
           # @return [Boolean] True if the record is valid, false otherwise
           def valid_record?(record)
-            required_fields = %i[category group stage section]
+            required_fields = %i[feature_category group stage section]
 
             required_fields.each do |field|
               if record[field].nil?

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -671,7 +671,7 @@ module GitlabQuality
         # @return [String] the reason to ignore the failures, or `nil` if any failures should not be ignored.
         def ignored_failure_reason(failures, failure_to_ignore)
           failures_to_ignore = compute_ignored_failures(failures, failure_to_ignore)
-          return if failures_to_ignore.empty?
+          return if failures_to_ignore.empty? || failures_to_ignore.size < failures.size
 
           "the errors included: #{failures_to_ignore.map { |e| "`#{e}`" }.join(', ')}"
         end

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.6.1"
+    VERSION = "3.7.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.6.1
+  version: 3.7.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-02-03 00:00:00.000000000 Z
+date: 2026-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control