gitlab_quality-test_tooling 3.18.0 → 3.18.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3770b676c5ff745323a47749eb04f305d435cff9769a9b567a1cc76289d5b2bb
-  data.tar.gz: 9cab37f5980a19ce80980dec4f97d867ca7bfe6e5524b383c71ea8d01509bbc8
+  metadata.gz: 779e0b86d15303be62790f4c55f12654922e2de18fe3505d4305e8a4f4f6320e
+  data.tar.gz: e2a2fc54bb3ee013dfad631c8433df72b07a68d9a20502b4ac7310a8d1500c9f
 SHA512:
-  metadata.gz: 42a46bd31d9d7bdf985ad8d430c02e0d7037e116b426ff6358f089ae619f08680cf091758cbe6b5b229612be29e691a5ee760a57fb87b0da91324860b3939590
-  data.tar.gz: 71e1eda39f86d10c71d4d14770a925eaad6dc09d019c1cbcf7d769c6e23a73de79cce7fab02412b53ddfee8d4f6e0fa56fa55ae9880636491ffa9aa1024121ae
+  metadata.gz: 026a676af6f0ccc7bef7e19778309843d19a17ab96a4d467d9be03def4c683f44ceb36d470611d0405083a4f1738d7bc6f8efd3ec604037c518234d86157f586
+  data.tar.gz: 74f87d3383d15086afacaa0dd7cb16336bb16cbe26e1fa32978f0a3cc41757d6e9015d31b0144848ad56efb6855d50aa8e5e00fc002e3bf798af13d456e1c8e2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (3.18.0)
+    gitlab_quality-test_tooling (3.18.2)
       activesupport (>= 7.0)
       amatch (~> 0.4.1)
       fog-google (~> 1.24, >= 1.24.1)

data/lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table.rb

@@ -20,7 +20,15 @@ module GitlabQuality
         # unioned into one row by the loader, not handed in as duplicates.
         class PerTestCoverageTable < GitlabQuality::TestTooling::CodeCoverage::ClickHouse::Table
           TABLE_NAME = "test_coverage_per_file"
-          BATCH_SIZE = 500
+          # Cap each INSERT by rendered byte size rather than a fixed row count.
+          # test_coverage_per_file is a SharedReplacingMergeTree partitioned by
+          # month, so every INSERT lands a part in one partition; hundreds of
+          # tiny inserts pile up parts faster than background merges collapse
+          # them, and ClickHouse throttles inserts once the per-partition part
+          # count climbs. Batching by bytes keeps the part count low while
+          # bounding each statement well within ClickHouse's limits, regardless
+          # of how wide a row's covered-line bitmap is.
+          MAX_BATCH_BYTES = 4 * 1024 * 1024
           # Intentionally generous ceiling on line numbers. Real source files
           # are thousands of lines; generated artifacts (large GraphQL schemas,
           # bundled JS, JSON manifests) can run past 100k. The cap is set to
@@ -45,10 +53,11 @@ module GitlabQuality
 
             return logger.warn("#{LOG_PREFIX} No valid data found after sanitization, skipping ClickHouse export!") if sanitized_data.empty?
 
-            total_batches = (sanitized_data.size.to_f / BATCH_SIZE).ceil
-            sanitized_data.each_slice(BATCH_SIZE).with_index do |batch, index|
-              logger.debug("#{LOG_PREFIX} Pushing batch #{index + 1} of #{total_batches} (#{batch.size} rows)")
-              client.query(build_insert_sql(batch), format: "TabSeparated")
+            batch_index = 0
+            each_byte_limited_batch(sanitized_data) do |rows_sql|
+              batch_index += 1
+              logger.debug("#{LOG_PREFIX} Pushing batch #{batch_index} (#{rows_sql.size} rows)")
+              client.query(build_insert_sql(rows_sql), format: "TabSeparated")
             end
             logger.info("#{LOG_PREFIX} Successfully pushed #{sanitized_data.size} records to #{full_table_name}")
           rescue StandardError => e
@@ -115,16 +124,40 @@ module GitlabQuality
             }
           end
 
-          def build_insert_sql(batch)
-            rows_sql = batch.map { |record| build_row_sql(record) }.join(",\n")
+          # Render an INSERT from already-built per-row SQL fragments. Callers
+          # group the fragments by byte size via `each_byte_limited_batch`.
+          def build_insert_sql(rows_sql)
             <<~SQL
               INSERT INTO #{full_table_name}
                 (timestamp, ci_project_path, test_file, source_file, covered_lines, total_lines, feature_category, `group`, stage, section, captured_sha)
               VALUES
-              #{rows_sql}
+              #{rows_sql.join(",\n")}
             SQL
           end
 
+          # Yield arrays of rendered row SQL grouped so each INSERT stays under
+          # MAX_BATCH_BYTES. A record whose own rendered size exceeds the cap is
+          # still emitted alone rather than dropped.
+          def each_byte_limited_batch(records)
+            batch = []
+            batch_bytes = 0
+
+            records.each do |record|
+              row_sql = build_row_sql(record)
+
+              if batch.any? && batch_bytes + row_sql.bytesize > MAX_BATCH_BYTES
+                yield batch
+                batch = []
+                batch_bytes = 0
+              end
+
+              batch << row_sql
+              batch_bytes += row_sql.bytesize
+            end
+
+            yield batch if batch.any?
+          end
+
           # Precondition: `record[:covered_lines]` is the sanitised integer
           # array produced upstream by `sanitize_lines` (via
           # `sanitized_data_record`). Values are positive integers within

data/lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data.rb

@@ -50,6 +50,12 @@ module GitlabQuality
         # catching unrelated standard exceptions.
         ParseError = Class.new(StandardError)
 
+        # Source files we never want in coverage risk. Vendored gems (bundled
+        # into vendor/ruby in CI) are touched by nearly every test, which both
+        # inflates the export volume and pollutes the coverage-risk metrics with
+        # third-party code we don't own. Dropped before aggregation.
+        EXCLUDED_SOURCE_PREFIXES = %w[vendor/].freeze
+
         # @param coverage_files [Array<String>] paths to per-test coverage JSON artifacts
         # @param tests_to_categories [Hash<String, Array<String>>] test_file => [feature_category]
         # @param feature_categories_to_teams [Hash<String, Hash>] category => {group:, stage:, section:}
@@ -73,6 +79,8 @@ module GitlabQuality
             each_example(path) do |example_id, files|
               test_file = extract_test_file_path(example_id)
               files.each do |source_file, line_hits|
+                next if excluded_source_file?(source_file)
+
                 covered, total = parse_line_hits(line_hits)
                 next if covered.empty?
 
@@ -168,6 +176,14 @@ module GitlabQuality
           end
           [covered, total]
         end
+
+        # True for source files excluded from coverage entirely (e.g. vendored
+        # gems). Strips a leading ./ so both "vendor/..." and "./vendor/..."
+        # match.
+        def excluded_source_file?(source_file)
+          normalized = source_file.to_s.delete_prefix('./')
+          EXCLUDED_SOURCE_PREFIXES.any? { |prefix| normalized.start_with?(prefix) }
+        end
       end
     end
   end

data/lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter.rb

@@ -33,7 +33,11 @@ module GitlabQuality
           @jest_quarantine_file = jest_quarantine_file
           @captured_sha = captured_sha.to_s
           @skip_aggregation = skip_aggregation
-          @logger = logger || ::Logger.new($stdout)
+          # Default to INFO. The export logs every batched INSERT at debug via
+          # the ClickHouse client, and at full-bucket scale that floods the CI
+          # job log past its size limit and buries real errors. Callers can pass
+          # a debug logger explicitly when they need the SQL.
+          @logger = logger || ::Logger.new($stdout, level: ::Logger::INFO)
         end
 
         # @return [void]

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "3.18.0"
+    VERSION = "3.18.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 3.18.0
+  version: 3.18.2
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-06-15 00:00:00.000000000 Z
+date: 2026-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control