gitlab_quality-test_tooling 2.16.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. checksums.yaml +4 -4
  2. data/.ruby-version +1 -1
  3. data/.tool-versions +1 -1
  4. data/Gemfile.lock +30 -28
  5. data/README.md +1 -1
  6. data/exe/epic-readiness-notification +58 -0
  7. data/exe/post-to-slack +4 -0
  8. data/exe/relate-failure-issue +9 -0
  9. data/exe/test-coverage +155 -0
  10. data/lib/gitlab_quality/test_tooling/click_house/client.rb +111 -0
  11. data/lib/gitlab_quality/test_tooling/code_coverage/artifacts.rb +92 -0
  12. data/lib/gitlab_quality/test_tooling/code_coverage/category_owners.rb +158 -0
  13. data/lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb +80 -0
  14. data/lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table.rb +140 -0
  15. data/lib/gitlab_quality/test_tooling/code_coverage/click_house/table.rb +75 -0
  16. data/lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb +100 -0
  17. data/lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb +169 -0
  18. data/lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb +43 -0
  19. data/lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier.rb +94 -0
  20. data/lib/gitlab_quality/test_tooling/code_coverage/test_map.rb +93 -0
  21. data/lib/gitlab_quality/test_tooling/code_coverage/test_report.rb +43 -0
  22. data/lib/gitlab_quality/test_tooling/code_coverage/utils.rb +18 -0
  23. data/lib/gitlab_quality/test_tooling/feature_readiness/concerns/issue_concern.rb +1 -1
  24. data/lib/gitlab_quality/test_tooling/feature_readiness/concerns/work_item_concern.rb +11 -0
  25. data/lib/gitlab_quality/test_tooling/feature_readiness/epic_readiness_notifier.rb +308 -0
  26. data/lib/gitlab_quality/test_tooling/gcs_tools.rb +49 -0
  27. data/lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb +2 -9
  28. data/lib/gitlab_quality/test_tooling/gitlab_client/group_labels_client.rb +34 -0
  29. data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb +1 -1
  30. data/lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb +2 -2
  31. data/lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb +1 -1
  32. data/lib/gitlab_quality/test_tooling/report/failed_test_issue.rb +1 -1
  33. data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb +2 -2
  34. data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb +2 -2
  35. data/lib/gitlab_quality/test_tooling/report/group_issues/error_message_normalizer.rb +49 -0
  36. data/lib/gitlab_quality/test_tooling/report/group_issues/error_pattern_matcher.rb +36 -0
  37. data/lib/gitlab_quality/test_tooling/report/group_issues/failure_processor.rb +73 -0
  38. data/lib/gitlab_quality/test_tooling/report/group_issues/group_results_in_issues.rb +48 -0
  39. data/lib/gitlab_quality/test_tooling/report/group_issues/incident_checker.rb +61 -0
  40. data/lib/gitlab_quality/test_tooling/report/group_issues/issue_base.rb +48 -0
  41. data/lib/gitlab_quality/test_tooling/report/group_issues/issue_creator.rb +44 -0
  42. data/lib/gitlab_quality/test_tooling/report/group_issues/issue_finder.rb +81 -0
  43. data/lib/gitlab_quality/test_tooling/report/group_issues/issue_formatter.rb +83 -0
  44. data/lib/gitlab_quality/test_tooling/report/group_issues/issue_manager.rb +33 -0
  45. data/lib/gitlab_quality/test_tooling/report/group_issues/issue_updater.rb +87 -0
  46. data/lib/gitlab_quality/test_tooling/report/health_problem_reporter.rb +6 -3
  47. data/lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb +1 -1
  48. data/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb +2 -6
  49. data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +176 -5
  50. data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb +0 -1
  51. data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb +2 -1
  52. data/lib/gitlab_quality/test_tooling/runtime/env.rb +9 -4
  53. data/lib/gitlab_quality/test_tooling/slack/post_to_slack.rb +103 -3
  54. data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/api_log_finder.rb +1 -1
  55. data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/application_log_finder.rb +1 -1
  56. data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/exception_log_finder.rb +1 -1
  57. data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/graphql_log_finder.rb +1 -1
  58. data/lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb +39 -11
  59. data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb +115 -15
  60. data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb +61 -36
  61. data/lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb +126 -80
  62. data/lib/gitlab_quality/test_tooling/test_metrics_exporter/utils.rb +96 -0
  63. data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb +6 -2
  64. data/lib/gitlab_quality/test_tooling/version.rb +1 -1
  65. data/lib/gitlab_quality/test_tooling.rb +3 -0
  66. metadata +84 -55
  67. data/lib/gitlab_quality/test_tooling/test_metrics_exporter/log_test_metrics.rb +0 -117
  68. data/lib/gitlab_quality/test_tooling/test_metrics_exporter/support/gcs_tools.rb +0 -49
  69. data/lib/gitlab_quality/test_tooling/test_metrics_exporter/support/influxdb_tools.rb +0 -33
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 0017f4373fb21a5d7313301ca050e381a4e462ec74a50a41aab37ab2afbd31d7
4
- data.tar.gz: 10c3f5d6750b27058de16f482b3286b5bc96c57b3565f2494965d2d255512462
3
+ metadata.gz: f037a73b3fd4a445324a37084915ea2d020475cca6f4e3b1d23045f290351082
4
+ data.tar.gz: e32bc045856df09004e725b9b84317c74ffb66b59ed640b078e292f46a3ff565
5
5
  SHA512:
6
- metadata.gz: c4963020e2f5e405efd8e984966e918430eedb95ba354a6aff4fef068330cd3a152d68d5008b0d4325c5871654c54251206f11e2386f58fa34893e7338f5bb76
7
- data.tar.gz: 5489a80567198e869517b5aca8e9f1d35871ec470d9e716f62c97e6b15e8afae7f5f7f7622e62e6273294f9268845e1874bcba9677c5f2f52bf5ccefb806774e
6
+ metadata.gz: 400c1e51bf57a3f10cdb36c083444153326da8710edd828839881867aed4d9282da4b21fa16357f49bb498ac48199dda888e422e1ddf4f5453769cb5bdc92211
7
+ data.tar.gz: cc071d0ca3c54a0d3408f6fc91165348c403ab5109c473d7bc518701a17f90d992f02139614dfb99fd53cac7e0ffef688241c7c232cf373344269ffe3e8b4436
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 3.2.5
1
+ 3.3.9
data/.tool-versions CHANGED
@@ -1,2 +1,2 @@
1
- ruby 3.2.5
1
+ ruby 3.3.9
2
2
  lefthook 1.7.14
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab_quality-test_tooling (2.16.0)
4
+ gitlab_quality-test_tooling (3.0.0)
5
5
  activesupport (>= 7.0, < 7.3)
6
6
  amatch (~> 0.4.1)
7
7
  fog-google (~> 1.24, >= 1.24.1)
@@ -128,13 +128,15 @@ GEM
128
128
  danger (>= 8.4.5)
129
129
  danger-gitlab (>= 8.0.0)
130
130
  rake
131
- gitlab-styles (12.0.1)
132
- rubocop (~> 1.62.1)
133
- rubocop-factory_bot (~> 2.25.1)
134
- rubocop-graphql (~> 1.5.0)
135
- rubocop-performance (~> 1.20.2)
136
- rubocop-rails (~> 2.24.0)
137
- rubocop-rspec (~> 2.27.1)
131
+ gitlab-styles (13.1.0)
132
+ rubocop (= 1.71.1)
133
+ rubocop-capybara (~> 2.21.0)
134
+ rubocop-factory_bot (~> 2.26.1)
135
+ rubocop-graphql (~> 1.5.4)
136
+ rubocop-performance (~> 1.21.1)
137
+ rubocop-rails (~> 2.26.0)
138
+ rubocop-rspec (~> 3.0.4)
139
+ rubocop-rspec_rails (~> 2.30.0)
138
140
  google-apis-compute_v1 (0.108.0)
139
141
  google-apis-core (>= 0.15.0, < 2.a)
140
142
  google-apis-core (0.15.1)
@@ -262,7 +264,7 @@ GEM
262
264
  pry (>= 0.13, < 0.15)
263
265
  public_suffix (6.0.1)
264
266
  racc (1.8.1)
265
- rack (3.1.7)
267
+ rack (3.2.1)
266
268
  rainbow (3.1.1)
267
269
  rake (13.2.1)
268
270
  rb-fsevent (0.11.2)
@@ -270,7 +272,7 @@ GEM
270
272
  ffi (~> 1.0)
271
273
  rbs (2.8.4)
272
274
  rchardet (1.8.0)
273
- regexp_parser (2.9.2)
275
+ regexp_parser (2.11.2)
274
276
  representable (3.2.0)
275
277
  declarative (< 0.1.0)
276
278
  trailblazer-option (>= 0.1.1, < 0.2.0)
@@ -311,37 +313,37 @@ GEM
311
313
  rspec-support (3.13.1)
312
314
  rspec_junit_formatter (0.6.0)
313
315
  rspec-core (>= 2, < 4, != 2.12.0)
314
- rubocop (1.62.1)
316
+ rubocop (1.71.1)
315
317
  json (~> 2.3)
316
318
  language_server-protocol (>= 3.17.0)
317
319
  parallel (~> 1.10)
318
320
  parser (>= 3.3.0.2)
319
321
  rainbow (>= 2.2.2, < 4.0)
320
- regexp_parser (>= 1.8, < 3.0)
321
- rexml (>= 3.2.5, < 4.0)
322
- rubocop-ast (>= 1.31.1, < 2.0)
322
+ regexp_parser (>= 2.9.3, < 3.0)
323
+ rubocop-ast (>= 1.38.0, < 2.0)
323
324
  ruby-progressbar (~> 1.7)
324
- unicode-display_width (>= 2.4.0, < 3.0)
325
- rubocop-ast (1.32.3)
325
+ unicode-display_width (>= 2.4.0, < 4.0)
326
+ rubocop-ast (1.40.0)
326
327
  parser (>= 3.3.1.0)
327
328
  rubocop-capybara (2.21.0)
328
329
  rubocop (~> 1.41)
329
- rubocop-factory_bot (2.25.1)
330
- rubocop (~> 1.41)
330
+ rubocop-factory_bot (2.26.1)
331
+ rubocop (~> 1.61)
331
332
  rubocop-graphql (1.5.4)
332
333
  rubocop (>= 1.50, < 2)
333
- rubocop-performance (1.20.2)
334
+ rubocop-performance (1.21.1)
334
335
  rubocop (>= 1.48.1, < 2.0)
335
- rubocop-ast (>= 1.30.0, < 2.0)
336
- rubocop-rails (2.24.1)
336
+ rubocop-ast (>= 1.31.1, < 2.0)
337
+ rubocop-rails (2.26.2)
337
338
  activesupport (>= 4.2.0)
338
339
  rack (>= 1.1)
339
- rubocop (>= 1.33.0, < 2.0)
340
+ rubocop (>= 1.52.0, < 2.0)
340
341
  rubocop-ast (>= 1.31.1, < 2.0)
341
- rubocop-rspec (2.27.1)
342
- rubocop (~> 1.40)
343
- rubocop-capybara (~> 2.17)
344
- rubocop-factory_bot (~> 2.22)
342
+ rubocop-rspec (3.0.5)
343
+ rubocop (~> 1.61)
344
+ rubocop-rspec_rails (2.30.0)
345
+ rubocop (~> 1.61)
346
+ rubocop-rspec (~> 3, >= 3.0.1)
345
347
  ruby-progressbar (1.13.0)
346
348
  sawyer (0.9.2)
347
349
  addressable (>= 2.3.5)
@@ -409,7 +411,7 @@ PLATFORMS
409
411
  DEPENDENCIES
410
412
  climate_control (~> 1.2)
411
413
  gitlab-dangerfiles (~> 3.8)
412
- gitlab-styles (~> 12.0)
414
+ gitlab-styles (~> 13.1)
413
415
  gitlab_quality-test_tooling!
414
416
  guard-rspec (~> 4.7)
415
417
  lefthook (~> 1.3)
@@ -425,4 +427,4 @@ DEPENDENCIES
425
427
  webmock (= 3.7.0)
426
428
 
427
429
  BUNDLED WITH
428
- 2.5.19
430
+ 2.7.2
data/README.md CHANGED
@@ -324,7 +324,7 @@ See an [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92580) fo
324
324
 
325
325
  [Automated gem release process](https://gitlab.com/gitlab-org/quality/pipeline-common#release-process) is used to release new version of `gitlab_quality-test_tooling` through pipelines, and this will:
326
326
 
327
- - Publish the gem: https://rubygems.org/gems/gitlab_quality-test_tooling
327
+ - Publish the gem: https://rubygems.org/gems/gitlab_quality-test_tooling (once the version bump is done below in [Steps to release](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling#steps-to-release))
328
328
  - Add a release in the `gitlab_quality-test_tooling` project: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/releases
329
329
  - Populate the release log with the API contents. For example: https://gitlab.com/api/v4/projects/19861191/repository/changelog?version=3.4.4
330
330
 
@@ -0,0 +1,58 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require "bundler/setup"
5
+ require "optparse"
6
+ require 'active_support/core_ext/hash'
7
+
8
+ require_relative "../lib/gitlab_quality/test_tooling"
9
+ params = {}
10
+
11
+ options = OptionParser.new do |opts|
12
+ opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
13
+
14
+ opts.on('-u', '--epic-urls URLS', String, 'Comma-separated list of epic URLs') do |urls|
15
+ params[:epic_urls] = urls.split(',').map(&:strip)
16
+ end
17
+
18
+ opts.on('-f', '--epic-urls-file FILE', String, 'File containing epic URLs (one per line)') do |file|
19
+ params[:epic_urls_file] = file
20
+ end
21
+
22
+ opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and appropriate permissions') do |token|
23
+ params[:token] = token
24
+ end
25
+
26
+ opts.on('-m', '--message MESSAGE', String, 'Custom message template (optional)') do |message|
27
+ params[:message] = message
28
+ end
29
+
30
+ opts.on('--dry-run', "Perform a dry-run (don't post comments)") do
31
+ params[:dry_run] = true
32
+ end
33
+
34
+ opts.on_tail('-v', '--version', 'Show the version') do
35
+ require_relative "../lib/gitlab_quality/test_tooling/version"
36
+ puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
37
+ exit
38
+ end
39
+
40
+ opts.on_tail('-h', '--help', 'Show the usage') do
41
+ puts "Purpose: Send feature readiness assessment notifications to epic authors via @mentions"
42
+ puts opts
43
+ exit
44
+ end
45
+
46
+ opts.parse(ARGV)
47
+ end
48
+
49
+ # Validate required arguments
50
+ raise ArgumentError, "Missing required argument: --token" unless params[:token]
51
+ raise ArgumentError, "Must provide either --epic-urls or --epic-urls-file" unless params[:epic_urls] || params[:epic_urls_file]
52
+
53
+ if params.any?
54
+ GitlabQuality::TestTooling::FeatureReadiness::EpicReadinessNotifier.new(**params).invoke!
55
+ else
56
+ puts options
57
+ exit 1
58
+ end
data/exe/post-to-slack CHANGED
@@ -77,6 +77,10 @@ options = OptionParser.new do |opts|
77
77
  params[:icon_emoji] = icon_emoji
78
78
  end
79
79
 
80
+ opts.on('-e', '--environment-issues-file FILE', String, 'Add environment issues alert based on grouped failure data in a file') do |file|
81
+ params[:environment_issues_file] = file
82
+ end
83
+
80
84
  opts.on_tail('-v', '--version', 'Show the version') do
81
85
  require_relative "../lib/gitlab_quality/test_tooling/version"
82
86
  puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
@@ -58,6 +58,14 @@ options = OptionParser.new do |opts|
58
58
  params[:dry_run] = true
59
59
  end
60
60
 
61
+ opts.on("--group-similar", "Enable grouping similar issues") do
62
+ params[:group_similar] = true
63
+ end
64
+
65
+ opts.on('--environment-issues-output-file FILE', String, 'Output file for environment issues data (JSON)') do |file|
66
+ params[:environment_issues_output_file] = file
67
+ end
68
+
61
69
  opts.on_tail('-v', '--version', 'Show the version') do
62
70
  require_relative "../lib/gitlab_quality/test_tooling/version"
63
71
  puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
@@ -66,6 +74,7 @@ options = OptionParser.new do |opts|
66
74
 
67
75
  opts.on_tail('-h', '--help', 'Show the usage') do
68
76
  puts "Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)"
77
+ puts ""
69
78
  puts opts
70
79
  exit
71
80
  end
data/exe/test-coverage ADDED
@@ -0,0 +1,155 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require "optparse"
5
+ require "uri"
6
+
7
+ require_relative "../lib/gitlab_quality/test_tooling"
8
+
9
+ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/category_owners'
10
+ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table'
11
+ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/click_house/coverage_metrics_table'
12
+ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/coverage_data'
13
+ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/lcov_file'
14
+ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/artifacts'
15
+ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_report'
16
+ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/test_map'
17
+ require_relative '../lib/gitlab_quality/test_tooling/code_coverage/source_file_classifier'
18
+
19
+ params = {}
20
+ required_params = [:test_reports, :coverage_report, :test_map, :clickhouse_url, :clickhouse_database, :clickhouse_username]
21
+
22
+ options = OptionParser.new do |opts|
23
+ opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
24
+
25
+ opts.separator ""
26
+ opts.separator "Options:"
27
+
28
+ opts.on('--test-reports GLOB',
29
+ 'Glob pattern for test JSON reports (RSpec or Jest) (e.g., "reports/**/*.json")') do |pattern|
30
+ params[:test_reports] = pattern
31
+ end
32
+
33
+ opts.on('--coverage-report PATH', 'Path to the LCOV coverage report (e.g., "coverage/lcov/gitlab.lcov")') do |path|
34
+ params[:coverage_report] = path
35
+ end
36
+
37
+ opts.on('--test-map PATH', 'Path to the test map file (e.g., "crystalball/packed-mapping.json.gz")') do |path|
38
+ params[:test_map] = path
39
+ end
40
+
41
+ opts.on('--clickhouse-url URL', 'ClickHouse server URL') do |url|
42
+ params[:clickhouse_url] = url
43
+ end
44
+
45
+ opts.on('--clickhouse-database DATABASE', 'ClickHouse database name') do |database|
46
+ params[:clickhouse_database] = database
47
+ end
48
+
49
+ opts.on('--clickhouse-username USERNAME', 'ClickHouse username') do |username|
50
+ params[:clickhouse_username] = username
51
+ end
52
+
53
+ opts.separator ""
54
+ opts.separator "Environment variables:"
55
+ opts.separator " GLCI_CLICKHOUSE_METRICS_PASSWORD ClickHouse password (required, not passed via CLI for security)"
56
+ opts.separator ""
57
+
58
+ opts.on('-h', '--help', 'Show the usage') do
59
+ puts opts
60
+ puts "\nExamples:"
61
+ puts " #{$PROGRAM_NAME}"
62
+ exit
63
+ end
64
+
65
+ opts.on_tail('-v', '--version', 'Show the version') do
66
+ require_relative "../lib/gitlab_quality/test_tooling/version"
67
+ puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
68
+ exit
69
+ end
70
+
71
+ opts.parse(ARGV)
72
+ end
73
+
74
+ if params.any? && (required_params - params.keys).none?
75
+ clickhouse_password = ENV.fetch('GLCI_CLICKHOUSE_METRICS_PASSWORD', nil)
76
+ if clickhouse_password.to_s.strip.empty?
77
+ puts "Error: GLCI_CLICKHOUSE_METRICS_PASSWORD environment variable must be set and not empty"
78
+ exit 1
79
+ end
80
+
81
+ [:clickhouse_url, :clickhouse_database, :clickhouse_username].each do |param|
82
+ if params[param].to_s.strip.empty?
83
+ puts "Error: --#{param.to_s.tr('_', '-')} cannot be empty"
84
+ exit 1
85
+ end
86
+ end
87
+
88
+ begin
89
+ uri = URI.parse(params[:clickhouse_url])
90
+ unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
91
+ puts "Error: --clickhouse-url must be a valid HTTP or HTTPS URL"
92
+ exit 1
93
+ end
94
+ rescue URI::InvalidURIError
95
+ puts "Error: --clickhouse-url is not a valid URL format"
96
+ exit 1
97
+ end
98
+
99
+ artifacts = GitlabQuality::TestTooling::CodeCoverage::Artifacts.new(
100
+ coverage_report: params[:coverage_report],
101
+ test_map: params[:test_map],
102
+ test_reports: params[:test_reports]
103
+ )
104
+
105
+ coverage_report = artifacts.coverage_report
106
+ test_map = artifacts.test_map
107
+
108
+ code_coverage_by_source_file = GitlabQuality::TestTooling::CodeCoverage::LcovFile.new(coverage_report).parsed_content
109
+
110
+ source_file_to_tests = GitlabQuality::TestTooling::CodeCoverage::TestMap.new(test_map).source_to_tests
111
+
112
+ # Process test reports
113
+ tests_to_categories = artifacts.test_reports.reduce({}) do |combined_hash, test_report_file|
114
+ file_categories = GitlabQuality::TestTooling::CodeCoverage::TestReport.new(test_report_file).tests_to_categories
115
+ combined_hash.merge(file_categories) { |_, old_val, new_val| (old_val + new_val).uniq }
116
+ end
117
+
118
+ category_owners = GitlabQuality::TestTooling::CodeCoverage::CategoryOwners.new
119
+
120
+ # Classify source files by type (frontend, backend, etc.)
121
+ source_file_classifier = GitlabQuality::TestTooling::CodeCoverage::SourceFileClassifier.new
122
+ source_file_types = source_file_classifier.classify(code_coverage_by_source_file.keys)
123
+
124
+ coverage_data = GitlabQuality::TestTooling::CodeCoverage::CoverageData.new(
125
+ code_coverage_by_source_file,
126
+ source_file_to_tests,
127
+ tests_to_categories,
128
+ category_owners.categories_to_teams,
129
+ source_file_types
130
+ )
131
+
132
+ clickhouse_data = {
133
+ url: params[:clickhouse_url],
134
+ database: params[:clickhouse_database],
135
+ username: params[:clickhouse_username],
136
+ password: clickhouse_password
137
+ }
138
+
139
+ category_owners_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CategoryOwnersTable.new(**clickhouse_data)
140
+ coverage_metrics_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CoverageMetricsTable.new(**clickhouse_data)
141
+
142
+ category_owners_table.create if ENV['CLICKHOUSE_CREATE_CATEGORY_OWNERS_TABLE'] == 'true'
143
+ coverage_metrics_table.create if ENV['CLICKHOUSE_CREATE_COVERAGE_METRICS_TABLE'] == 'true'
144
+
145
+ if ENV['CLICKHOUSE_PUSH_CATEGORY_DATA'] == 'true'
146
+ category_owners_table.truncate
147
+ category_owners_table.push(category_owners.as_db_table)
148
+ end
149
+
150
+ coverage_metrics_table.push(coverage_data.as_db_table)
151
+ else
152
+ puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
153
+ puts options
154
+ exit 1
155
+ end
@@ -0,0 +1,111 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "httparty"
4
+ require "json"
5
+ require "logger"
6
+ require "active_support/core_ext/object/blank"
7
+
8
+ module GitlabQuality
9
+ module TestTooling
10
+ module ClickHouse
11
+ class Client
12
+ DEFAULT_BATCH_SIZE = 100_000
13
+ LOG_PREFIX = "[ClickHouse]"
14
+
15
+ def initialize(url:, database:, username: nil, password: nil, logger: ::Logger.new($stdout, level: 1))
16
+ @url = url
17
+ @database = database
18
+ @username = username
19
+ @password = password
20
+ @logger = logger
21
+ end
22
+
23
+ # Perform sql query
24
+ #
25
+ # @param sql [String]
26
+ # @param format [String]
27
+ # @return [Array, Hash, String]
28
+ def query(sql, format: "JSONEachRow")
29
+ logger.debug("Running #{sql}")
30
+ response = post(
31
+ body: sql,
32
+ content_type: "text/plain",
33
+ query_opts: { default_format: format }
34
+ )
35
+ raise "ClickHouse query failed: code: #{response.code}, error: #{response.body}" if response.code != 200
36
+
37
+ if format == "JSONEachRow"
38
+ response.body.split("\n").map { |row| JSON.parse(row) }
39
+ elsif %w[JSON JSONCompact].include?(format)
40
+ JSON.parse(response.body.presence || "{}")
41
+ else
42
+ response.body
43
+ end
44
+ end
45
+
46
+ # Push data to ClickHouse
47
+ #
48
+ # @param table_name [String]
49
+ # @param data [Array<Hash>]
50
+ # @param batch_size [Integer]
51
+ # @return [void]
52
+ def insert_json_data(table_name, data, batch_size: DEFAULT_BATCH_SIZE) # rubocop:disable Metrics/AbcSize
53
+ raise ArgumentError, "Expected data to be an Array, got #{data.class}" unless data.is_a?(Array)
54
+ raise ArgumentError, "Expected all elements of array to be hashes" unless data.is_a?(Array) && data.all?(Hash)
55
+ raise ArgumentError, "Expected data to not be empty" if data.empty?
56
+
57
+ total_batches = (data.size.to_f / batch_size).ceil
58
+ results = data.each_slice(batch_size).with_index.map do |batch, index|
59
+ logger.debug("#{LOG_PREFIX} Pushing batch #{index + 1} of #{total_batches}")
60
+ send_batch(table_name, batch)
61
+ end
62
+ logger.debug("#{LOG_PREFIX} Processed #{results.size} result batches")
63
+ return if results.all? { |res| res[:success] }
64
+
65
+ err = results
66
+ .reject { |res| res[:success] }
67
+ .map { |res| "batch_size: #{res[:count]}, err: #{res[:error]}" }
68
+ .join("\n")
69
+ raise "Failures detected when pushing data to ClickHouse, errors:\n#{err}"
70
+ end
71
+
72
+ private
73
+
74
+ attr_reader :url, :database, :username, :password, :logger
75
+
76
+ # Push batch of data
77
+ #
78
+ # @param table_name [String] table name
79
+ # @param batch [Array<Hash>] data batch
80
+ # @return [Hash]
81
+ def send_batch(table_name, batch)
82
+ response = post(
83
+ body: batch.map(&:to_json).join("\n"),
84
+ content_type: 'application/json',
85
+ query_opts: { query: "INSERT INTO #{table_name} FORMAT JSONEachRow" }
86
+ )
87
+ return { success: true, count: batch.size, response: response.body } if response.code == 200
88
+
89
+ { success: false, count: batch.size, error: response.body }
90
+ rescue StandardError => e
91
+ { success: false, count: batch.size, error: e.message }
92
+ end
93
+
94
+ # Execute post request
95
+ #
96
+ # @param body [String]
97
+ # @param content_type [String]
98
+ # @param query_opts [Hash] additional query options
99
+ # @return [HTTParty::Response]
100
+ def post(body:, content_type:, query_opts: {})
101
+ HTTParty.post(url, {
102
+ body: body,
103
+ headers: { "Content-Type" => content_type },
104
+ query: { database: database, **query_opts }.compact,
105
+ basic_auth: !!(username && password) ? { username: username, password: password } : nil
106
+ }.compact)
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,92 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require 'stringio'
5
+ require 'zlib'
6
+ require 'active_support/core_ext/object/blank'
7
+
8
+ module GitlabQuality
9
+ module TestTooling
10
+ module CodeCoverage
11
+ class Artifacts
12
+ # Loads coverage artifacts from the filesystem
13
+ #
14
+ # @param test_reports [String] Glob pattern for test JSON report files (RSpec or Jest) (e.g., "reports/**/*.json")
15
+ # @param coverage_report [String] Path to the LCOV coverage report file (e.g., "coverage/lcov/gitlab.lcov")
16
+ # @param test_map [String] Path to the test map file, gzipped or plain JSON (e.g., "crystalball/packed-mapping.json.gz")
17
+ def initialize(coverage_report:, test_map:, test_reports:)
18
+ raise ArgumentError, "test_reports cannot be blank" if test_reports.blank?
19
+
20
+ @test_reports_glob = test_reports
21
+ @coverage_report_path = coverage_report
22
+ @test_map_path = test_map
23
+ end
24
+
25
+ # Loads and parses test JSON report files (RSpec or Jest)
26
+ #
27
+ # @return [Array<Hash>] Array of parsed JSON test reports
28
+ # @raise [RuntimeError] If no test reports are found or if JSON parsing fails
29
+ def test_reports
30
+ @test_report_files ||= test_reports_paths.map do |report_path|
31
+ JSON.parse(File.read(report_path))
32
+ rescue JSON::ParserError => e
33
+ raise "Invalid JSON in test report file #{report_path}: #{e.message}"
34
+ end
35
+ end
36
+
37
+ # Loads the LCOV coverage report file
38
+ #
39
+ # @return [String] Raw content of the LCOV coverage report
40
+ # @raise [RuntimeError] If the coverage report file is not found
41
+ def coverage_report
42
+ @coverage_report ||= read_coverage_reports
43
+ end
44
+
45
+ # Loads and parses the test map file (supports gzipped or plain JSON)
46
+ #
47
+ # @return [Hash] Parsed test map data
48
+ # @raise [RuntimeError] If the test map file is not found or cannot be parsed
49
+ def test_map
50
+ @test_map ||= fetch_test_map
51
+ end
52
+
53
+ private
54
+
55
+ def test_reports_paths
56
+ @test_reports_paths ||= begin
57
+ paths = Dir.glob(@test_reports_glob)
58
+
59
+ raise "No test reports found matching pattern: #{@test_reports_glob}" if paths.empty?
60
+
61
+ paths
62
+ end
63
+ end
64
+
65
+ def read_coverage_reports
66
+ raise "Coverage report not found in: #{@coverage_report_path}" unless File.exist?(@coverage_report_path)
67
+
68
+ File.read(@coverage_report_path)
69
+ end
70
+
71
+ def fetch_test_map
72
+ raise "Test map file not found: #{@test_map_path}" unless File.exist?(@test_map_path)
73
+
74
+ begin
75
+ content = File.read(@test_map_path)
76
+
77
+ # If it's a gzipped file, decompress it
78
+ content = decompressed_gzip(content) if @test_map_path.end_with?('.gz')
79
+
80
+ JSON.parse(content)
81
+ rescue StandardError => e
82
+ raise "Failed to read test map from #{@test_map_path}: #{e.message}"
83
+ end
84
+ end
85
+
86
+ def decompressed_gzip(gzipped_data)
87
+ Zlib::GzipReader.new(StringIO.new(gzipped_data)).read
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end