gitlab_quality-test_tooling 2.0.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4a5b20eb9951ec812418e3b1f5774674419d29961fabf4710f32e71d7a7f6c73
4
- data.tar.gz: 7ec895c4672772ef7801cf1fa864c87d1f76b59f2834434658f13e7c64cc0615
3
+ metadata.gz: 7437a045ff423ca3c6585be2aa13b11d97fd94abb5d8ffd95a14d11f48970b68
4
+ data.tar.gz: fd91b826d48c661a7b05f196857ec95c7fc59d478e8718ffa9ca5d534f38ffe5
5
5
  SHA512:
6
- metadata.gz: f013d08976015caf80341b6aa86ecbb1b7198adeac56911dc89a9e3958a78a8001c065cb7a777ed8248109fd72d5b2109a577692c494ad785a3f55e0296c2be3
7
- data.tar.gz: 66be21506f82e4cbf566216f546308df9e1607aaa709792f35c9053c117da2a2845abb376fe88ff743790a0d5006a6d71e2ee2b9365f109b50772367a6c2bf8a
6
+ metadata.gz: 4de31f3a187af7c92df5b015c631e5fa1278fa6a3d62f8111294ce07c245236b9e30e78a6760859eee4d024ee0de62dd544201e2d3e4beec73ef7be00f443def
7
+ data.tar.gz: 1d44821f07f2d61ac2d1b31b0837149bcf015070ffa1c4e6791d4685c3be322ab0991a28bf1551130eaf274045d58855a405b680948673f78aa208c9a34cab77
data/.tool-versions CHANGED
@@ -1 +1,2 @@
1
1
  ruby 3.2.5
2
+ lefthook 1.7.14
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab_quality-test_tooling (2.0.0)
4
+ gitlab_quality-test_tooling (2.2.0)
5
5
  activesupport (>= 7.0, < 7.2)
6
6
  amatch (~> 0.4.1)
7
7
  fog-google (~> 1.24, >= 1.24.1)
@@ -302,6 +302,8 @@ GEM
302
302
  binding_of_caller
303
303
  rspec-parameterized-core (< 2)
304
304
  rspec-support (3.13.1)
305
+ rspec_junit_formatter (0.6.0)
306
+ rspec-core (>= 2, < 4, != 2.12.0)
305
307
  rubocop (1.62.1)
306
308
  json (~> 2.3)
307
309
  language_server-protocol (>= 3.17.0)
@@ -407,6 +409,7 @@ DEPENDENCIES
407
409
  pry-byebug (= 3.10.1)
408
410
  rake (~> 13.0)
409
411
  rspec (~> 3.12)
412
+ rspec_junit_formatter (~> 0.6.0)
410
413
  simplecov (~> 0.22)
411
414
  simplecov-cobertura (~> 2.1)
412
415
  solargraph (~> 0.41)
data/README.md CHANGED
@@ -177,6 +177,20 @@ Usage: exe/failed-test-issues [options]
177
177
  -h, --help Show the usage
178
178
  ```
179
179
 
180
+ ### `exe/existing-test-health-issue`
181
+
182
+ ```shell
183
+ Purpose: Checks whether tests coming from the rspec JSON report files has an existing test health issue opened.
184
+ Usage: exe/existing-test-health-issue [options]
185
+ -i, --input-files INPUT_FILES JSON rspec-retry report files
186
+ -p, --project PROJECT Can be an integer or a group/project string
187
+ -t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
188
+ --health-problem-type PROBLEM_TYPE
189
+ Look for the given health problem type (failures, pass-after-retry, slow)
190
+ -v, --version Show the version
191
+ -h, --help Show the usage
192
+ ```
193
+
180
194
  ### `exe/flaky-test-issues`
181
195
 
182
196
  ```shell
@@ -0,0 +1,59 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require "bundler/setup"
5
+ require "optparse"
6
+
7
+ require_relative "../lib/gitlab_quality/test_tooling"
8
+
9
+ params = {}
10
+ HEALTH_PROBLEM_TYPES = GitlabQuality::TestTooling::Report::TestHealthIssueFinder::HEALTH_PROBLEM_TYPE_TO_LABEL.keys
11
+
12
+ options = OptionParser.new do |opts|
13
+ opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
14
+
15
+ opts.on('-i', '--input-files INPUT_FILES', String, 'JSON rspec-retry report files') do |input_files|
16
+ params[:input_files] = input_files
17
+ end
18
+
19
+ opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
20
+ params[:project] = project
21
+ end
22
+
23
+ opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
24
+ params[:token] = token
25
+ end
26
+
27
+ opts.on("--health-problem-type PROBLEM_TYPE", String, "Look for the given health problem type (#{HEALTH_PROBLEM_TYPES.join(', ')})") do |value|
28
+ raise ArgumentError, "Invalid health problem type: #{value}. Valid options are: #{HEALTH_PROBLEM_TYPES.join(', ')}" unless HEALTH_PROBLEM_TYPES.include?(value)
29
+
30
+ params[:health_problem_type] = value
31
+ end
32
+
33
+ opts.on_tail('-v', '--version', 'Show the version') do
34
+ require_relative "../lib/gitlab_quality/test_tooling/version"
35
+ puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
36
+ exit
37
+ end
38
+
39
+ opts.on_tail('-h', '--help', 'Show the usage') do
40
+ puts "Purpose: Checks whether tests coming from the rspec JSON report files has an existing test health issue opened."
41
+ puts opts
42
+ exit
43
+ end
44
+
45
+ opts.parse(ARGV)
46
+ end
47
+
48
+ if params.any?
49
+ raise ArgumentError, "No health problem type given. Valid options are: #{HEALTH_PROBLEM_TYPES.join(', ')}" unless params.key?(:health_problem_type)
50
+
51
+ if GitlabQuality::TestTooling::Report::TestHealthIssueFinder.new(**params).found_existing_unhealthy_test_issue?
52
+ exit 0
53
+ else
54
+ exit 1
55
+ end
56
+ else
57
+ puts options
58
+ exit 1
59
+ end
@@ -11,13 +11,15 @@ module GitlabQuality
11
11
  FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
12
12
  REPORT_ITEM_REGEX = /^1\. (?<report_date>\d{4}-\d{2}-\d{2}): #{JOB_URL_REGEX} \((?<pipeline_url>\S+)\) ?(?<extra_content>.*)$/
13
13
  LATEST_REPORTS_TO_SHOW = 10
14
+ DISPLAYED_HISTORY_REPORTS_THRESHOLD = 510 # https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/587
14
15
  DAILY_REPORTS_THRESHOLDS = 10
15
16
 
16
17
  class ReportsList
17
- def initialize(preserved_content:, section_header:, reports:, extra_content:)
18
+ def initialize(preserved_content:, section_header:, reports:, total_reports_count:, extra_content:)
18
19
  @preserved_content = preserved_content
19
20
  @section_header = section_header
20
21
  @reports = reports
22
+ @total_reports_count = total_reports_count
21
23
  @extra_content = extra_content
22
24
  end
23
25
 
@@ -31,14 +33,14 @@ module GitlabQuality
31
33
  end
32
34
 
33
35
  def reports_count
34
- reports.size
36
+ total_reports_count
35
37
  end
36
38
 
37
39
  def to_s
38
40
  [
39
41
  preserved_content,
40
42
  "#{section_header} (#{reports_count})",
41
- reports_list,
43
+ reports_list(total_reports_count),
42
44
  extra_content
43
45
  ].reject(&:blank?).compact.join("\n\n")
44
46
  end
@@ -53,15 +55,16 @@ module GitlabQuality
53
55
 
54
56
  private
55
57
 
56
- attr_reader :preserved_content, :section_header, :reports, :extra_content
58
+ attr_reader :preserved_content, :section_header, :reports, :total_reports_count, :extra_content
57
59
 
58
- def reports_list
60
+ def reports_list(total_reports_count)
59
61
  if sorted_reports.size > LATEST_REPORTS_TO_SHOW
62
+ max_displayed_reports = DISPLAYED_HISTORY_REPORTS_THRESHOLD - LATEST_REPORTS_TO_SHOW
60
63
  [
61
64
  "Last #{LATEST_REPORTS_TO_SHOW} reports:",
62
- sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
63
- "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
64
- sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
65
+ displayed_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
66
+ "<details><summary>With #{total_reports_count - LATEST_REPORTS_TO_SHOW} more reports (displaying up to #{max_displayed_reports} reports) </summary>",
67
+ displayed_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
65
68
  "</details>"
66
69
  ].join("\n\n")
67
70
  else
@@ -70,7 +73,12 @@ module GitlabQuality
70
73
  end
71
74
 
72
75
  def sorted_reports
73
- @sorted_reports ||= reports.sort.reverse
76
+ @sorted_reports ||=
77
+ reports.sort_by { |report| [report.report_date, report.job_id, report.to_s] }.reverse
78
+ end
79
+
80
+ def displayed_reports
81
+ sorted_reports[0...DISPLAYED_HISTORY_REPORTS_THRESHOLD]
74
82
  end
75
83
  end
76
84
 
@@ -88,8 +96,8 @@ module GitlabQuality
88
96
  "1. #{report_date}: #{job_url} (#{pipeline_url}) #{extra_content}".strip
89
97
  end
90
98
 
91
- def <=>(other)
92
- to_s <=> other.to_s
99
+ def job_id
100
+ job_url.split('/').last
93
101
  end
94
102
 
95
103
  private
@@ -122,14 +130,33 @@ module GitlabQuality
122
130
  preserved_content = current_reports_content.split(reports_section_header).first&.strip
123
131
  reports = report_lines(current_reports_content) + [ReportsList.report_list_item(test, item_extra_content: item_extra_content)]
124
132
 
133
+ total_reports_count = increment_total_reports_count(
134
+ reports_section_header: reports_section_header,
135
+ content: current_reports_content,
136
+ reports: reports
137
+ )
138
+
125
139
  ReportsList.new(
126
140
  preserved_content: preserved_content,
127
141
  section_header: reports_section_header,
128
142
  reports: reports,
143
+ total_reports_count: total_reports_count,
129
144
  extra_content: reports_extra_content
130
145
  )
131
146
  end
132
147
 
148
+ def increment_total_reports_count(reports_section_header:, content:, reports:)
149
+ reports_count = reports.count
150
+
151
+ return reports_count if reports_count < DISPLAYED_HISTORY_REPORTS_THRESHOLD
152
+
153
+ count_match = content.match(/#{Regexp.escape(reports_section_header)} \((\d+)\)/)
154
+
155
+ return reports_count unless count_match
156
+
157
+ count_match[1].to_i + 1
158
+ end
159
+
133
160
  def failed_issue_job_url(issue)
134
161
  job_urls_from_description(issue.description, REPORT_ITEM_REGEX).last ||
135
162
  # Legacy format
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'http'
4
+ require 'json'
5
+
6
+ module GitlabQuality
7
+ module TestTooling
8
+ module Report
9
+ class TestHealthIssueFinder < ReportAsIssue
10
+ HEALTH_PROBLEM_TYPE_TO_LABEL = {
11
+ 'pass-after-retry' => 'test-health:pass-after-retry',
12
+ 'slow' => 'test-health:slow',
13
+ 'failures' => 'test-health:failures'
14
+ }.freeze
15
+
16
+ def initialize(health_problem_type: [], **kwargs)
17
+ super(**kwargs)
18
+
19
+ @health_problem_type = health_problem_type
20
+ end
21
+
22
+ def found_existing_unhealthy_test_issue?
23
+ issue_url = invoke!
24
+
25
+ !issue_url.nil? && !issue_url.empty?
26
+ end
27
+
28
+ def run!
29
+ existing_issue_found = nil
30
+
31
+ applicable_tests.each do |test|
32
+ issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: search_labels)
33
+ next if issues.empty?
34
+
35
+ existing_issue_found = issues.first.web_url
36
+ puts "Found an existing test health issue of type #{health_problem_type} for test #{test.file}:#{test.line_number}: #{existing_issue_found}."
37
+ break
38
+ end
39
+
40
+ puts "Did not find an existing test health issue of type #{health_problem_type}." unless existing_issue_found
41
+
42
+ existing_issue_found
43
+ end
44
+
45
+ def applicable_tests
46
+ applicable_tests = []
47
+
48
+ TestResults::Builder.new(file_glob: files, token: token, project: project).test_results_per_file do |test_results|
49
+ applicable_tests = test_results.select { |test| test_is_applicable?(test) }
50
+ end
51
+
52
+ applicable_tests
53
+ end
54
+
55
+ private
56
+
57
+ attr_reader :health_problem_type
58
+
59
+ # Be mindful about the number of tests this method would return,
60
+ # as we will make at least one API request per test.
61
+ def test_is_applicable?(test)
62
+ expected_test_status =
63
+ case health_problem_type
64
+ when 'failures'
65
+ 'failed'
66
+ else
67
+ 'passed'
68
+ end
69
+
70
+ test.status == expected_test_status
71
+ end
72
+
73
+ def search_labels
74
+ ['test', HEALTH_PROBLEM_TYPE_TO_LABEL[health_problem_type]]
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
@@ -16,9 +16,14 @@ module GitlabQuality
16
16
 
17
17
  def parse
18
18
  JSON.parse(File.read(path))
19
+ rescue JSON::ParserError
20
+ Runtime::Logger.debug("#{self.class.name}##{__method__} attempted to parse invalid JSON at path: #{path}")
21
+ {}
19
22
  end
20
23
 
21
24
  def process
25
+ return [] if results.empty?
26
+
22
27
  results['examples'].map do |test|
23
28
  GitlabQuality::TestTooling::TestResult::JsonTestResult.new(report: test, project: project, token: token)
24
29
  end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module GitlabQuality
4
4
  module TestTooling
5
- VERSION = "2.0.0"
5
+ VERSION = "2.2.0"
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab_quality-test_tooling
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - GitLab Quality
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-10-14 00:00:00.000000000 Z
11
+ date: 2024-11-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: climate_control
@@ -192,6 +192,20 @@ dependencies:
192
192
  - - '='
193
193
  - !ruby/object:Gem::Version
194
194
  version: 3.7.0
195
+ - !ruby/object:Gem::Dependency
196
+ name: rspec_junit_formatter
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: 0.6.0
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: 0.6.0
195
209
  - !ruby/object:Gem::Dependency
196
210
  name: activesupport
197
211
  requirement: !ruby/object:Gem::Requirement
@@ -400,6 +414,7 @@ description: A collection of test-related tools.
400
414
  email:
401
415
  - quality@gitlab.com
402
416
  executables:
417
+ - existing-test-health-issue
403
418
  - failed-test-issues
404
419
  - flaky-test-issues
405
420
  - generate-test-session
@@ -429,6 +444,7 @@ files:
429
444
  - LICENSE.txt
430
445
  - README.md
431
446
  - Rakefile
447
+ - exe/existing-test-health-issue
432
448
  - exe/failed-test-issues
433
449
  - exe/flaky-test-issues
434
450
  - exe/generate-test-session
@@ -477,6 +493,7 @@ files:
477
493
  - lib/gitlab_quality/test_tooling/report/results_in_issues.rb
478
494
  - lib/gitlab_quality/test_tooling/report/results_in_test_cases.rb
479
495
  - lib/gitlab_quality/test_tooling/report/slow_test_issue.rb
496
+ - lib/gitlab_quality/test_tooling/report/test_health_issue_finder.rb
480
497
  - lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb
481
498
  - lib/gitlab_quality/test_tooling/runtime/env.rb
482
499
  - lib/gitlab_quality/test_tooling/runtime/logger.rb