gitlab_quality-test_tooling 0.8.1 → 0.8.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.tool-versions +1 -0
- data/Gemfile.lock +1 -1
- data/README.md +12 -0
- data/exe/slow-test-issues +50 -0
- data/lib/gitlab_quality/test_tooling/report/concerns/group_and_category_labels.rb +24 -0
- data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +1 -12
- data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb +3 -3
- data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb +95 -0
- data/lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb +16 -18
- data/lib/gitlab_quality/test_tooling/test_results/test_result.rb +12 -0
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a3472eb65666a6691797e9e0f0332a1773bb6babe898245a375bd9976523866e
|
4
|
+
data.tar.gz: 6372e6df38afb56bbe300b39d25a65cb7cdd205765ded99cab3745f4fbe7d812
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 899483550e08f6f873d137d6a1bc90620ed58eff093827e76e5b4c92de8b7b1f95bba60181d2b726d79bf91b08b627cc93fb3817662dff73db8ba46e81557aad
|
7
|
+
data.tar.gz: 6b21fae0eb76efa44dd868113d22d36c30a9f5d06b5bb1b625dd42129a463a24ae77abf4edf6df05d06800b29261116492cb39aa892490aaa22cfca9ce442aa4
|
data/.tool-versions
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby 3.0.5
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -121,6 +121,18 @@ Usage: exe/update-screenshot-paths [options]
|
|
121
121
|
-h, --help Show the usage
|
122
122
|
```
|
123
123
|
|
124
|
+
### `slow-test-issues`
|
125
|
+
|
126
|
+
```shell
|
127
|
+
Purpose: Create slow test issues from JSON RSpec report files
|
128
|
+
Usage: exe/slow-test-issue [options]
|
129
|
+
-i, --input-files INPUT_FILES JSON RSpec report files JSON
|
130
|
+
-p, --project PROJECT Can be an integer or a group/project string
|
131
|
+
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
132
|
+
--dry-run Perform a dry-run (don't create issues)
|
133
|
+
-v, --version Show the version
|
134
|
+
-h, --help Show the usage
|
135
|
+
```
|
124
136
|
## Development
|
125
137
|
|
126
138
|
### Initial setup
|
@@ -0,0 +1,50 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "optparse"
|
6
|
+
|
7
|
+
require_relative "../lib/gitlab_quality/test_tooling"
|
8
|
+
|
9
|
+
params = {}
|
10
|
+
|
11
|
+
options = OptionParser.new do |opts|
|
12
|
+
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
|
+
|
14
|
+
opts.on('-i', '--input-files INPUT_FILES', String, 'JSON RSpec report files JSON') do |input_files|
|
15
|
+
params[:input_files] = input_files
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
|
19
|
+
params[:project] = project
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
|
23
|
+
params[:token] = token
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on('--dry-run', "Perform a dry-run (don't create issues)") do
|
27
|
+
params[:dry_run] = true
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on_tail('-v', '--version', 'Show the version') do
|
31
|
+
require_relative "../lib/gitlab_quality/test_tooling/version"
|
32
|
+
puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
|
33
|
+
exit
|
34
|
+
end
|
35
|
+
|
36
|
+
opts.on_tail('-h', '--help', 'Show the usage') do
|
37
|
+
puts "Purpose: Create slow test issues from JSON RSpec report files"
|
38
|
+
puts opts
|
39
|
+
exit
|
40
|
+
end
|
41
|
+
|
42
|
+
opts.parse(ARGV)
|
43
|
+
end
|
44
|
+
|
45
|
+
if params.any?
|
46
|
+
GitlabQuality::TestTooling::Report::SlowTestIssue.new(**params).invoke!
|
47
|
+
else
|
48
|
+
puts options
|
49
|
+
exit 1
|
50
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module Report
|
6
|
+
module Concerns
|
7
|
+
module GroupAndCategoryLabels
|
8
|
+
def labels_inference
|
9
|
+
@labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
|
10
|
+
end
|
11
|
+
|
12
|
+
def new_issue_labels(test)
|
13
|
+
puts " => [DEBUG] product_group: #{test.product_group}; feature_category: #{test.feature_category}"
|
14
|
+
|
15
|
+
new_labels = self.class::NEW_ISSUE_LABELS +
|
16
|
+
labels_inference.infer_labels_from_product_group(test.product_group) +
|
17
|
+
labels_inference.infer_labels_from_feature_category(test.feature_category)
|
18
|
+
up_to_date_labels(test: test, new_labels: new_labels)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -16,6 +16,7 @@ module GitlabQuality
|
|
16
16
|
# - Add the failed job to the issue description, and update labels
|
17
17
|
class RelateFailureIssue < ReportAsIssue
|
18
18
|
include Concerns::FindSetDri
|
19
|
+
include Concerns::GroupAndCategoryLabels
|
19
20
|
|
20
21
|
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
|
21
22
|
SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
|
@@ -313,18 +314,6 @@ module GitlabQuality
|
|
313
314
|
"1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
|
314
315
|
end
|
315
316
|
|
316
|
-
def labels_inference
|
317
|
-
@labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
|
318
|
-
end
|
319
|
-
|
320
|
-
def new_issue_labels(test)
|
321
|
-
puts " => [DEBUG] product_group: #{test.product_group}; feature_category: #{test.feature_category}"
|
322
|
-
new_labels = NEW_ISSUE_LABELS +
|
323
|
-
labels_inference.infer_labels_from_product_group(test.product_group) +
|
324
|
-
labels_inference.infer_labels_from_feature_category(test.feature_category)
|
325
|
-
up_to_date_labels(test: test, new_labels: new_labels)
|
326
|
-
end
|
327
|
-
|
328
317
|
def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
|
329
318
|
(Set.new(base_issue_labels) + (super << pipeline_name_label)).to_a
|
330
319
|
end
|
@@ -30,7 +30,7 @@ module GitlabQuality
|
|
30
30
|
raise NotImplementedError
|
31
31
|
end
|
32
32
|
|
33
|
-
def
|
33
|
+
def test_hash(test)
|
34
34
|
OpenSSL::Digest::SHA256.hexdigest(test.file + test.name)
|
35
35
|
end
|
36
36
|
|
@@ -42,7 +42,7 @@ module GitlabQuality
|
|
42
42
|
| ------ | ------ |
|
43
43
|
| File | #{test_file_link(test)} |
|
44
44
|
| Description | `#{test.name}` |
|
45
|
-
| Hash | `#{
|
45
|
+
| Hash | `#{test_hash(test)}` |
|
46
46
|
#{"| Test case | #{test.testcase} |" if test.testcase}
|
47
47
|
DESCRIPTION
|
48
48
|
end
|
@@ -50,7 +50,7 @@ module GitlabQuality
|
|
50
50
|
def test_file_link(test)
|
51
51
|
path_prefix = test.file.start_with?('qa/') ? 'qa/' : ''
|
52
52
|
|
53
|
-
"[`#{path_prefix}#{test.file}`](#{FILE_BASE_URL}#{path_prefix}#{test.file})"
|
53
|
+
"[`#{path_prefix}#{test.file}`](#{FILE_BASE_URL}#{path_prefix}#{test.file}##{test.line_number})"
|
54
54
|
end
|
55
55
|
|
56
56
|
def new_issue_labels(_test)
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module Report
|
6
|
+
# Uses the API to create GitLab issues for slow tests
|
7
|
+
#
|
8
|
+
# - Takes the JSON test reports like rspec-*.json`
|
9
|
+
# - Takes a project where slow issues should be created
|
10
|
+
# - Find issue by title (with test description or test file)
|
11
|
+
# - Add test metadata, duration to the issue with group and category labels
|
12
|
+
class SlowTestIssue < ReportAsIssue
|
13
|
+
include Concerns::FindSetDri
|
14
|
+
include Concerns::GroupAndCategoryLabels
|
15
|
+
|
16
|
+
NEW_ISSUE_LABELS = Set.new(%w[test type::maintenance maintenance::performance priority::3 severity::3]).freeze
|
17
|
+
SEARCH_LABELS = %w[test maintenance::performance].freeze
|
18
|
+
|
19
|
+
MultipleIssuesFound = Class.new(StandardError)
|
20
|
+
|
21
|
+
TestLevelSpecification = Struct.new(:regex, :max_duration)
|
22
|
+
|
23
|
+
OTHER_TESTS_MAX_DURATION = 45.40 # seconds
|
24
|
+
|
25
|
+
TEST_LEVEL_SPECIFICATIONS = [
|
26
|
+
TestLevelSpecification.new(%r{/features/}, 50.13),
|
27
|
+
TestLevelSpecification.new(%r{/controllers|requests/}, 19.20),
|
28
|
+
TestLevelSpecification.new(%r{/lib/}, 27.12)
|
29
|
+
].freeze
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def run!
|
34
|
+
puts "Reporting slow tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
35
|
+
|
36
|
+
TestResults::Builder.new(files).test_results_per_file do |test_results|
|
37
|
+
puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
|
38
|
+
|
39
|
+
test_results.each do |test|
|
40
|
+
create_slow_issue(test) if should_create_slow_issue?(test)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def new_issue_title(test)
|
46
|
+
"Slow test in #{super}"
|
47
|
+
end
|
48
|
+
|
49
|
+
def new_issue_description(test)
|
50
|
+
super + [
|
51
|
+
"\n### Slow test",
|
52
|
+
"Slow tests detected, see guides for more details and how to improve them:",
|
53
|
+
"- [Top slow tests](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#top-slow-tests)",
|
54
|
+
"- [Test speed](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)",
|
55
|
+
"**Duration**: #{test.run_time} seconds"
|
56
|
+
].compact.join("\n\n")
|
57
|
+
end
|
58
|
+
|
59
|
+
def create_slow_issue(test)
|
60
|
+
puts " => Finding existing issues for slow test '#{test.name}' (run time: #{test.run_time} seconds)..."
|
61
|
+
|
62
|
+
issue = find_issue(test)
|
63
|
+
|
64
|
+
puts " => Existing issue link #{issue['web_url']}" if issue.present?
|
65
|
+
|
66
|
+
create_issue(test) unless issue.present?
|
67
|
+
rescue MultipleIssuesFound => e
|
68
|
+
warn(e.message)
|
69
|
+
end
|
70
|
+
|
71
|
+
def find_issue(test)
|
72
|
+
search_labels = SEARCH_LABELS
|
73
|
+
|
74
|
+
gitlab.find_issues(options: { state: 'opened', labels: search_labels.to_a }).find do |issue|
|
75
|
+
issue_title = issue.title.strip
|
76
|
+
issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def should_create_slow_issue?(test)
|
81
|
+
test.run_time > max_duration_for_test(test)
|
82
|
+
end
|
83
|
+
|
84
|
+
def max_duration_for_test(test)
|
85
|
+
test_level_specification = TEST_LEVEL_SPECIFICATIONS.find do |test_level_specification|
|
86
|
+
test.example_id =~ test_level_specification.regex
|
87
|
+
end
|
88
|
+
return OTHER_TESTS_MAX_DURATION unless test_level_specification
|
89
|
+
|
90
|
+
test_level_specification.max_duration
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -12,18 +12,12 @@ module GitlabQuality
|
|
12
12
|
@input_files = input_files
|
13
13
|
end
|
14
14
|
|
15
|
-
REGEX = %r{(?<gitlab_qa_run>gitlab-qa-run-.*?(?=/))/(?<gitlab_ce_ee_qa>gitlab-(?:ee|ce)-qa-.*?(?=/))}
|
16
15
|
CONTAINER_PATH = File.join('/home', 'gitlab', 'qa', 'tmp').freeze
|
17
16
|
|
18
17
|
def invoke!
|
19
18
|
Dir.glob(input_files).each do |input_file|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
host_relative_path = "#{match_data[:gitlab_qa_run]}/#{match_data[:gitlab_ce_ee_qa]}"
|
24
|
-
|
25
|
-
rewrite_schreenshot_paths_in_junit_file(input_file, host_relative_path)
|
26
|
-
rewrite_schreenshot_paths_in_json_file(input_file.gsub('.xml', '.json'), host_relative_path)
|
19
|
+
rewrite_schreenshot_paths_in_junit_file(input_file)
|
20
|
+
rewrite_schreenshot_paths_in_json_file(input_file.gsub('.xml', '.json'))
|
27
21
|
end
|
28
22
|
end
|
29
23
|
|
@@ -31,35 +25,35 @@ module GitlabQuality
|
|
31
25
|
|
32
26
|
attr_reader :input_files
|
33
27
|
|
34
|
-
def rewrite_schreenshot_paths_in_junit_file(junit_file
|
28
|
+
def rewrite_schreenshot_paths_in_junit_file(junit_file)
|
35
29
|
File.write(
|
36
30
|
junit_file,
|
37
|
-
rewrite_each_junit_screenshot_path(junit_file
|
31
|
+
rewrite_each_junit_screenshot_path(junit_file).to_s
|
38
32
|
)
|
39
33
|
|
40
34
|
puts "Saved #{junit_file}"
|
41
35
|
end
|
42
36
|
|
43
|
-
def rewrite_schreenshot_paths_in_json_file(json_file
|
37
|
+
def rewrite_schreenshot_paths_in_json_file(json_file)
|
44
38
|
File.write(
|
45
39
|
json_file,
|
46
40
|
JSON.pretty_generate(
|
47
|
-
rewrite_each_json_screenshot_path(json_file
|
41
|
+
rewrite_each_json_screenshot_path(json_file)
|
48
42
|
)
|
49
43
|
)
|
50
44
|
|
51
45
|
puts "Saved #{json_file}"
|
52
46
|
end
|
53
47
|
|
54
|
-
def rewrite_each_junit_screenshot_path(junit_file
|
48
|
+
def rewrite_each_junit_screenshot_path(junit_file)
|
55
49
|
Nokogiri::XML(File.open(junit_file)).tap do |report|
|
56
50
|
report.xpath('//system-out').each do |system_out|
|
57
|
-
system_out.content = remove_container_absolute_path_prefix(system_out.content,
|
51
|
+
system_out.content = remove_container_absolute_path_prefix(system_out.content, test_artifacts_directory(junit_file))
|
58
52
|
end
|
59
53
|
end
|
60
54
|
end
|
61
55
|
|
62
|
-
def rewrite_each_json_screenshot_path(json_file
|
56
|
+
def rewrite_each_json_screenshot_path(json_file)
|
63
57
|
JSON.parse(File.read(json_file)).tap do |report|
|
64
58
|
examples = report['examples']
|
65
59
|
|
@@ -67,13 +61,17 @@ module GitlabQuality
|
|
67
61
|
next unless example['screenshot'].present?
|
68
62
|
|
69
63
|
example['screenshot']['image'] =
|
70
|
-
remove_container_absolute_path_prefix(example.dig('screenshot', 'image'),
|
64
|
+
remove_container_absolute_path_prefix(example.dig('screenshot', 'image'), test_artifacts_directory(json_file))
|
71
65
|
end
|
72
66
|
end
|
73
67
|
end
|
74
68
|
|
75
|
-
def remove_container_absolute_path_prefix(image_container_absolute_path,
|
76
|
-
image_container_absolute_path.gsub(CONTAINER_PATH,
|
69
|
+
def remove_container_absolute_path_prefix(image_container_absolute_path, test_artifacts_dir)
|
70
|
+
image_container_absolute_path.gsub(CONTAINER_PATH, test_artifacts_dir)
|
71
|
+
end
|
72
|
+
|
73
|
+
def test_artifacts_directory(filepath)
|
74
|
+
File.dirname(filepath)
|
77
75
|
end
|
78
76
|
end
|
79
77
|
end
|
@@ -115,6 +115,18 @@ module GitlabQuality
|
|
115
115
|
report['feature_category']
|
116
116
|
end
|
117
117
|
|
118
|
+
def run_time
|
119
|
+
report['run_time'].to_f.round(2)
|
120
|
+
end
|
121
|
+
|
122
|
+
def example_id
|
123
|
+
report['id']
|
124
|
+
end
|
125
|
+
|
126
|
+
def line_number
|
127
|
+
report['line_number']
|
128
|
+
end
|
129
|
+
|
118
130
|
private
|
119
131
|
|
120
132
|
# rubocop:disable Metrics/AbcSize
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab_quality-test_tooling
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-06-
|
11
|
+
date: 2023-06-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -323,6 +323,7 @@ executables:
|
|
323
323
|
- prepare-stage-reports
|
324
324
|
- relate-failure-issue
|
325
325
|
- report-results
|
326
|
+
- slow-test-issues
|
326
327
|
- update-screenshot-paths
|
327
328
|
extensions: []
|
328
329
|
extra_rdoc_files: []
|
@@ -331,6 +332,7 @@ files:
|
|
331
332
|
- ".rubocop.yml"
|
332
333
|
- ".rubocop_todo.yml"
|
333
334
|
- ".ruby-version"
|
335
|
+
- ".tool-versions"
|
334
336
|
- CODE_OF_CONDUCT.md
|
335
337
|
- CONTRIBUTING.md
|
336
338
|
- Dangerfile
|
@@ -345,6 +347,7 @@ files:
|
|
345
347
|
- exe/prepare-stage-reports
|
346
348
|
- exe/relate-failure-issue
|
347
349
|
- exe/report-results
|
350
|
+
- exe/slow-test-issues
|
348
351
|
- exe/update-screenshot-paths
|
349
352
|
- lefthook.yml
|
350
353
|
- lib/gitlab_quality/test_tooling.rb
|
@@ -352,6 +355,7 @@ files:
|
|
352
355
|
- lib/gitlab_quality/test_tooling/gitlab_issue_dry_client.rb
|
353
356
|
- lib/gitlab_quality/test_tooling/labels_inference.rb
|
354
357
|
- lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb
|
358
|
+
- lib/gitlab_quality/test_tooling/report/concerns/group_and_category_labels.rb
|
355
359
|
- lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
|
356
360
|
- lib/gitlab_quality/test_tooling/report/concerns/utils.rb
|
357
361
|
- lib/gitlab_quality/test_tooling/report/generate_test_session.rb
|
@@ -361,6 +365,7 @@ files:
|
|
361
365
|
- lib/gitlab_quality/test_tooling/report/report_results.rb
|
362
366
|
- lib/gitlab_quality/test_tooling/report/results_in_issues.rb
|
363
367
|
- lib/gitlab_quality/test_tooling/report/results_in_test_cases.rb
|
368
|
+
- lib/gitlab_quality/test_tooling/report/slow_test_issue.rb
|
364
369
|
- lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb
|
365
370
|
- lib/gitlab_quality/test_tooling/runtime/env.rb
|
366
371
|
- lib/gitlab_quality/test_tooling/runtime/logger.rb
|