gitlab_quality-test_tooling 1.15.0 → 1.17.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8364acd36abab9ce89c0846c346b893ca041a6a740ced7dc5c784da5990ea74c
4
- data.tar.gz: 9468d8b559aeec587f9c3722e4f6e9d539d790828f9ecc7f815cc39ca22f2657
3
+ metadata.gz: b8709476aafe6dc96d2b49d1d74b18b2e3dbf6c37cb8ebb0acd631d05dbfe2ac
4
+ data.tar.gz: 26d4d54613a522014b5f3b93f38c0cb62027c2e209573727265a5c2b630a6d9d
5
5
  SHA512:
6
- metadata.gz: 4d287ca74d5fe7bc1bd9acd25541eb6f150f717f95f9e8c0bdf7a53ac139f3dab5186565ec74f2f7279e50dd47268d22121eb94ba2e3955cc5f6de33e7a952c0
7
- data.tar.gz: e2954fea3d65bdce6e72f26a201965f45cc3aab583a34af9835f4d42f71540ad21582f4f8a96a12cd377a949644ab458ea213febfb2e6e90fbdc25c1867ddb72
6
+ metadata.gz: 194294ffed245e88a96e35480485f98c1748a8417766cdca1cf737ece2b090a4e29efc444fa449f5f569e292333ba50954f7c07a72fbb2f9225c9195c4b6f411
7
+ data.tar.gz: 30cc7f3a5da1c15b5c3508e0850c007bddf9f22a85c9ec1361821be7bc330afd3869cd9fe74d3b424afc2d0ad2d180c4c0649b5f502f4740ca68f1b2c8119933
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab_quality-test_tooling (1.15.0)
4
+ gitlab_quality-test_tooling (1.17.0)
5
5
  activesupport (>= 6.1, < 7.1)
6
6
  amatch (~> 0.4.1)
7
7
  gitlab (~> 4.19)
@@ -318,4 +318,4 @@ DEPENDENCIES
318
318
  webmock (= 3.7.0)
319
319
 
320
320
  BUNDLED WITH
321
- 2.4.2
321
+ 2.5.4
@@ -19,10 +19,6 @@ options = OptionParser.new do |opts|
19
19
  params[:project] = project
20
20
  end
21
21
 
22
- opts.on('-m', '--merge_request_iid MERGE_REQUEST_IID', String, 'An integer merge request IID') do |merge_request_iid|
23
- params[:merge_request_iid] = merge_request_iid
24
- end
25
-
26
22
  opts.on('--base-issue-labels BASE_ISSUE_LABELS', String,
27
23
  'Comma-separated labels (without tilde) to add to new flaky test issues') do |base_issue_labels|
28
24
  params[:base_issue_labels] = base_issue_labels.split(',')
@@ -32,6 +28,10 @@ options = OptionParser.new do |opts|
32
28
  params[:token] = token
33
29
  end
34
30
 
31
+ opts.on('-r', '--related-issues-file RELATED_ISSUES_FILE', String, 'The file path for the related issues') do |related_issues_file|
32
+ params[:related_issues_file] = related_issues_file
33
+ end
34
+
35
35
  opts.on('--dry-run', "Perform a dry-run (don't create issues)") do
36
36
  params[:dry_run] = true
37
37
  end
data/lefthook.yml CHANGED
@@ -13,3 +13,16 @@ pre-push:
13
13
  rubocop:
14
14
  run: bundle exec rubocop
15
15
  glob: '*.rb'
16
+
17
+ # Changelog git trailer for the first commit of the branch
18
+ changelog-on-first-commit:
19
+ run: |
20
+ first_commit_message=$(git log --format=%B -n 1 $(git log main..HEAD --pretty=format:"%h" | tail -1))
21
+ if ! echo ${first_commit_message} | grep "Changelog:"; then
22
+ echo Could not find a Changelog: git trailer on the first commit for this branch.
23
+ echo
24
+ echo Please add a trailer by amending the git commit message.
25
+ echo
26
+ echo See https://docs.gitlab.com/ee/development/changelog.html#overview for more info.
27
+ exit 1
28
+ fi
@@ -15,7 +15,7 @@ module GitlabQuality
15
15
  @retry_backoff = 0
16
16
  end
17
17
 
18
- def handle_gitlab_client_exceptions # rubocop:disable Metrics/AbcSize
18
+ def handle_gitlab_client_exceptions
19
19
  yield
20
20
  rescue Gitlab::Error::NotFound
21
21
  # This error could be raised in assert_user_permission!
@@ -27,12 +27,20 @@ module GitlabQuality
27
27
 
28
28
  raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS
29
29
 
30
- warn_exception(e)
30
+ warn("#{error.class.name} #{error.message}")
31
31
  warn("Sleeping for #{@retry_backoff} seconds before retrying...")
32
32
  sleep @retry_backoff
33
33
 
34
34
  retry
35
35
  rescue StandardError => e
36
+ post_exception_to_slack(e) if Runtime::Env.ci_commit_ref_name == Runtime::Env.default_branch
37
+
38
+ raise e
39
+ end
40
+
41
+ def post_exception_to_slack(error)
42
+ return unless ENV['CI_SLACK_WEBHOOK_URL']
43
+
36
44
  pipeline = Runtime::Env.pipeline_from_project_name
37
45
  channel = case pipeline
38
46
  when "canary"
@@ -42,9 +50,6 @@ module GitlabQuality
42
50
  else
43
51
  "qa-#{pipeline}"
44
52
  end
45
- error_msg = warn_exception(e)
46
-
47
- return unless Runtime::Env.ci_commit_ref_name == Runtime::Env.default_branch
48
53
 
49
54
  slack_options = {
50
55
  slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
@@ -54,7 +59,7 @@ module GitlabQuality
54
59
  message: <<~MSG
55
60
  An unexpected error occurred while reporting test results in issues.
56
61
  The error occurred in job: #{Runtime::Env.ci_job_url}
57
- `#{error_msg}`
62
+ `#{error.class.name} #{error.message}`
58
63
  MSG
59
64
  }
60
65
  puts "Posting Slack message to channel: #{channel}"
@@ -79,12 +84,6 @@ module GitlabQuality
79
84
  private_token: token
80
85
  )
81
86
  end
82
-
83
- def warn_exception(error)
84
- error_msg = "#{error.class.name} #{error.message}"
85
- warn(error_msg)
86
- error_msg
87
- end
88
87
  end
89
88
  end
90
89
  end
@@ -48,6 +48,12 @@ module GitlabQuality
48
48
  end
49
49
  end
50
50
 
51
+ def find_issue_notes(iid:)
52
+ handle_gitlab_client_exceptions do
53
+ client.issue_notes(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
54
+ end
55
+ end
56
+
51
57
  def find_issue_discussions(iid:)
52
58
  handle_gitlab_client_exceptions do
53
59
  client.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
@@ -75,12 +81,6 @@ module GitlabQuality
75
81
  end
76
82
  end
77
83
 
78
- def find_issue_notes(iid:)
79
- handle_gitlab_client_exceptions do
80
- client.issue_notes(project, iid, order_by: 'created_at', sort: 'asc')&.auto_paginate
81
- end
82
- end
83
-
84
84
  def create_issue_note(iid:, note:)
85
85
  handle_gitlab_client_exceptions do
86
86
  client.create_issue_note(project, iid, note)
@@ -1,13 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'active_support/core_ext/object/blank'
4
+
3
5
  module GitlabQuality
4
6
  module TestTooling
5
7
  module Report
6
8
  module Concerns
7
9
  module IssueReports
8
- JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
10
+ JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
9
11
  FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
10
- REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
12
+ REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>\S+)\)/
13
+ LATEST_REPORTS_TO_SHOW = 10
11
14
 
12
15
  def initial_reports_section(test)
13
16
  <<~REPORTS
@@ -17,22 +20,21 @@ module GitlabQuality
17
20
  REPORTS
18
21
  end
19
22
 
20
- def add_report_to_issue_description(issue, test)
21
- issue_description = issue.description
22
-
23
- # We include the number of reports in the header, for visibility.
24
- new_issue_description =
25
- if issue_description.include?('### Reports')
26
- # We count the number of existing reports.
27
- reports_count = issue_description
28
- .scan(REPORT_ITEM_REGEX)
29
- .size.to_i + 1
30
- issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
31
- else # For issue with the legacy format, we add the Reports section
32
- update_legacy_issue_description(issue_description)
33
- end
23
+ def increment_reports(
24
+ current_reports_content:,
25
+ test:,
26
+ reports_section_header: '### Reports',
27
+ item_extra_content: nil,
28
+ reports_extra_content: nil)
29
+ preserved_content = current_reports_content.split(reports_section_header).first&.strip
30
+ reports = report_lines(current_reports_content) + [report_list_item(test, item_extra_content: item_extra_content)]
34
31
 
35
- [new_issue_description, report_list_item(test)].join("\n")
32
+ [
33
+ preserved_content,
34
+ "#{reports_section_header} (#{reports.size})",
35
+ reports_list(reports),
36
+ reports_extra_content
37
+ ].reject(&:blank?).compact.join("\n\n")
36
38
  end
37
39
 
38
40
  def failed_issue_job_url(issue)
@@ -49,8 +51,28 @@ module GitlabQuality
49
51
 
50
52
  private
51
53
 
52
- def report_list_item(test)
53
- "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
54
+ def report_lines(content)
55
+ content.lines.grep(REPORT_ITEM_REGEX).map(&:strip)
56
+ end
57
+
58
+ def reports_list(reports)
59
+ sorted_reports = reports.sort.reverse
60
+
61
+ if sorted_reports.size > LATEST_REPORTS_TO_SHOW
62
+ [
63
+ "Last 10 reports:",
64
+ sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
65
+ "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
66
+ sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
67
+ "</details>"
68
+ ].join("\n\n")
69
+ else
70
+ sorted_reports.join("\n")
71
+ end
72
+ end
73
+
74
+ def report_list_item(test, item_extra_content: nil)
75
+ "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')}) #{item_extra_content}".strip
54
76
  end
55
77
 
56
78
  def job_urls_from_description(issue_description, regex)
@@ -60,15 +82,6 @@ module GitlabQuality
60
82
  end
61
83
  end
62
84
 
63
- def update_legacy_issue_description(issue_description)
64
- test_captures = issue_description.scan(JOB_URL_REGEX)
65
- reports_count = test_captures.size.to_i + 1
66
-
67
- updated_description = "#{issue_description}\n\n### Reports (#{reports_count})\n"
68
- updated_description = [updated_description, *test_captures_to_report_items(test_captures)].join("\n") unless test_captures.empty?
69
- updated_description
70
- end
71
-
72
85
  def test_captures_to_report_items(test_captures)
73
86
  test_captures.map do |ci_job_url, _, _|
74
87
  report_list_item(GitlabQuality::TestTooling::TestResult::JsonTestResult.new(
@@ -16,7 +16,7 @@ module GitlabQuality
16
16
  end
17
17
 
18
18
  def new_issue_title(test)
19
- "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
19
+ "[Test] #{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
20
20
  end
21
21
 
22
22
  def partial_file_path(path)
@@ -13,11 +13,19 @@ module GitlabQuality
13
13
  # - Find issue by test hash
14
14
  # - Reopen issue if it already exists, but is closed
15
15
  class FlakyTestIssue < ReportAsIssue
16
+ include Concerns::GroupAndCategoryLabels
16
17
  include Concerns::IssueReports
17
18
 
18
- NEW_ISSUE_LABELS = Set.new(['type::maintenance', 'priority::3', 'severity::3', 'failure::flaky-test']).freeze
19
- FOUND_IN_MR_LABEL = 'found:in MR'
20
- FOUND_IN_MASTER_LABEL = 'found:master'
19
+ IDENTITY_LABELS = ['test', 'failure::flaky-test', 'automation:bot-authored'].freeze
20
+ NEW_ISSUE_LABELS = Set.new(['type::maintenance', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
21
+ SEARCH_LABELS = ['test'].freeze
22
+ FOUND_IN_MR_LABEL = '~"found:in MR"'
23
+ FOUND_IN_MASTER_LABEL = '~"found:master"'
24
+ REPORT_SECTION_HEADER = '### Flakiness reports'
25
+ REPORTS_DOCUMENTATION = <<~DOC
26
+ Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html)
27
+ to learn more about how to reproduce them.
28
+ DOC
21
29
 
22
30
  def initialize(
23
31
  token:,
@@ -25,18 +33,16 @@ module GitlabQuality
25
33
  base_issue_labels: nil,
26
34
  confidential: false,
27
35
  dry_run: false,
28
- merge_request_iid: nil,
29
36
  project: nil,
30
37
  **_kwargs)
31
38
  super(token: token, input_files: input_files, project: project, confidential: confidential, dry_run: dry_run)
32
39
 
33
40
  @base_issue_labels = Set.new(base_issue_labels)
34
- @merge_request_iid = merge_request_iid
35
41
  end
36
42
 
37
43
  private
38
44
 
39
- attr_reader :base_issue_labels, :merge_request_iid
45
+ attr_reader :base_issue_labels
40
46
 
41
47
  def run!
42
48
  puts "Reporting flaky tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
@@ -44,62 +50,91 @@ module GitlabQuality
44
50
  TestResults::Builder.new(files).test_results_per_file do |test_results|
45
51
  puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
46
52
 
47
- test_results.each do |test|
48
- next if test.status != 'passed' # We only want failed tests that passed in the end
49
-
50
- create_flaky_issue(test)
51
- end
53
+ process_test_results(test_results)
52
54
  end
53
55
  end
54
56
 
55
- def new_issue_title(test)
56
- "Flaky test in #{super}"
57
- end
57
+ def process_test_results(test_results)
58
+ test_results.each do |test|
59
+ next unless test_is_applicable?(test)
58
60
 
59
- def new_issue_labels(_test)
60
- found_label =
61
- if !merge_request_iid || merge_request_iid.empty?
62
- FOUND_IN_MASTER_LABEL
63
- else
64
- FOUND_IN_MR_LABEL
65
- end
61
+ puts " => Reporting flakiness for test '#{test.name}'..."
66
62
 
67
- NEW_ISSUE_LABELS + base_issue_labels + [found_label]
63
+ issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: SEARCH_LABELS)
64
+ issues << create_issue(test) if issues.empty?
65
+
66
+ update_reports(issues, test)
67
+ collect_issues(test, issues)
68
+ end
68
69
  end
69
70
 
70
- def new_issue_description(test)
71
- super + [
72
- "Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html) " \
73
- "to learn more about how to reproduce them.",
74
- initial_reports_section(test)
75
- ].compact.join("\n\n")
71
+ def test_is_applicable?(test)
72
+ test.status == 'passed' # We only want failed tests that passed in the end
76
73
  end
77
74
 
78
- def create_flaky_issue(test)
79
- puts " => Finding existing issues for flaky test '#{test.name}' (run time: #{test.run_time} seconds)..."
75
+ def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
76
+ (base_issue_labels + super).to_a
77
+ end
80
78
 
81
- issues = find_issues_by_hash(test_hash(test))
79
+ def update_reports(issues, test)
82
80
  issues.each do |issue|
83
- puts " => Existing issue link #{issue.web_url}."
81
+ puts " => Adding the flaky test to the existing issue: #{issue.web_url}"
82
+ add_report_to_issue(issue: issue, test: test, related_issues: (issues - [issue]))
83
+ end
84
+ end
84
85
 
85
- puts " => Adding the flaky test to the existing issue..."
86
- add_report_to_issue(issue, test)
86
+ def add_report_to_issue(issue:, test:, related_issues:)
87
+ reports_note = existing_reports_note(issue: issue)
88
+ note_body = [
89
+ report_body(reports_note: reports_note, test: test),
90
+ identity_labels_quick_action,
91
+ relate_issues_quick_actions(related_issues)
92
+ ].join("\n")
93
+
94
+ if reports_note
95
+ gitlab.edit_issue_note(
96
+ issue_iid: issue.iid,
97
+ note_id: reports_note.id,
98
+ note: note_body
99
+ )
100
+ else
101
+ gitlab.create_issue_note(iid: issue.iid, note: note_body)
102
+ end
103
+ end
87
104
 
88
- if issue.state == 'closed'
89
- puts " => Issue is closed. Reopening it."
90
- reopen_issue(issue)
91
- end
105
+ def existing_reports_note(issue:)
106
+ gitlab.find_issue_notes(iid: issue.iid).find do |note|
107
+ note.body.start_with?(REPORT_SECTION_HEADER)
92
108
  end
109
+ end
93
110
 
94
- create_issue(test) unless issues.any?
111
+ def report_body(reports_note:, test:)
112
+ increment_reports(
113
+ current_reports_content: reports_note&.body.to_s,
114
+ test: test,
115
+ reports_section_header: REPORT_SECTION_HEADER,
116
+ item_extra_content: found_label,
117
+ reports_extra_content: REPORTS_DOCUMENTATION
118
+ )
119
+ end
120
+
121
+ def found_label
122
+ if ENV.key?('CI_MERGE_REQUEST_IID')
123
+ FOUND_IN_MR_LABEL
124
+ else
125
+ FOUND_IN_MASTER_LABEL
126
+ end
95
127
  end
96
128
 
97
- def add_report_to_issue(issue, test)
98
- gitlab.edit_issue(iid: issue.iid, options: { description: add_report_to_issue_description(issue, test) })
129
+ def identity_labels_quick_action
130
+ labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
131
+ %(/label #{labels_list})
99
132
  end
100
133
 
101
- def reopen_issue(issue)
102
- gitlab.edit_issue(iid: issue.iid, options: { state_event: 'reopen' })
134
+ def relate_issues_quick_actions(issues)
135
+ issues.map do |issue|
136
+ "/relate #{issue.web_url}"
137
+ end.join("\n")
103
138
  end
104
139
  end
105
140
  end
@@ -205,12 +205,9 @@ module GitlabQuality
205
205
 
206
206
  def generate_test_text(testcase, tests_with_same_testcase, passed)
207
207
  text = tests_with_same_testcase.map(&:name).uniq.join(', ')
208
- encoded_text = ERB::Util.url_encode(text)
209
208
 
210
209
  if testcase && !passed
211
- # Workaround for reducing system notes on testcase issues
212
- # The first regex extracts the link to the issues list page from a link to a single issue show page by removing the issue id.
213
- "[#{text}](#{testcase.match(%r{[\s\S]+/[^/\d]+})}?state=opened&search=#{encoded_text})"
210
+ "[#{text}](#{testcase})"
214
211
  else
215
212
  text
216
213
  end
@@ -99,7 +99,6 @@ module GitlabQuality
99
99
  end
100
100
 
101
101
  def update_issue(issue:, spec_run_time:)
102
- state_event = issue.state == 'closed' ? 'reopen' : nil
103
102
  updated_description = <<~MARKDOWN.chomp
104
103
  #{issue.description}
105
104
 
@@ -110,8 +109,6 @@ module GitlabQuality
110
109
  description: updated_description
111
110
  }
112
111
 
113
- issue_attrs[:state_event] = state_event if state_event
114
-
115
112
  gitlab.edit_issue(iid: issue.iid, options: issue_attrs)
116
113
  puts " => Added a report in #{issue.web_url}!"
117
114
  end
@@ -190,8 +190,8 @@ module GitlabQuality
190
190
  end
191
191
 
192
192
  def failure_issues(test)
193
- find_issues_for_test(
194
- test,
193
+ find_issues_by_hash(
194
+ test_hash(test),
195
195
  state: 'opened',
196
196
  labels: base_issue_labels + Set.new(%w[test]),
197
197
  not_labels: exclude_labels_for_search
@@ -347,7 +347,7 @@ module GitlabQuality
347
347
  state_event = issue.state == 'closed' ? 'reopen' : nil
348
348
 
349
349
  issue_attrs = {
350
- description: add_report_to_issue_description(issue, test),
350
+ description: increment_reports(current_reports_content: issue.description, test: test),
351
351
  labels: up_to_date_labels(test: test, issue: issue)
352
352
  }
353
353
  issue_attrs[:state_event] = state_event if state_event
@@ -356,10 +356,6 @@ module GitlabQuality
356
356
  puts " => Added a report in '#{issue.title}': #{issue.web_url}!"
357
357
  end
358
358
 
359
- def new_issue_title(test)
360
- "Failure in #{super}"
361
- end
362
-
363
359
  def screenshot_section(test)
364
360
  return unless test.screenshot?
365
361
 
@@ -60,7 +60,7 @@ module GitlabQuality
60
60
  | Description | `#{test.name}` |
61
61
  | Test level | #{test.level} |
62
62
  | Hash | `#{test_hash(test)}` |
63
- | Duration | #{test.run_time} seconds |
63
+ | Reference duration | #{test.run_time} seconds |
64
64
  | Expected duration | < #{test.max_duration_for_test} seconds |
65
65
  #{"| Test case | #{test.testcase} |" if test.testcase}
66
66
  DESCRIPTION
@@ -142,8 +142,10 @@ module GitlabQuality
142
142
  labels
143
143
  end
144
144
 
145
- def find_issues_by_hash(test_hash)
146
- search_options = { search: test_hash }
145
+ def find_issues_by_hash(test_hash, labels: Set.new, not_labels: Set.new, state: nil)
146
+ search_options = { search: test_hash, labels: labels.to_a, not: { labels: not_labels.to_a } }
147
+ search_options[:state] = state if state
148
+ search_options[:in] = 'description'
147
149
  gitlab.find_issues(options: search_options)
148
150
  end
149
151
 
@@ -10,16 +10,21 @@ module GitlabQuality
10
10
  # - Find issue by title (with test description or test file)
11
11
  # - Add test metadata, duration to the issue with group and category labels
12
12
  class SlowTestIssue < ReportAsIssue
13
- include TestTooling::Concerns::FindSetDri
14
13
  include Concerns::GroupAndCategoryLabels
14
+ include Concerns::IssueReports
15
15
 
16
- NEW_ISSUE_LABELS = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'rspec profiling', 'rspec:slow test']).freeze
17
- SEARCH_LABELS = %w[test maintenance::performance].freeze
16
+ IDENTITY_LABELS = ['test', 'rspec:slow test', 'rspec profiling', 'automation:bot-authored'].freeze
17
+ NEW_ISSUE_LABELS = Set.new(['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3']).freeze
18
+ SEARCH_LABELS = ['test'].freeze
19
+ FOUND_IN_MR_LABEL = '~"found:in MR"'
20
+ FOUND_IN_MASTER_LABEL = '~"found:master"'
21
+ REPORT_SECTION_HEADER = '### Slowness reports'
22
+ REPORTS_DOCUMENTATION = <<~DOC
23
+ Slow tests were detected, please see the [test speed best practices guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)
24
+ to improve them. More context available about this issue in the [top slow tests guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#top-slow-tests).
18
25
 
19
- JOB_URL_REGEX = %r{(?<job_url>https://(?<host>[\w.]+)/(?<project_path>[\w\-./]+)/-/jobs/\d+)}
20
- REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
21
-
22
- MultipleIssuesFound = Class.new(StandardError)
26
+ Add `allowed_to_be_slow: true` to the RSpec test if this is a legit slow test and close the issue.
27
+ DOC
23
28
 
24
29
  private
25
30
 
@@ -29,99 +34,87 @@ module GitlabQuality
29
34
  TestResults::Builder.new(files).test_results_per_file do |test_results|
30
35
  puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
31
36
 
32
- test_results.each do |test|
33
- create_slow_issue(test) if test.slow_test?
34
- end
37
+ process_test_results(test_results)
35
38
  end
36
39
  end
37
40
 
38
- def new_issue_title(test)
39
- "Slow test in #{super}"
40
- end
41
+ def process_test_results(test_results)
42
+ test_results.each do |test|
43
+ next unless test.slow_test?
41
44
 
42
- def new_issue_description(test)
43
- super +
44
- <<~DESCRIPTION.chomp
45
- Slow tests were detected, please see the [test speed best practices guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)
46
- to improve them. More context available about this issue in the [top slow tests guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#top-slow-tests).
45
+ puts " => Reporting slowness for test '#{test.name}'..."
47
46
 
48
- Add `allowed_to_be_slow: true` to the RSpec test if this is a legit slow test and close the issue.
47
+ issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: SEARCH_LABELS)
48
+ issues << create_issue(test) if issues.empty?
49
49
 
50
- #{reports_section(test)}
51
- DESCRIPTION
50
+ update_reports(issues, test)
51
+ collect_issues(test, issues)
52
+ end
52
53
  end
53
54
 
54
- def reports_section(test)
55
- <<~REPORTS
56
- ### Reports (1)
57
-
58
- #{report_list_item(test)}
59
- REPORTS
55
+ def test_is_applicable?(test)
56
+ test.slow_test?
60
57
  end
61
58
 
62
- def report_list_item(test)
63
- "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
59
+ def update_reports(issues, test)
60
+ issues.each do |issue|
61
+ puts " => Adding the slow test to the existing issue: #{issue.web_url}"
62
+ add_report_to_issue(issue: issue, test: test, related_issues: (issues - [issue]))
63
+ end
64
64
  end
65
65
 
66
- def slow_test_issues(test)
67
- find_issues_for_test(
68
- test,
69
- state: 'opened',
70
- labels: SEARCH_LABELS
71
- )
66
+ def add_report_to_issue(issue:, test:, related_issues:)
67
+ reports_note = existing_reports_note(issue: issue)
68
+ note_body = [
69
+ report_body(reports_note: reports_note, test: test),
70
+ identity_labels_quick_action,
71
+ relate_issues_quick_actions(related_issues)
72
+ ].join("\n")
73
+
74
+ if reports_note
75
+ gitlab.edit_issue_note(
76
+ issue_iid: issue.iid,
77
+ note_id: reports_note.id,
78
+ note: note_body
79
+ )
80
+ else
81
+ gitlab.create_issue_note(iid: issue.iid, note: note_body)
82
+ end
72
83
  end
73
84
 
74
- def create_slow_issue(test)
75
- puts " => Finding existing issues for slow test '#{test.name}' (run time: #{test.run_time} seconds)..."
85
+ def existing_reports_note(issue:)
86
+ gitlab.find_issue_notes(iid: issue.iid).find do |note|
87
+ note.body.start_with?(REPORT_SECTION_HEADER)
88
+ end
89
+ end
76
90
 
77
- issues = slow_test_issues(test)
91
+ def report_body(reports_note:, test:)
92
+ increment_reports(
93
+ current_reports_content: reports_note&.body.to_s,
94
+ test: test,
95
+ reports_section_header: REPORT_SECTION_HEADER,
96
+ item_extra_content: found_label,
97
+ reports_extra_content: REPORTS_DOCUMENTATION
98
+ )
99
+ end
78
100
 
79
- if issues.blank?
80
- issues << create_issue(test)
101
+ def found_label
102
+ if ENV.key?('CI_MERGE_REQUEST_IID')
103
+ FOUND_IN_MR_LABEL
81
104
  else
82
- issues.each do |issue|
83
- puts " => Existing issue link #{issue['web_url']}"
84
-
85
- update_reports(issue, test)
86
- end
105
+ FOUND_IN_MASTER_LABEL
87
106
  end
88
-
89
- collect_issues(test, issues)
90
- rescue MultipleIssuesFound => e
91
- warn(e.message)
92
107
  end
93
108
 
94
- def update_reports(issue, test)
95
- # We reopen closed issues to not lose any history
96
- state_event = issue.state == 'closed' ? 'reopen' : nil
97
-
98
- issue_attrs = {
99
- description: up_to_date_issue_description(issue.description, test)
100
- }
101
-
102
- issue_attrs[:state_event] = state_event if state_event
103
-
104
- gitlab.edit_issue(iid: issue.iid, options: issue_attrs)
105
- puts " => Added a report in '#{issue.title}': #{issue.web_url}!"
109
+ def identity_labels_quick_action
110
+ labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
111
+ %(/label #{labels_list})
106
112
  end
107
113
 
108
- def up_to_date_issue_description(issue_description, test)
109
- new_issue_description =
110
- if issue_description.include?('### Reports')
111
- # We count the number of existing reports.
112
- reports_count = issue_description
113
- .scan(REPORT_ITEM_REGEX)
114
- .size.to_i + 1
115
- issue_description.sub(/^### Reports.*$/, "### Reports (#{reports_count})")
116
- else # For issue with the legacy format, we add the Reports section
117
- reports_count = issue_description
118
- .scan(JOB_URL_REGEX)
119
- .size.to_i + 1
120
-
121
- "#{issue_description}\n\n### Reports (#{reports_count})"
122
- end
123
-
124
- "#{new_issue_description}\n#{report_list_item(test)}"
114
+ def relate_issues_quick_actions(issues)
115
+ issues.map do |issue|
116
+ "/relate #{issue.web_url}"
117
+ end.join("\n")
125
118
  end
126
119
  end
127
120
  end
@@ -16,10 +16,11 @@ module GitlabQuality
16
16
  @context = context
17
17
  @existing_mrs = nil
18
18
  @file_path = spec["file_path"]
19
+ testcase = spec["testcase"]
19
20
  devops_stage = spec["stage"]
20
21
  product_group = spec["product_group"]
21
22
  @example_name = spec["name"]
22
- @mr_title = format("%{prefix} %{example_name}", prefix: '[PROMOTE TO BLOCKING]', example_name: example_name)
23
+ @mr_title = format("%{prefix} %{example_name}", prefix: '[PROMOTE TO BLOCKING]', example_name: example_name).truncate(72, omission: '')
23
24
 
24
25
  @file_contents = context.get_file_contents(file_path)
25
26
 
@@ -48,8 +49,13 @@ module GitlabQuality
48
49
 
49
50
  This test was identified in the reliable e2e test report: #{context.report_issue}
50
51
 
52
+ [Testcase link](#{testcase})
53
+
54
+ [Spec metrics link](#{context.single_spec_metrics_link(example_name)})
55
+
51
56
  /label ~"Quality" ~"QA" ~"type::maintenance"
52
57
  /label ~"devops::#{devops_stage}"
58
+ #{context.label_from_product_group(product_group)}
53
59
 
54
60
  <div align="center">
55
61
  (This MR was automatically generated by [`gitlab_quality-test_tooling`](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling) at #{Time.now.utc})
@@ -120,8 +126,8 @@ module GitlabQuality
120
126
  end
121
127
 
122
128
  context.update_matched_line(matched_lines.last, file_contents.dup) do |line|
123
- if line.include?(',')
124
- line[line.index(',')] = format(BLOCKING_METADATA, suffix: ',')
129
+ if line.sub(DESCRIPTION_REGEX, '').include?(',')
130
+ line[line.index(',', end_of_description_index(line))] = format(BLOCKING_METADATA, suffix: ',')
125
131
  else
126
132
  line[line.rindex(' ')] = format(BLOCKING_METADATA, suffix: ' ')
127
133
  end
@@ -26,11 +26,13 @@ module GitlabQuality
26
26
  @context = context
27
27
  @existing_mrs = nil
28
28
  @file_path = spec["file_path"]
29
+ testcase = spec["testcase"]
29
30
  devops_stage = spec["stage"]
31
+ product_group = spec["product_group"]
30
32
  @failure_issue_url = spec["failure_issue"]
31
33
  @example_name = spec["name"]
32
34
  @issue_id = failure_issue_url.split('/').last # split url segment, last segment of path is the issue id
33
- @mr_title = format("%{prefix} %{example_name}", prefix: '[QUARANTINE]', example_name: example_name)
35
+ @mr_title = format("%{prefix} %{example_name}", prefix: '[QUARANTINE]', example_name: example_name).truncate(72, omission: '')
34
36
  @failure_issue = context.fetch_issue(iid: issue_id)
35
37
 
36
38
  @file_contents = context.get_file_contents(file_path)
@@ -57,6 +59,10 @@ module GitlabQuality
57
59
 
58
60
  This test was identified in the reliable e2e test report: #{context.report_issue}
59
61
 
62
+ [Testcase link](#{testcase})
63
+
64
+ [Spec metrics link](#{context.single_spec_metrics_link(example_name)})
65
+
60
66
  ### E2E Test Failure issue(s)
61
67
 
62
68
  #{failure_issue_url}
@@ -81,6 +87,7 @@ module GitlabQuality
81
87
  `qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb`.
82
88
  -->
83
89
  /label ~"devops::#{devops_stage}"
90
+ #{context.label_from_product_group(product_group)}
84
91
 
85
92
  <div align="center">
86
93
  (This MR was automatically generated by [`gitlab_quality-test_tooling`](https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling) at #{Time.now.utc})
@@ -153,7 +160,7 @@ module GitlabQuality
153
160
  context.update_matched_line(matched_lines.last, file_contents.dup) do |line|
154
161
  indentation = context.indentation(line)
155
162
 
156
- if line.include?(',') && line.split.last != 'do'
163
+ if line.sub(DESCRIPTION_REGEX, '').include?(',') && line.split.last != 'do'
157
164
  line[line.rindex(',')] = format(QUARANTINE_METADATA.rstrip, issue_url: failure_issue_url, indentation: indentation, suffix: ',', quarantine_type: quarantine_type)
158
165
  else
159
166
  line[line.rindex(' ')] = format(QUARANTINE_METADATA.rstrip, issue_url: failure_issue_url, indentation: indentation, suffix: ' ', quarantine_type: quarantine_type)
@@ -170,8 +177,6 @@ module GitlabQuality
170
177
  case context.issue_scoped_label(failure_issue, 'failure')&.split('::')&.last
171
178
  when 'new', 'investigating'
172
179
  ':investigating'
173
- when 'external-dependency'
174
- ':external_dependency'
175
180
  when 'broken-test'
176
181
  ':broken'
177
182
  when 'bug'
@@ -8,6 +8,8 @@ module GitlabQuality
8
8
  module Processor
9
9
  class MetaProcessor
10
10
  class << self
11
+ DESCRIPTION_REGEX = /('.*?')|(".*?")/
12
+
11
13
  def execute
12
14
  raise 'method not implemented'
13
15
  end
@@ -24,6 +26,16 @@ module GitlabQuality
24
26
  def existing_mrs
25
27
  @existing_mrs ||= context.existing_merge_requests(title: mr_title)
26
28
  end
29
+
30
+ # Returns the index of the end of test description
31
+ #
32
+ # @param [String] line The line containing the test description
33
+ # @return [Integer]
34
+ def end_of_description_index(line)
35
+ description_length = line.match(DESCRIPTION_REGEX)[0].length
36
+ description_start_index = line.index(DESCRIPTION_REGEX)
37
+ description_start_index + description_length
38
+ end
27
39
  end
28
40
  end
29
41
  end
@@ -252,6 +252,25 @@ module GitlabQuality
252
252
  processed_records[file_path] && processed_records[file_path] == changed_line_no
253
253
  end
254
254
 
255
+ # Infers product group label from the provided product group
256
+ #
257
+ # @param [String] product_group product group
258
+ # @return [String]
259
+ def label_from_product_group(product_group)
260
+ label = labels_inference.infer_labels_from_product_group(product_group).to_a.first
261
+
262
+ label ? %(/label ~"#{label}") : ''
263
+ end
264
+
265
+ # Returns the link to the Grafana dashboard for single spec metrics
266
+ #
267
+ # @param [String] example_name the full example name
268
+ # @return [String]
269
+ def single_spec_metrics_link(example_name)
270
+ base_url = "https://dashboards.quality.gitlab.net/d/cW0UMgv7k/single-spec-metrics?orgId=1&var-run_type=All&var-name="
271
+ base_url + CGI.escape(example_name)
272
+ end
273
+
255
274
  private
256
275
 
257
276
  attr_reader :token, :specs_file, :dry_run, :processor
@@ -281,6 +300,13 @@ module GitlabQuality
281
300
  project: project
282
301
  )
283
302
  end
303
+
304
+ # Returns a cached instance of GitlabQuality::TestTooling::LabelsInference
305
+ #
306
+ # @return [GitlabQuality::TestTooling::LabelsInference]
307
+ def labels_inference
308
+ @labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
309
+ end
284
310
  end
285
311
  end
286
312
  end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module GitlabQuality
4
4
  module TestTooling
5
- VERSION = "1.15.0"
5
+ VERSION = "1.17.0"
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab_quality-test_tooling
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.15.0
4
+ version: 1.17.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - GitLab Quality
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-02-08 00:00:00.000000000 Z
11
+ date: 2024-03-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: climate_control