gitlab_quality-test_tooling 1.22.0 → 1.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 755ed1e2f1e0a7ac2e1d1264eeed686c76fb23c90d5eb6ebfbcaa64f931045f6
4
- data.tar.gz: 5bbc1d0b7b93d8db268e2b0b0081b3878ff4117e97b2a3426023180cdec945ec
3
+ metadata.gz: '009e41374e42e637a571410da7cd95aa23ab4d5a6f1599f080a0ccea8432b820'
4
+ data.tar.gz: b81d4043e16bc5da888a102d3765250691acb705bb3edac3762c6baad0fe490f
5
5
  SHA512:
6
- metadata.gz: 3078229a51d390624b6e148444735f38581383319e377db26986d7d1b77d2b53447b440d1fc3eecb8eaf52e92a3cc0c927e4ab44930e34bfb83165e88b011097
7
- data.tar.gz: a0ac07071de3a7fe50144f2527faff4a6e5c5530477681b5ebf1137fc1d89aa8991f2aa88ef255bd0ffa84c3a3d8440ab91e3b3b7ef4c28fed9ae3b11c7709e0
6
+ metadata.gz: f1d34b6b4862000282c042e25b5f0cd02e23e72dee4307d0eb9d2309a0e5d352dfeeeadab2f5da7d01b91fe798a192bf8f2ed075608e9f73854842d107bef55a
7
+ data.tar.gz: eb3746a140cb622b9ede27514cb26ea7681b1bafffc5320dd9ce6dce83951db2e752f5458fa7af09906488fd6ef871c589c232794a4bf3d8a6962613cc35e484
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab_quality-test_tooling (1.22.0)
4
+ gitlab_quality-test_tooling (1.24.0)
5
5
  activesupport (>= 6.1, < 7.2)
6
6
  amatch (~> 0.4.1)
7
7
  gitlab (~> 4.19)
@@ -130,6 +130,7 @@ GEM
130
130
  multi_xml (>= 0.5.2)
131
131
  i18n (1.14.4)
132
132
  concurrent-ruby (~> 1.0)
133
+ influxdb-client (3.1.0)
133
134
  jaro_winkler (1.5.6)
134
135
  json (2.7.1)
135
136
  kramdown (2.4.0)
@@ -312,11 +313,13 @@ PLATFORMS
312
313
  ruby
313
314
 
314
315
  DEPENDENCIES
316
+ activesupport (>= 6.1, < 7.2)
315
317
  climate_control (~> 1.2)
316
318
  gitlab-dangerfiles (~> 3.8)
317
319
  gitlab-styles (~> 10.0)
318
320
  gitlab_quality-test_tooling!
319
321
  guard-rspec (~> 4.7)
322
+ influxdb-client (~> 3.1)
320
323
  lefthook (~> 1.3)
321
324
  pry-byebug (= 3.10.1)
322
325
  rake (~> 13.0)
@@ -328,4 +331,4 @@ DEPENDENCIES
328
331
  webmock (= 3.7.0)
329
332
 
330
333
  BUNDLED WITH
331
- 2.5.4
334
+ 2.5.6
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module GitlabQuality
4
+ module TestTooling
5
+ module Concerns
6
+ module InfluxdbTools
7
+ # InfluxDb client
8
+ #
9
+ # @return [InfluxDB2::Client]
10
+ def influx_client(url:, token:, bucket:)
11
+ @influx_client ||= InfluxDB2::Client.new(
12
+ url || raise('Missing influxdb_url'),
13
+ token || raise('Missing influxdb_token'),
14
+ bucket: bucket || raise('Missing influxdb_bucket'),
15
+ org: "gitlab-qa",
16
+ precision: InfluxDB2::WritePrecision::NANOSECOND
17
+ )
18
+ end
19
+
20
+ # Write client
21
+ #
22
+ # @return [WriteApi]
23
+ def write_api(url:, token:, bucket:)
24
+ @write_api ||= influx_client(url: url, token: token, bucket: bucket).create_write_api
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,145 @@
1
+ # frozen_string_literal: true
2
+
3
+ module GitlabQuality
4
+ module TestTooling
5
+ module Concerns
6
+ module TestMetrics
7
+ # Single common timestamp for all exported example metrics to keep data points consistently grouped
8
+ #
9
+ # @return [Time]
10
+ def time
11
+ return @time if defined?(@time)
12
+
13
+ created_at = Time.strptime(env('CI_PIPELINE_CREATED_AT'), '%Y-%m-%dT%H:%M:%S%z') if env('CI_PIPELINE_CREATED_AT')
14
+ @time = (created_at || Time.now).utc.strftime('%Y-%m-%dT%H:%M:%S%z')
15
+ end
16
+
17
+ # rubocop:disable Metrics/AbcSize
18
+ # Metrics tags
19
+ #
20
+ # @param [RSpec::Core::Example] example
21
+ # @param [Array<String>] custom_keys
22
+ # @param [String]
23
+ # @return [Hash]
24
+ def tags(example, custom_keys, run_type)
25
+ {
26
+ name: example.full_description,
27
+ file_path: example.metadata[:file_path].sub(/\A./, ''),
28
+ status: status(example),
29
+ quarantined: quarantined(example),
30
+ job_name: job_name,
31
+ merge_request: merge_request,
32
+ run_type: run_type,
33
+ stage: example.metadata[:product_stage] || example.metadata[:product_category],
34
+ product_group: example.metadata[:product_group],
35
+ exception_class: example.execution_result.exception&.class&.to_s,
36
+ **custom_metrics(example.metadata, custom_keys)
37
+ }.compact
38
+ end
39
+ # rubocop:enable Metrics/AbcSize
40
+
41
+ # Metrics fields
42
+ #
43
+ # @param [RSpec::Core::Example] example
44
+ # @param [Array<String>] custom_keys
45
+ # @return [Hash]
46
+ def fields(example, custom_keys)
47
+ {
48
+ id: example.id,
49
+ run_time: (example.execution_result.run_time * 1000).round,
50
+ job_url: Runtime::Env.ci_job_url,
51
+ pipeline_url: env('CI_PIPELINE_URL'),
52
+ pipeline_id: env('CI_PIPELINE_ID'),
53
+ job_id: env('CI_JOB_ID'),
54
+ merge_request_iid: merge_request_iid,
55
+ failure_exception: example.execution_result.exception.to_s.delete("\n"),
56
+ **custom_metrics(example.metadata, custom_keys)
57
+ }.compact
58
+ end
59
+
60
+ # Return a more detailed status
61
+ #
62
+ # - if test is failed or pending, return rspec status
63
+ # - if test passed but had more than 1 attempt, consider test flaky
64
+ #
65
+ # @param [RSpec::Core::Example] example
66
+ # @return [Symbol]
67
+ def status(example)
68
+ rspec_status = example.execution_result.status
69
+ return rspec_status if [:pending, :failed].include?(rspec_status)
70
+
71
+ retry_attempts(example.metadata).positive? ? :flaky : :passed
72
+ end
73
+
74
+ # Retry attempts
75
+ #
76
+ # @param [Hash] example
77
+ # @return [Integer]
78
+ def retry_attempts(metadata)
79
+ metadata[:retry_attempts] || 0
80
+ end
81
+
82
+ # Checks if spec is quarantined
83
+ #
84
+ # @param [RSpec::Core::Example] example
85
+ # @return [String]
86
+ def quarantined(example)
87
+ return "false" unless example.metadata.key?(:quarantine)
88
+
89
+ # if quarantine key is present and status is pending, consider it quarantined
90
+ (example.execution_result.status == :pending).to_s
91
+ end
92
+
93
+ # Base ci job name
94
+ #
95
+ # @return [String]
96
+ def job_name
97
+ @job_name ||= Runtime::Env.ci_job_name&.gsub(%r{ \d{1,2}/\d{1,2}}, '')
98
+ end
99
+
100
+ # Check if it is a merge request execution
101
+ #
102
+ # @return [String]
103
+ def merge_request
104
+ (!!merge_request_iid).to_s
105
+ end
106
+
107
+ # Merge request iid
108
+ #
109
+ # @return [String]
110
+ def merge_request_iid
111
+ env('CI_MERGE_REQUEST_IID') || env('TOP_UPSTREAM_MERGE_REQUEST_IID')
112
+ end
113
+
114
+ # Custom test metrics
115
+ #
116
+ # @param [Hash] metadata
117
+ # @param [Array] array of custom metrics keys
118
+ # @return [Hash]
119
+ def custom_metrics(metadata, custom_keys)
120
+ return {} if custom_keys.nil?
121
+
122
+ custom_metrics = {}
123
+ custom_keys.each do |k|
124
+ value = metadata[k.to_sym]
125
+ v = value.is_a?(Numeric) || value.nil? ? value : value.to_s
126
+
127
+ custom_metrics[k.to_sym] = v
128
+ end
129
+
130
+ custom_metrics
131
+ end
132
+
133
+ # Return non empty environment variable value
134
+ #
135
+ # @param [String] name
136
+ # @return [String, nil]
137
+ def env(name)
138
+ return unless ENV[name] && !ENV[name].empty?
139
+
140
+ ENV.fetch(name)
141
+ end
142
+ end
143
+ end
144
+ end
145
+ end
@@ -12,11 +12,57 @@ module GitlabQuality
12
12
  REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>\S+)\)/
13
13
  LATEST_REPORTS_TO_SHOW = 10
14
14
 
15
+ class ReportsList
16
+ def initialize(preserved_content:, section_header:, reports:, extra_content:)
17
+ @preserved_content = preserved_content
18
+ @section_header = section_header
19
+ @reports = reports
20
+ @extra_content = extra_content
21
+ end
22
+
23
+ def self.report_list_item(test, item_extra_content: nil)
24
+ "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')}) #{item_extra_content}".strip
25
+ end
26
+
27
+ def reports_count
28
+ reports.size
29
+ end
30
+
31
+ def to_s
32
+ [
33
+ preserved_content,
34
+ "#{section_header} (#{reports_count})",
35
+ reports_list(reports),
36
+ extra_content
37
+ ].reject(&:blank?).compact.join("\n\n")
38
+ end
39
+
40
+ private
41
+
42
+ attr_reader :preserved_content, :section_header, :reports, :extra_content
43
+
44
+ def reports_list(reports)
45
+ sorted_reports = reports.sort.reverse
46
+
47
+ if sorted_reports.size > LATEST_REPORTS_TO_SHOW
48
+ [
49
+ "Last 10 reports:",
50
+ sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
51
+ "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
52
+ sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
53
+ "</details>"
54
+ ].join("\n\n")
55
+ else
56
+ sorted_reports.join("\n")
57
+ end
58
+ end
59
+ end
60
+
15
61
  def initial_reports_section(test)
16
62
  <<~REPORTS
17
63
  ### Reports (1)
18
64
 
19
- #{report_list_item(test)}
65
+ #{ReportsList.report_list_item(test)}
20
66
  REPORTS
21
67
  end
22
68
 
@@ -27,14 +73,14 @@ module GitlabQuality
27
73
  item_extra_content: nil,
28
74
  reports_extra_content: nil)
29
75
  preserved_content = current_reports_content.split(reports_section_header).first&.strip
30
- reports = report_lines(current_reports_content) + [report_list_item(test, item_extra_content: item_extra_content)]
31
-
32
- [
33
- preserved_content,
34
- "#{reports_section_header} (#{reports.size})",
35
- reports_list(reports),
36
- reports_extra_content
37
- ].reject(&:blank?).compact.join("\n\n")
76
+ reports = report_lines(current_reports_content) + [ReportsList.report_list_item(test, item_extra_content: item_extra_content)]
77
+
78
+ ReportsList.new(
79
+ preserved_content: preserved_content,
80
+ section_header: reports_section_header,
81
+ reports: reports,
82
+ extra_content: reports_extra_content
83
+ )
38
84
  end
39
85
 
40
86
  def failed_issue_job_url(issue)
@@ -55,40 +101,12 @@ module GitlabQuality
55
101
  content.lines.grep(REPORT_ITEM_REGEX).map(&:strip)
56
102
  end
57
103
 
58
- def reports_list(reports)
59
- sorted_reports = reports.sort.reverse
60
-
61
- if sorted_reports.size > LATEST_REPORTS_TO_SHOW
62
- [
63
- "Last 10 reports:",
64
- sorted_reports[...LATEST_REPORTS_TO_SHOW].join("\n"),
65
- "<details><summary>See #{sorted_reports.size - LATEST_REPORTS_TO_SHOW} more reports</summary>",
66
- sorted_reports[LATEST_REPORTS_TO_SHOW..].join("\n"),
67
- "</details>"
68
- ].join("\n\n")
69
- else
70
- sorted_reports.join("\n")
71
- end
72
- end
73
-
74
- def report_list_item(test, item_extra_content: nil)
75
- "1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')}) #{item_extra_content}".strip
76
- end
77
-
78
104
  def job_urls_from_description(issue_description, regex)
79
105
  issue_description.lines.filter_map do |line|
80
106
  match = line.match(regex)
81
107
  match[:job_url] if match
82
108
  end
83
109
  end
84
-
85
- def test_captures_to_report_items(test_captures)
86
- test_captures.map do |ci_job_url, _, _|
87
- report_list_item(GitlabQuality::TestTooling::TestResult::JsonTestResult.new(
88
- 'ci_job_url' => ci_job_url
89
- ))
90
- end
91
- end
92
110
  end
93
111
  end
94
112
  end
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'amatch'
4
-
5
3
  module GitlabQuality
6
4
  module TestTooling
7
5
  module Report
@@ -15,7 +13,6 @@ module GitlabQuality
15
13
  class FailedTestIssue < ReportAsIssue
16
14
  include Concerns::GroupAndCategoryLabels
17
15
  include Concerns::IssueReports
18
- include Amatch
19
16
 
20
17
  IDENTITY_LABELS = ['test', 'automation:bot-authored'].freeze
21
18
  NEW_ISSUE_LABELS = Set.new(['type::maintenance', 'failure::new', 'priority::3', 'severity::3', *IDENTITY_LABELS]).freeze
@@ -25,10 +22,6 @@ module GitlabQuality
25
22
  REPORTS_DISCUSSION_HEADER = '### Failure reports'
26
23
  REPORT_SECTION_HEADER = '#### Failure reports'
27
24
 
28
- IGNORED_FAILURES = [
29
- 'Net::ReadTimeout',
30
- '403 Forbidden - Your account has been blocked'
31
- ].freeze
32
25
  FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
33
26
  ISSUE_STACKTRACE_REGEX = /##### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*/m
34
27
  DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
@@ -92,20 +85,21 @@ module GitlabQuality
92
85
  end
93
86
  end
94
87
 
95
- def add_report_to_issue(issue:, test:, related_issues:)
88
+ def add_report_to_issue(issue:, test:, related_issues:) # rubocop:disable Metrics/AbcSize:
96
89
  reports_discussion = find_or_create_reports_discussion(issue: issue)
97
- failure_discussion_note = find_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion)
90
+ current_reports_note = find_failure_discussion_note(issue: issue, test: test, reports_discussion: reports_discussion)
91
+ new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test)
98
92
 
99
93
  note_body = [
100
- report_body(reports_note: failure_discussion_note, test: test),
94
+ new_reports_list.to_s,
101
95
  identity_labels_quick_action,
102
96
  relate_issues_quick_actions(related_issues)
103
97
  ].join("\n")
104
98
 
105
- if failure_discussion_note
99
+ if current_reports_note
106
100
  gitlab.edit_issue_note(
107
101
  issue_iid: issue.iid,
108
- note_id: failure_discussion_note.id,
102
+ note_id: current_reports_note.id,
109
103
  note: note_body
110
104
  )
111
105
  else
@@ -158,6 +152,8 @@ module GitlabQuality
158
152
 
159
153
  clean_test_stacktrace = cleaned_stack_trace_from_test(test: test)
160
154
 
155
+ # We're skipping the first note of the discussion as this is the "non-collapsible note", aka
156
+ # the "header note", which doesn't contain any stack trace.
161
157
  reports_discussion.notes[1..].each_with_object({}) do |note, memo|
162
158
  clean_note_stacktrace = cleaned_stack_trace_from_note(issue: issue, note: note)
163
159
  diff_ratio = diff_ratio_between_test_and_note_stacktraces(
@@ -171,7 +167,7 @@ module GitlabQuality
171
167
  end
172
168
 
173
169
  def cleaned_stack_trace_from_test(test:)
174
- sanitize_stacktrace(stacktrace: full_stacktrace(test: test), regex: FAILURE_STACKTRACE_REGEX) || full_stacktrace(test: test)
170
+ sanitize_stacktrace(stacktrace: test.full_stacktrace, regex: FAILURE_STACKTRACE_REGEX) || test.full_stacktrace
175
171
  end
176
172
 
177
173
  def cleaned_stack_trace_from_note(issue:, note:)
@@ -191,53 +187,33 @@ module GitlabQuality
191
187
  end
192
188
  end
193
189
 
194
- def full_stacktrace(test:)
195
- test.failures.each do |failure|
196
- message = failure['message'] || ""
197
- message_lines = failure['message_lines'] || []
198
-
199
- next if IGNORED_FAILURES.any? { |e| message.include?(e) }
200
-
201
- return message_lines.empty? ? message : message_lines.join("\n")
202
- end
203
- end
204
-
205
190
  def diff_ratio_between_test_and_note_stacktraces(issue:, note:, test_stacktrace:, note_stacktrace:)
206
191
  return if note_stacktrace.nil?
207
192
 
208
- diff_ratio = compare_stack_traces(test_stacktrace, note_stacktrace)
193
+ stack_trace_comparator = StackTraceComparator.new(test_stacktrace, note_stacktrace)
209
194
 
210
- if diff_ratio <= max_diff_ratio
211
- puts " => [DEBUG] Note #{issue.web_url}#note_#{note.id} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
195
+ if stack_trace_comparator.lower_or_equal_to_diff_ratio?(max_diff_ratio)
196
+ puts " => [DEBUG] Note #{issue.web_url}#note_#{note.id} has an acceptable diff ratio of #{stack_trace_comparator.diff_percent}%."
212
197
  # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
213
198
  # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
214
199
  # See:
215
200
  # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
216
201
  # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
217
- diff_ratio
202
+ stack_trace_comparator.diff_ratio
218
203
  else
219
- puts " => [DEBUG] Found note #{issue.web_url}#note_#{note.id} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
204
+ puts " => [DEBUG] Found note #{issue.web_url}#note_#{note.id} but stacktraces are too different (#{stack_trace_comparator.diff_percent}%).\n"
220
205
  puts " => [DEBUG] Issue stacktrace:\n----------------\n#{note_stacktrace}\n----------------\n"
221
206
  puts " => [DEBUG] Failure stacktrace:\n----------------\n#{test_stacktrace}\n----------------\n"
222
207
  end
223
208
  end
224
209
 
225
- def compare_stack_traces(stack_trace_first, stack_trace_second)
226
- calculate_diff_ratio(stack_trace_first, stack_trace_second)
227
- end
228
-
229
- def calculate_diff_ratio(stack_trace_first, stack_trace_second)
230
- distance = Levenshtein.new(stack_trace_first).match(stack_trace_second)
231
- distance.zero? ? 0.0 : (distance.to_f / stack_trace_first.size).round(3)
232
- end
233
-
234
- def report_body(reports_note:, test:)
210
+ def add_report_for_test(current_reports_content:, test:)
235
211
  increment_reports(
236
- current_reports_content: reports_note&.body.to_s,
212
+ current_reports_content: current_reports_content,
237
213
  test: test,
238
214
  reports_section_header: REPORT_SECTION_HEADER,
239
215
  item_extra_content: found_label,
240
- reports_extra_content: "##### Stack trace\n\n```\n#{full_stacktrace(test: test)}\n```"
216
+ reports_extra_content: "##### Stack trace\n\n```\n#{test.full_stacktrace}\n```"
241
217
  )
242
218
  end
243
219
 
@@ -22,7 +22,7 @@ module GitlabQuality
22
22
  FOUND_IN_MASTER_LABEL = '~"found:master"'
23
23
  REPORT_SECTION_HEADER = '### Flakiness reports'
24
24
  REPORTS_DOCUMENTATION = <<~DOC
25
- Flaky tests were detected, please refer to the [Flaky tests documentation](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html)
25
+ Flaky tests were detected. Please refer to the [Flaky tests reproducibility instructions](https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#how-to-reproduce-a-flaky-test-locally)
26
26
  to learn more about how to reproduce them.
27
27
  DOC
28
28
 
@@ -83,17 +83,20 @@ module GitlabQuality
83
83
  end
84
84
 
85
85
  def add_report_to_issue(issue:, test:, related_issues:)
86
- reports_note = existing_reports_note(issue: issue)
86
+ current_reports_note = existing_reports_note(issue: issue)
87
+ new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test)
88
+
87
89
  note_body = [
88
- report_body(reports_note: reports_note, test: test),
90
+ new_reports_list.to_s,
91
+ flakiness_status_labels_quick_action(new_reports_list.reports_count),
89
92
  identity_labels_quick_action,
90
93
  relate_issues_quick_actions(related_issues)
91
94
  ].join("\n")
92
95
 
93
- if reports_note
96
+ if current_reports_note
94
97
  gitlab.edit_issue_note(
95
98
  issue_iid: issue.iid,
96
- note_id: reports_note.id,
99
+ note_id: current_reports_note.id,
97
100
  note: note_body
98
101
  )
99
102
  else
@@ -107,9 +110,9 @@ module GitlabQuality
107
110
  end
108
111
  end
109
112
 
110
- def report_body(reports_note:, test:)
113
+ def add_report_for_test(current_reports_content:, test:)
111
114
  increment_reports(
112
- current_reports_content: reports_note&.body.to_s,
115
+ current_reports_content: current_reports_content,
113
116
  test: test,
114
117
  reports_section_header: REPORT_SECTION_HEADER,
115
118
  item_extra_content: found_label,
@@ -125,6 +128,28 @@ module GitlabQuality
125
128
  end
126
129
  end
127
130
 
131
+ # The report count is based on the percentiles of flakiness issues.
132
+ #
133
+ # See https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/scripts/unhealthy_test_issues_statistics.rb
134
+ # to gather these statistics.
135
+ #
136
+ # P75 => flakiness::4
137
+ # P90 => flakiness::3
138
+ # P95 => flakiness::2
139
+ # Above P95 => flakiness::1
140
+ def flakiness_status_labels_quick_action(reports_count)
141
+ case reports_count
142
+ when 42..Float::INFINITY
143
+ '/label ~"flakiness::1"'
144
+ when 22..41
145
+ '/label ~"flakiness::2"'
146
+ when 4..21
147
+ '/label ~"flakiness::3"'
148
+ else
149
+ '/label ~"flakiness::4"'
150
+ end
151
+ end
152
+
128
153
  def identity_labels_quick_action
129
154
  labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
130
155
  %(/label #{labels_list})
@@ -2,7 +2,6 @@
2
2
 
3
3
  require 'nokogiri'
4
4
  require 'rubygems/text'
5
- require 'amatch'
6
5
 
7
6
  module GitlabQuality
8
7
  module TestTooling
@@ -17,7 +16,6 @@ module GitlabQuality
17
16
  include TestTooling::Concerns::FindSetDri
18
17
  include Concerns::GroupAndCategoryLabels
19
18
  include Concerns::IssueReports
20
- include Amatch
21
19
 
22
20
  DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
23
21
  SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
@@ -26,10 +24,6 @@ module GitlabQuality
26
24
  ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*\n###/m
27
25
 
28
26
  NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
29
- IGNORED_FAILURES = [
30
- 'Net::ReadTimeout',
31
- '403 Forbidden - Your account has been blocked'
32
- ].freeze
33
27
  SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
34
28
 
35
29
  MultipleIssuesFound = Class.new(StandardError)
@@ -168,7 +162,9 @@ module GitlabQuality
168
162
 
169
163
  next if pipeline != pipeline_env_from_job_url(job_url_from_issue)
170
164
 
171
- compare_stack_traces(cleaned_stack_trace_from_test(test), cleaned_stack_trace_from_issue(issue)) < max_diff_ratio
165
+ stack_trace_comparator = StackTraceComparator.new(cleaned_stack_trace_from_test(test), cleaned_stack_trace_from_issue(issue))
166
+
167
+ stack_trace_comparator.lower_than_diff_ratio?(max_diff_ratio)
172
168
  end
173
169
  end
174
170
 
@@ -196,17 +192,6 @@ module GitlabQuality
196
192
  )
197
193
  end
198
194
 
199
- def full_stacktrace(test)
200
- test.failures.each do |failure|
201
- message = failure['message'] || ""
202
- message_lines = failure['message_lines'] || []
203
-
204
- next if IGNORED_FAILURES.any? { |e| message.include?(e) }
205
-
206
- return message_lines.empty? ? message : message_lines.join("\n")
207
- end
208
- end
209
-
210
195
  def cleaned_stack_trace_from_issue(issue)
211
196
  relevant_issue_stacktrace = find_issue_stacktrace(issue)
212
197
  return unless relevant_issue_stacktrace
@@ -215,20 +200,11 @@ module GitlabQuality
215
200
  end
216
201
 
217
202
  def cleaned_stack_trace_from_test(test)
218
- test_failure_stacktrace = sanitize_stacktrace(full_stacktrace(test),
219
- FAILURE_STACKTRACE_REGEX) || full_stacktrace(test)
203
+ test_failure_stacktrace = sanitize_stacktrace(test.full_stacktrace,
204
+ FAILURE_STACKTRACE_REGEX) || test.full_stacktrace
220
205
  remove_unique_resource_names(test_failure_stacktrace)
221
206
  end
222
207
 
223
- def compare_stack_traces(stack_trace_first, stack_trace_second)
224
- calculate_diff_ratio(stack_trace_first, stack_trace_second)
225
- end
226
-
227
- def calculate_diff_ratio(stack_trace_first, stack_trace_second)
228
- distance = Levenshtein.new(stack_trace_first).match(stack_trace_second)
229
- distance.zero? ? 0.0 : (distance.to_f / stack_trace_first.size).round(3)
230
- end
231
-
232
208
  def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
233
209
  clean_first_test_failure_stacktrace = cleaned_stack_trace_from_test(test)
234
210
  # Search with the `search` param returns 500 errors, so we filter by `base_issue_labels` and then filter further in Ruby
@@ -236,17 +212,18 @@ module GitlabQuality
236
212
  clean_relevant_issue_stacktrace = cleaned_stack_trace_from_issue(issue)
237
213
  next if clean_relevant_issue_stacktrace.nil?
238
214
 
239
- diff_ratio = compare_stack_traces(clean_first_test_failure_stacktrace, clean_relevant_issue_stacktrace)
240
- if diff_ratio <= max_diff_ratio
241
- puts " => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
215
+ stack_trace_comparator = StackTraceComparator.new(clean_first_test_failure_stacktrace, clean_relevant_issue_stacktrace)
216
+
217
+ if stack_trace_comparator.lower_or_equal_to_diff_ratio?(max_diff_ratio)
218
+ puts " => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{stack_trace_comparator.diff_percent}%."
242
219
  # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
243
220
  # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
244
221
  # See:
245
222
  # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
246
223
  # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
247
- memo[issue.to_h] = diff_ratio
224
+ memo[issue.to_h] = stack_trace_comparator.diff_ratio
248
225
  else
249
- puts " => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
226
+ puts " => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{stack_trace_comparator.diff_percent}%).\n"
250
227
  puts " => [DEBUG] Issue stacktrace:\n----------------\n#{clean_relevant_issue_stacktrace}\n----------------\n"
251
228
  puts " => [DEBUG] Failure stacktrace:\n----------------\n#{clean_first_test_failure_stacktrace}\n----------------\n"
252
229
  end
@@ -295,7 +272,7 @@ module GitlabQuality
295
272
  def new_issue_description(test)
296
273
  super + [
297
274
  "\n### Stack trace",
298
- "```\n#{full_stacktrace(test)}\n```",
275
+ "```\n#{test.full_stacktrace}\n```",
299
276
  screenshot_section(test),
300
277
  system_log_errors_section(test),
301
278
  initial_reports_section(test)
@@ -345,7 +322,7 @@ module GitlabQuality
345
322
  state_event = issue.state == 'closed' ? 'reopen' : nil
346
323
 
347
324
  issue_attrs = {
348
- description: increment_reports(current_reports_content: issue.description, test: test),
325
+ description: increment_reports(current_reports_content: issue.description, test: test).to_s,
349
326
  labels: up_to_date_labels(test: test, issue: issue)
350
327
  }
351
328
  issue_attrs[:state_event] = state_event if state_event
@@ -357,7 +334,7 @@ module GitlabQuality
357
334
  def screenshot_section(test)
358
335
  return unless test.screenshot?
359
336
 
360
- failure = full_stacktrace(test)
337
+ failure = test.full_stacktrace
361
338
  return if SCREENSHOT_IGNORED_ERRORS.any? { |e| failure.include?(e) }
362
339
 
363
340
  relative_url = gitlab.upload_file(file_fullpath: test.screenshot_image)
@@ -374,7 +351,7 @@ module GitlabQuality
374
351
  return false unless test.failures?
375
352
 
376
353
  puts " => Systemic failures detected: #{systemic_failure_messages}" if systemic_failure_messages.any?
377
- failure_to_ignore = IGNORED_FAILURES + systemic_failure_messages
354
+ failure_to_ignore = TestResult::BaseTestResult::IGNORED_FAILURES + systemic_failure_messages
378
355
 
379
356
  reason = ignored_failure_reason(test.failures, failure_to_ignore)
380
357
 
@@ -64,17 +64,20 @@ module GitlabQuality
64
64
  end
65
65
 
66
66
  def add_report_to_issue(issue:, test:, related_issues:)
67
- reports_note = existing_reports_note(issue: issue)
67
+ current_reports_note = existing_reports_note(issue: issue)
68
+ new_reports_list = add_report_for_test(current_reports_content: current_reports_note&.body.to_s, test: test)
69
+
68
70
  note_body = [
69
- report_body(reports_note: reports_note, test: test),
71
+ new_reports_list.to_s,
72
+ slowness_status_labels_quick_action(new_reports_list.reports_count),
70
73
  identity_labels_quick_action,
71
74
  relate_issues_quick_actions(related_issues)
72
75
  ].join("\n")
73
76
 
74
- if reports_note
77
+ if current_reports_note
75
78
  gitlab.edit_issue_note(
76
79
  issue_iid: issue.iid,
77
- note_id: reports_note.id,
80
+ note_id: current_reports_note.id,
78
81
  note: note_body
79
82
  )
80
83
  else
@@ -88,9 +91,9 @@ module GitlabQuality
88
91
  end
89
92
  end
90
93
 
91
- def report_body(reports_note:, test:)
94
+ def add_report_for_test(current_reports_content:, test:)
92
95
  increment_reports(
93
- current_reports_content: reports_note&.body.to_s,
96
+ current_reports_content: current_reports_content,
94
97
  test: test,
95
98
  reports_section_header: REPORT_SECTION_HEADER,
96
99
  item_extra_content: "(#{test.run_time} seconds) #{found_label}",
@@ -106,6 +109,28 @@ module GitlabQuality
106
109
  end
107
110
  end
108
111
 
112
+ # The report count is based on the percentiles of slowness issues.
113
+ #
114
+ # See https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/scripts/unhealthy_test_issues_statistics.rb
115
+ # to gather these statistics.
116
+ #
117
+ # P75 => slowness::4
118
+ # P90 => slowness::3
119
+ # P95 => slowness::2
120
+ # Above P95 => slowness::1
121
+ def slowness_status_labels_quick_action(reports_count)
122
+ case reports_count
123
+ when 40..Float::INFINITY
124
+ '/label ~"slowness::1"'
125
+ when 28..39
126
+ '/label ~"slowness::2"'
127
+ when 12..27
128
+ '/label ~"slowness::3"'
129
+ else
130
+ '/label ~"slowness::4"'
131
+ end
132
+ end
133
+
109
134
  def identity_labels_quick_action
110
135
  labels_list = IDENTITY_LABELS.map { |label| %(~"#{label}") }.join(' ')
111
136
  %(/label #{labels_list})
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'amatch'
4
+
5
+ module GitlabQuality
6
+ module TestTooling
7
+ class StackTraceComparator
8
+ include Amatch
9
+
10
+ def initialize(first_trace, second_trace)
11
+ @first_trace = first_trace
12
+ @second_trace = second_trace
13
+ end
14
+
15
+ def diff_ratio
16
+ @diff_ratio ||= (1 - first_trace.levenshtein_similar(second_trace))
17
+ end
18
+
19
+ def diff_percent
20
+ (diff_ratio * 100).round(2)
21
+ end
22
+
23
+ def lower_than_diff_ratio?(max_diff_ratio)
24
+ diff_ratio < max_diff_ratio
25
+ end
26
+
27
+ def lower_or_equal_to_diff_ratio?(max_diff_ratio)
28
+ diff_ratio <= max_diff_ratio
29
+ end
30
+
31
+ private
32
+
33
+ attr_reader :first_trace, :second_trace
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module GitlabQuality
6
+ module TestTooling
7
+ module TestMetric
8
+ class LogTestMetrics
9
+ include Concerns::TestMetrics
10
+ include Concerns::InfluxdbTools
11
+
12
+ CUSTOM_METRICS_KEY = :custom_test_metrics
13
+
14
+ def initialize(examples:, influxdb_url: nil, influxdb_token: nil, influxdb_bucket: nil, run_type: nil)
15
+ @examples = examples
16
+ @influxdb_url = influxdb_url
17
+ @influxdb_token = influxdb_token
18
+ @influxdb_bucket = influxdb_bucket
19
+ @run_type = run_type
20
+ end
21
+
22
+ # Push test execution metrics to influxdb
23
+ #
24
+ # @param [Array<String>] custom_keys_tags
25
+ # @param [Array<String>] custom_keys_fields
26
+ # @return [nil]
27
+ def push_test_metrics(custom_keys_tags: nil, custom_keys_fields: nil)
28
+ @test_metrics ||= examples.filter_map { |example| parse_test_results(example, custom_keys_tags, custom_keys_fields) }
29
+
30
+ write_api(url: influxdb_url, token: influxdb_token, bucket: influxdb_bucket).write(data: test_metrics)
31
+ Runtime::Logger.debug("Pushed #{test_metrics.length} test execution entries to influxdb")
32
+ rescue StandardError => e
33
+ Runtime::Logger.error("Failed to push test execution metrics to influxdb, error: #{e}")
34
+ end
35
+
36
+ # Save metrics in json file
37
+ #
38
+ # @param [String] file_name
39
+ # @param [Array<String>] custom_keys_tags
40
+ # @param [Array<String>] custom_keys_fields
41
+ # @return [nil]
42
+ def save_test_metrics(file_name, custom_keys_tags: nil, custom_keys_fields: nil)
43
+ @test_metrics ||= examples.filter_map { |example| parse_test_results(example, custom_keys_tags, custom_keys_fields) }
44
+ file = "tmp/#{file_name}"
45
+
46
+ File.write(file, test_metrics.to_json) && Runtime::Logger.debug("Saved test metrics to #{file}")
47
+ rescue StandardError => e
48
+ Runtime::Logger.error("Failed to save test execution metrics, error: #{e}")
49
+ end
50
+
51
+ private
52
+
53
+ attr_reader :examples, :test_metrics, :influxdb_url, :influxdb_token, :influxdb_bucket, :run_type
54
+
55
+ # Transform example to influxdb compatible metrics data
56
+ # https://github.com/influxdata/influxdb-client-ruby#data-format
57
+ #
58
+ # @param [RSpec::Core::Example] example
59
+ # @param [Array<String>] custom_keys_tags
60
+ # @param [Array<String>] custom_keys_fields
61
+ # @return [Hash]
62
+ def parse_test_results(example, custom_keys_tags, custom_keys_fields)
63
+ {
64
+ name: 'test-stats',
65
+ time: time,
66
+ tags: tags(example, custom_keys_tags, run_type),
67
+ fields: fields(example, custom_keys_fields)
68
+ }
69
+ rescue StandardError => e
70
+ Runtime::Logger.error("Failed to transform example '#{example.id}', error: #{e}")
71
+ nil
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
@@ -4,6 +4,11 @@ module GitlabQuality
4
4
  module TestTooling
5
5
  module TestResult
6
6
  class BaseTestResult
7
+ IGNORED_FAILURES = [
8
+ 'Net::ReadTimeout',
9
+ '403 Forbidden - Your account has been blocked'
10
+ ].freeze
11
+
7
12
  attr_reader :report
8
13
 
9
14
  def initialize(report)
@@ -41,6 +46,17 @@ module GitlabQuality
41
46
  def failures?
42
47
  failures.any?
43
48
  end
49
+
50
+ def full_stacktrace
51
+ failures.each do |failure|
52
+ message = failure['message'] || ""
53
+ message_lines = failure['message_lines'] || []
54
+
55
+ next if IGNORED_FAILURES.any? { |e| message.include?(e) }
56
+
57
+ return message_lines.empty? ? message : message_lines.join("\n")
58
+ end
59
+ end
44
60
  end
45
61
  end
46
62
  end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module GitlabQuality
4
4
  module TestTooling
5
- VERSION = "1.22.0"
5
+ VERSION = "1.24.0"
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,15 +1,35 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab_quality-test_tooling
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.22.0
4
+ version: 1.24.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - GitLab Quality
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-04-10 00:00:00.000000000 Z
11
+ date: 2024-04-29 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: activesupport
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '6.1'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '7.2'
23
+ type: :development
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '6.1'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '7.2'
13
33
  - !ruby/object:Gem::Dependency
14
34
  name: climate_control
15
35
  requirement: !ruby/object:Gem::Requirement
@@ -66,6 +86,20 @@ dependencies:
66
86
  - - "~>"
67
87
  - !ruby/object:Gem::Version
68
88
  version: '4.7'
89
+ - !ruby/object:Gem::Dependency
90
+ name: influxdb-client
91
+ requirement: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - "~>"
94
+ - !ruby/object:Gem::Version
95
+ version: '3.1'
96
+ type: :development
97
+ prerelease: false
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - "~>"
101
+ - !ruby/object:Gem::Version
102
+ version: '3.1'
69
103
  - !ruby/object:Gem::Dependency
70
104
  name: lefthook
71
105
  requirement: !ruby/object:Gem::Requirement
@@ -404,6 +438,8 @@ files:
404
438
  - lefthook.yml
405
439
  - lib/gitlab_quality/test_tooling.rb
406
440
  - lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb
441
+ - lib/gitlab_quality/test_tooling/concerns/influxdb_tools.rb
442
+ - lib/gitlab_quality/test_tooling/concerns/test_metrics.rb
407
443
  - lib/gitlab_quality/test_tooling/failed_jobs_table.rb
408
444
  - lib/gitlab_quality/test_tooling/gitlab_client/branches_client.rb
409
445
  - lib/gitlab_quality/test_tooling/gitlab_client/branches_dry_client.rb
@@ -441,6 +477,7 @@ files:
441
477
  - lib/gitlab_quality/test_tooling/runtime/logger.rb
442
478
  - lib/gitlab_quality/test_tooling/slack/post_to_slack.rb
443
479
  - lib/gitlab_quality/test_tooling/slack/post_to_slack_dry.rb
480
+ - lib/gitlab_quality/test_tooling/stack_trace_comparator.rb
444
481
  - lib/gitlab_quality/test_tooling/summary_table.rb
445
482
  - lib/gitlab_quality/test_tooling/support/http_request.rb
446
483
  - lib/gitlab_quality/test_tooling/system_logs/finders/json_log_finder.rb
@@ -460,6 +497,7 @@ files:
460
497
  - lib/gitlab_quality/test_tooling/test_meta/processor/meta_processor.rb
461
498
  - lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb
462
499
  - lib/gitlab_quality/test_tooling/test_metric/json_test_metric.rb
500
+ - lib/gitlab_quality/test_tooling/test_metric/log_test_metrics.rb
463
501
  - lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb
464
502
  - lib/gitlab_quality/test_tooling/test_result/base_test_result.rb
465
503
  - lib/gitlab_quality/test_tooling/test_result/j_unit_test_result.rb
@@ -492,7 +530,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
492
530
  - !ruby/object:Gem::Version
493
531
  version: '0'
494
532
  requirements: []
495
- rubygems_version: 3.3.26
533
+ rubygems_version: 3.3.27
496
534
  signing_key:
497
535
  specification_version: 4
498
536
  summary: A collection of test-related tools.