gitlab-qa 6.10.0 → 6.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e7b24a0d671a29568447a0db95b57a06c43ed750d220dc13a0ad8216b7ba6297
4
- data.tar.gz: 9e03938be0b67f4f9a522703156bfc1b191ac0b8b97661c5eb74b82bfd91c84f
3
+ metadata.gz: 79cd3084cf93dc40c0935800fd13991f2b8e057fd08aca2b91cffddc776675c3
4
+ data.tar.gz: 0ad9aecd55123681543e84b7ff2150ec8d4a0f004374dd43cd7ced64a89b11f9
5
5
  SHA512:
6
- metadata.gz: 21ea402dd74ab453e9ac303303f50925a32a82e58b7e4ff0aa57721d4e8301717febc4da56b1b5b3c7141ad7f0e613b56fc7d47b7a25256b83f07ec53980db33
7
- data.tar.gz: e31d991f0f78bfc510dbaafb3a844126d95c2e46655b2d655c256dcdfda673ca8038085c33deb65caa00728effb63e75020e3f6c217ddc697e87ceaded1270ba
6
+ metadata.gz: 29b7032d6ad65269e2aa27d03bea3bff2805bd160550c69fefd08475bb60ad5160c2417422e58b2b29d1ef4c2c31d3ebc6661cb0cbb151522dedb6dda97e64d7
7
+ data.tar.gz: 16d5d8ae03e4542c7518e1466723eee32e927a523890c3d56c565df2cabd363a180116a2d4db5d2eefd323dadb355f1471c7d7da9a0fcff61eb13f607a174b95
@@ -49,9 +49,9 @@ variables:
49
49
  QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
50
50
  QA_CAN_TEST_PRAEFECT: "false"
51
51
  QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
52
- QA_FAILURES_REPORTING_PROJECT: "gitlab-org/gitlab"
52
+ QA_FAILURES_REPORTING_PROJECT: "rymai/gitlab-qa-issues"
53
53
  # The --dry-run or --max-diff-ratio option can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
54
- QA_FAILURES_REPORTER_OPTIONS: "--dry-run"
54
+ QA_FAILURES_REPORTER_OPTIONS: ""
55
55
  QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
56
56
 
57
57
  .check-base:
@@ -95,8 +95,8 @@ release:
95
95
  reports:
96
96
  junit: gitlab-qa-run-*/**/rspec-*.xml
97
97
  script:
98
- - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
99
- - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
98
+ - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
99
+ - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
100
100
  - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
101
101
  - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
102
102
  - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
@@ -175,8 +175,8 @@ ee:sanity-framework:
175
175
  # The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues
176
176
  ce:custom-parallel:
177
177
  script:
178
- - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
179
- - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
178
+ - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
179
+ - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
180
180
  extends:
181
181
  - .test
182
182
  - .high-capacity
@@ -187,8 +187,8 @@ ce:custom-parallel:
187
187
 
188
188
  ee:custom-parallel:
189
189
  script:
190
- - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
191
- - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
190
+ - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
191
+ - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
192
192
  extends:
193
193
  - .test
194
194
  - .high-capacity
@@ -4,7 +4,7 @@
4
4
 
5
5
  The [maven repository spec](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/ee/browser_ui/5_package/maven_repository_spec.rb) creates a Maven artifact and links it to a GitLab project. The artifact is created within a [Maven docker image](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/service/docker_run/maven.rb#L8).
6
6
 
7
- Using `gitlab-qa` to run these tests reduces the liklihood of network errors between the maven container and GitLab instance.
7
+ Using `gitlab-qa` to run these tests reduces the likelihood of network errors between the maven container and GitLab instance.
8
8
 
9
9
  To run this with `gitlab-qa` you can use the `Test::Instance::Image` that is needed for your test. For example:
10
10
 
@@ -84,6 +84,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
84
84
  | `JIRA_ADMIN_PASSWORD` |- | Password for authenticating with Jira server as admin. | No|
85
85
  | `CACHE_NAMESPACE_NAME` | `true` | Cache namespace name for groups. | No|
86
86
  | `DEPLOY_VERSION` |- | The version of GitLab being tested against. | No|
87
+ | `GITLAB_QA_USER_AGENT` |- | The browser user-agent to use instead of the default Chrome user-agent. | No|
87
88
 
88
89
  ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
89
90
 
@@ -6,6 +6,7 @@ module Gitlab
6
6
  class GenerateTestSession < ReportAsIssue
7
7
  private
8
8
 
9
+ # rubocop:disable Metrics/AbcSize
9
10
  def run!
10
11
  puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
11
12
 
@@ -18,11 +19,12 @@ module Gitlab
18
19
  issue = gitlab.create_issue(
19
20
  title: "Test session report | #{Runtime::Env.deploy_environment}",
20
21
  description: generate_description(tests),
21
- labels: ['Quality', 'QA', 'triage report']
22
+ labels: ['Quality', 'QA', 'triage report', deploy_environment_label]
22
23
  )
23
24
 
24
25
  File.write('REPORT_ISSUE_URL', issue.web_url)
25
26
  end
27
+ # rubocop:enable Metrics/AbcSize
26
28
 
27
29
  def generate_description(tests)
28
30
  <<~MARKDOWN
@@ -32,6 +34,8 @@ module Gitlab
32
34
  * Pipeline: [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
33
35
  #{generate_summary(tests: tests)}
34
36
 
37
+ #{generate_failed_jobs_listing}
38
+
35
39
  #{generate_stages_listing(tests)}
36
40
 
37
41
  ## Release QA issue
@@ -55,6 +59,33 @@ module Gitlab
55
59
  MARKDOWN
56
60
  end
57
61
 
62
+ def generate_failed_jobs_listing
63
+ failed_jobs = []
64
+
65
+ client = Gitlab.client(
66
+ endpoint: Runtime::Env.ci_api_v4_url,
67
+ private_token: Runtime::Env.gitlab_ci_api_token)
68
+
69
+ gitlab.handle_gitlab_client_exceptions do
70
+ failed_jobs = client.pipeline_jobs(
71
+ Runtime::Env.ci_project_id,
72
+ Runtime::Env.ci_pipeline_id,
73
+ scope: 'failed')
74
+ end
75
+
76
+ listings = failed_jobs.map do |job|
77
+ allowed_to_fail = ' (allowed to fail)' if job.allow_failure
78
+
79
+ "* [#{job.name}](#{job.web_url})#{allowed_to_fail}"
80
+ end.join("\n")
81
+
82
+ <<~MARKDOWN.chomp if failed_jobs.any?
83
+ ## Failed jobs
84
+
85
+ #{listings}
86
+ MARKDOWN
87
+ end
88
+
58
89
  def generate_stages_listing(tests)
59
90
  generate_tests_by_stage(tests).map do |stage, tests_for_stage|
60
91
  tests_by_status = tests_for_stage.group_by(&:status)
@@ -185,7 +216,7 @@ module Gitlab
185
216
  tests_with_same_testcase.select(&:failure_issue)
186
217
 
187
218
  if tests_having_failure_issue.any?
188
- items = tests_having_failure_issue.map do |test|
219
+ items = tests_having_failure_issue.uniq(&:failure_issue).map do |test|
189
220
  "<li>[ ] [failure issue](#{test.failure_issue})</li>"
190
221
  end.join(' ')
191
222
 
@@ -109,6 +109,9 @@ module Gitlab
109
109
  pipeline = QA::Runtime::Env.pipeline_from_project_name
110
110
  channel = pipeline == "canary" ? "qa-production" : "qa-#{pipeline}"
111
111
  error_msg = warn_exception(e)
112
+
113
+ return unless QA::Runtime::Env.ci_commit_ref_name == 'master'
114
+
112
115
  slack_options = {
113
116
  channel: channel,
114
117
  icon_emoji: ':ci_failing:',
@@ -10,7 +10,7 @@ module Gitlab
10
10
  # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
11
11
  class RelateFailureIssue < ReportAsIssue
12
12
  DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
13
- STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error: .+)(\1)}m.freeze
13
+ STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error:.+)(\1)}m.freeze
14
14
  NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
15
15
 
16
16
  MultipleIssuesFound = Class.new(StandardError)
@@ -47,6 +47,7 @@ module Gitlab
47
47
  issue = find_or_create_issue(test)
48
48
  return unless issue
49
49
 
50
+ update_labels(issue, test)
50
51
  post_failed_job_note(issue, test)
51
52
  puts " => Marked #{issue.web_url} as related to #{test.testcase}."
52
53
  rescue MultipleIssuesFound => e
@@ -81,17 +82,20 @@ module Gitlab
81
82
  # Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
82
83
  failure_issues(test).each_with_object({}) do |issue, memo|
83
84
  relevant_issue_stacktrace = find_issue_stacktrace(issue)
84
- unless relevant_issue_stacktrace
85
- puts " => [DEBUG] Found issue #{issue.web_url} but stacktrace doesn't match."
86
- next
87
- end
85
+ next unless relevant_issue_stacktrace
88
86
 
89
87
  distance = ld.call(first_test_failure_stacktrace, relevant_issue_stacktrace)
90
88
  diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
91
89
  if diff_ratio <= max_diff_ratio
92
- memo[issue] = diff_ratio
90
+ puts " => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
91
+ # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
92
+ # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
93
+ # See:
94
+ # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
95
+ # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
96
+ memo[issue.to_h] = diff_ratio
93
97
  else
94
- puts " => [DEBUG] Found issue #{issue.web_url} but stacktrace is too different (#{(diff_ratio * 100).round(2)}%)."
98
+ puts " => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%)."
95
99
  end
96
100
  end
97
101
  end
@@ -102,7 +106,7 @@ module Gitlab
102
106
  if issue_stacktrace_match
103
107
  issue_stacktrace_match[2].gsub(/^#.*$/, '').strip
104
108
  else
105
- puts "\n => Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
109
+ puts " => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
106
110
  end
107
111
  end
108
112
 
@@ -117,24 +121,35 @@ module Gitlab
117
121
  raise(MultipleIssuesFound, %(Too many issues found for test '#{test.name}' (`#{test.file}`)!))
118
122
  end
119
123
 
124
+ # Re-instantiate a `Gitlab::ObjectifiedHash` object after having converted it to a hash in #find_relevant_failure_issues above.
125
+ best_matching_issue = Gitlab::ObjectifiedHash.new(best_matching_issue)
126
+
120
127
  test.failure_issue ||= best_matching_issue.web_url
121
128
 
122
129
  [best_matching_issue, smaller_diff_ratio]
123
130
  end
124
131
 
125
132
  def new_issue_description(test)
126
- super + "\n\n### Stack trace\n\n```\n#{test.failures.first['message_lines'].join("\n")}\n```"
133
+ super + [
134
+ "\n\n### Stack trace",
135
+ "```\n#{test.failures.first['message_lines'].join("\n")}\n```",
136
+ "First happened in #{test.ci_job_url}."
137
+ ].join("\n\n")
127
138
  end
128
139
 
129
140
  def new_issue_labels(test)
130
141
  NEW_ISSUE_LABELS + up_to_date_labels(test: test)
131
142
  end
132
143
 
144
+ def up_to_date_labels(test:, issue: nil)
145
+ super << deploy_environment_label
146
+ end
147
+
133
148
  def post_failed_job_note(issue, test)
134
149
  gitlab.create_issue_note(iid: issue.iid, note: "/relate #{test.testcase}")
135
150
  end
136
151
 
137
- def title_from_test(test)
152
+ def new_issue_title(test)
138
153
  "Failure in #{super}"
139
154
  end
140
155
  end
@@ -28,6 +28,10 @@ module Gitlab
28
28
  raise NotImplementedError
29
29
  end
30
30
 
31
+ def new_issue_title(test)
32
+ "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
33
+ end
34
+
31
35
  def new_issue_description(test)
32
36
  "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}"
33
37
  end
@@ -100,6 +104,23 @@ module Gitlab
100
104
  labels
101
105
  end
102
106
 
107
+ def deploy_environment_label
108
+ environment = Runtime::Env.deploy_environment
109
+
110
+ case environment
111
+ when 'production'
112
+ 'found:gitlab.com'
113
+ when 'canary', 'staging'
114
+ "found:#{environment}.gitlab.com"
115
+ when 'preprod'
116
+ 'found:pre.gitlab.com'
117
+ when 'staging-orchestrated', 'nightly', 'master'
118
+ "found:#{environment}"
119
+ else
120
+ raise "No `found:*` label for the `#{environment}` environment!"
121
+ end
122
+ end
123
+
103
124
  def ee_test?(test)
104
125
  test.file =~ %r{features/ee/(api|browser_ui)}
105
126
  end
@@ -113,7 +134,7 @@ module Gitlab
113
134
  end
114
135
 
115
136
  def title_from_test(test)
116
- title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
137
+ title = new_issue_title(test)
117
138
 
118
139
  return title unless title.length > MAX_TITLE_LENGTH
119
140
 
@@ -25,8 +25,6 @@ module Gitlab
25
25
  end
26
26
 
27
27
  def report_test(test)
28
- return if test.skipped
29
-
30
28
  puts "Reporting test: #{test.file} | #{test.name}"
31
29
 
32
30
  issue = find_issue(test)
@@ -34,16 +32,19 @@ module Gitlab
34
32
  if issue
35
33
  puts "Found existing issue: #{issue.web_url}"
36
34
  else
35
+ # Don't create new issues for skipped tests
36
+ return if test.skipped
37
+
37
38
  issue = create_issue(test)
38
39
  puts "Created new issue: #{issue.web_url}"
39
40
  end
40
41
 
41
42
  test.testcase ||= issue.web_url
42
43
 
43
- update_labels(issue, test)
44
+ labels_updated = update_labels(issue, test)
44
45
  note_posted = note_status(issue, test)
45
46
 
46
- if note_posted
47
+ if labels_updated || note_posted
47
48
  puts "Issue updated."
48
49
  else
49
50
  puts "Test passed, no update needed."
@@ -83,6 +84,7 @@ module Gitlab
83
84
  end
84
85
 
85
86
  def note_status(issue, test)
87
+ return false if test.skipped
86
88
  return false if test.failures.empty?
87
89
 
88
90
  note = note_content(test)
@@ -85,7 +85,8 @@ module Gitlab
85
85
  'JIRA_ADMIN_USERNAME' => :jira_admin_username,
86
86
  'JIRA_ADMIN_PASSWORD' => :jira_admin_password,
87
87
  'CACHE_NAMESPACE_NAME' => :cache_namespace_name,
88
- 'DEPLOY_VERSION' => :deploy_version
88
+ 'DEPLOY_VERSION' => :deploy_version,
89
+ 'GITLAB_QA_USER_AGENT' => :gitlab_qa_user_agent
89
90
  }.freeze
90
91
 
91
92
  ENV_VARIABLES.each do |env_name, method_name|
@@ -115,6 +116,14 @@ module Gitlab
115
116
  ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
116
117
  end
117
118
 
119
+ def gitlab_ci_api_token
120
+ ENV['GITLAB_CI_API_TOKEN']
121
+ end
122
+
123
+ def ci_api_v4_url
124
+ ENV['CI_API_V4_URL']
125
+ end
126
+
118
127
  def ci_job_name
119
128
  ENV['CI_JOB_NAME']
120
129
  end
@@ -135,8 +144,12 @@ module Gitlab
135
144
  ENV['CI_PIPELINE_ID']
136
145
  end
137
146
 
138
- def ci_project_name
139
- ENV['CI_PROJECT_NAME']
147
+ def ci_project_id
148
+ ENV['CI_PROJECT_ID']
149
+ end
150
+
151
+ def ci_commit_ref_name
152
+ ENV['CI_COMMIT_REF_NAME']
140
153
  end
141
154
 
142
155
  def pipeline_from_project_name
@@ -1,5 +1,5 @@
1
1
  module Gitlab
2
2
  module QA
3
- VERSION = '6.10.0'.freeze
3
+ VERSION = '6.14.0'.freeze
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-qa
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.10.0
4
+ version: 6.14.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Grzegorz Bizon
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-11-17 00:00:00.000000000 Z
11
+ date: 2020-11-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: climate_control