testrailtagging 0.3.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/CODE_OF_CONDUCT.md +49 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +75 -0
- data/Rakefile +2 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/lib/files/RSpecParser.rb +177 -0
- data/lib/files/TestCase.rb +38 -0
- data/lib/files/testcase_modifications.rb +393 -0
- data/lib/files/testrail_apiclient_retry.rb +46 -0
- data/lib/files/testrail_operations.rb +369 -0
- data/lib/files/testrail_queries.rb +108 -0
- data/lib/files/testrail_rspec_integration.rb +385 -0
- data/lib/files/version.rb +3 -0
- data/lib/testrailtagging.rb +4 -0
- data/testrailtagging.gemspec +28 -0
- metadata +133 -0
@@ -0,0 +1,393 @@
|
|
1
|
+
require_relative "testrail_operations"
|
2
|
+
require_relative "RSpecParser"
|
3
|
+
|
4
|
+
# =================================================================================
|
5
|
+
#
|
6
|
+
# Test case parsing and modifications
|
7
|
+
#
|
8
|
+
# =================================================================================
|
9
|
+
|
10
|
+
module TestCaseModifications
|
11
|
+
# Given a line of code extracted from an RSPEC example, it returns an array of numbers (integers)
|
12
|
+
# that correspond to the testrail_id's.
|
13
|
+
def self.get_example_testrail_ids(line)
|
14
|
+
# puts line
|
15
|
+
testrail_id = "testrail_id:"
|
16
|
+
length = testrail_id.size
|
17
|
+
index = line.index(testrail_id)
|
18
|
+
|
19
|
+
substr = line[(index + length)..-1]
|
20
|
+
comma = substr.index(",")
|
21
|
+
|
22
|
+
if comma
|
23
|
+
substr = substr[0, comma]
|
24
|
+
end
|
25
|
+
substr = substr.gsub(" do", "")
|
26
|
+
substr = substr.gsub("%w[", "")
|
27
|
+
substr = substr.gsub("]", "")
|
28
|
+
|
29
|
+
numbers = substr.split(" ")
|
30
|
+
result = []
|
31
|
+
numbers.each do |str|
|
32
|
+
result << str.to_i
|
33
|
+
end
|
34
|
+
result
|
35
|
+
end
|
36
|
+
|
37
|
+
# Gets a description of the minimum screen size given a line of text from the rspec.
|
38
|
+
# Given a line of code that was extracted from the test case contains a testrail_id tag,
|
39
|
+
# this function will extract the test rail ID, look it up in a list of test cases
|
40
|
+
# and return the type of screen size this minimally works on.
|
41
|
+
def self.get_example_device_type(line, test_cases)
|
42
|
+
ids = get_example_testrail_ids(line)
|
43
|
+
# puts ids
|
44
|
+
if (ids.size == 1)
|
45
|
+
id = ids[0]
|
46
|
+
tc = test_cases[id]
|
47
|
+
if tc
|
48
|
+
tc.screen_size
|
49
|
+
else
|
50
|
+
# It could be in some other test rail project
|
51
|
+
nil
|
52
|
+
end
|
53
|
+
else
|
54
|
+
# Multiple test rail id's specified for the example.
|
55
|
+
desktop_count = 0
|
56
|
+
tablet_count = 0
|
57
|
+
phone_count = 0
|
58
|
+
ids.each do |id|
|
59
|
+
tc = test_cases[id]
|
60
|
+
next if tc.nil?
|
61
|
+
|
62
|
+
case tc.screen_size
|
63
|
+
when "Desktop"
|
64
|
+
desktop_count += 1
|
65
|
+
when "Tablet"
|
66
|
+
tablet_count += 1
|
67
|
+
when "Phone"
|
68
|
+
phone_count += 1
|
69
|
+
end
|
70
|
+
end
|
71
|
+
# Always go with the smallest minimum size specified in the test rail id list
|
72
|
+
if phone_count != 0
|
73
|
+
"Phone"
|
74
|
+
elsif tablet_count != 0
|
75
|
+
"Tablet"
|
76
|
+
elsif desktop_count != 0
|
77
|
+
"Desktop"
|
78
|
+
else
|
79
|
+
"<-- ========================== ERROR ============================== -->"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
# Finds the lowest priority for an array of test cases
|
85
|
+
# ids - array of test cases integer ID's.
|
86
|
+
# test_cases - array of TestCase instances
|
87
|
+
# returns the lower priority for a given set of test cases
|
88
|
+
def self.lowest_priority_of(ids, test_cases)
|
89
|
+
p0_count = 0
|
90
|
+
p1_count = 0
|
91
|
+
p2_count = 0
|
92
|
+
p3_count = 0
|
93
|
+
ids.each do |id|
|
94
|
+
tc = test_cases[id]
|
95
|
+
next if tc.nil?
|
96
|
+
|
97
|
+
case tc.priority
|
98
|
+
when 0
|
99
|
+
p0_count += 1
|
100
|
+
when 1
|
101
|
+
p1_count += 1
|
102
|
+
when 2
|
103
|
+
p2_count += 1
|
104
|
+
when 3
|
105
|
+
p3_count += 1
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
# Always go with the lowest priority specified in the test rail id list
|
110
|
+
if p0_count != 0
|
111
|
+
0
|
112
|
+
elsif p1_count != 0
|
113
|
+
1
|
114
|
+
elsif p2_count != 0
|
115
|
+
2
|
116
|
+
elsif p3_count != 0
|
117
|
+
3
|
118
|
+
else
|
119
|
+
"<-- ========================== ERROR ============================== -->"
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
# Get the priority of the test case given a line of text from the rspec.
|
124
|
+
# Given something like this:
|
125
|
+
# example "I make a course", testrail_id: %w[123456], priority: 1 do
|
126
|
+
# it will return a numeric value of the test priority
|
127
|
+
def self.get_example_priority(line, test_cases)
|
128
|
+
ids = get_example_testrail_ids(line)
|
129
|
+
if (ids.size == 1)
|
130
|
+
id = ids[0]
|
131
|
+
tc = test_cases[id]
|
132
|
+
if tc
|
133
|
+
tc.priority
|
134
|
+
else
|
135
|
+
# It could be in some other test rail project
|
136
|
+
nil
|
137
|
+
end
|
138
|
+
else
|
139
|
+
# Multiple test rail id's specified for the example.
|
140
|
+
lowest_priority_of(ids, test_cases)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
# PreCondition: This is only called on examples that are verified to have test id's attached to them.
|
145
|
+
# Removes and adds in a tag to the string string parameter 'line'.
|
146
|
+
def self.add_tag(tag, line, type_string, prepend_colon:true)
|
147
|
+
tag_index = line.index(tag)
|
148
|
+
modified_line = "0xDEADBEAF"
|
149
|
+
if tag_index
|
150
|
+
# Found an existing tag:, remove the existing one first.
|
151
|
+
next_comma = line.index(",", tag_index)
|
152
|
+
if next_comma
|
153
|
+
modified_line = line[0, tag_index - 2] + line[next_comma..-1]
|
154
|
+
else
|
155
|
+
# It's at the end, Look for the 'do' keyword
|
156
|
+
last_do = line.index("do\n")
|
157
|
+
first_part = line[0, tag_index - 2]
|
158
|
+
second_part = line[last_do..-1]
|
159
|
+
modified_line = first_part + " " + second_part
|
160
|
+
end
|
161
|
+
line = modified_line
|
162
|
+
end
|
163
|
+
|
164
|
+
colon = ":" if prepend_colon
|
165
|
+
|
166
|
+
tag_index = line.index(tag)
|
167
|
+
unless tag_index
|
168
|
+
# None found, add one after testrail_id:
|
169
|
+
testrail_index = line.index("testrail_id:")
|
170
|
+
if testrail_index
|
171
|
+
next_comma = line.index(",", testrail_index)
|
172
|
+
if next_comma
|
173
|
+
modified_line = line.insert(next_comma, ", #{tag} #{colon}#{type_string.downcase}")
|
174
|
+
else
|
175
|
+
# It was at the end already.
|
176
|
+
last_bracket = line.index("]", testrail_index)
|
177
|
+
modified_line = line.insert(last_bracket + 1, ", #{tag} #{colon}#{type_string.downcase}")
|
178
|
+
end
|
179
|
+
end
|
180
|
+
end
|
181
|
+
modified_line
|
182
|
+
end
|
183
|
+
|
184
|
+
# Modifies the Rspec files to add metadata to each test example block.
|
185
|
+
# This will pull down data from testrail.com for each test case. It will then extract the priority
|
186
|
+
# and the desktop size that the test case can run on, and then add those as metadata to the
|
187
|
+
# example. This will modify all the rspec files in the regression_spec folder.
|
188
|
+
def self.add_tags_to_rspec_tests(tag_priority: true, tag_device: true)
|
189
|
+
test_cases = TestRailOperations.get_test_rail_cases
|
190
|
+
spec_files = Dir["regression_spec/**/*_spec.rb"]
|
191
|
+
spec_files.each do |file|
|
192
|
+
puts "Rspec file: #{file}"
|
193
|
+
changes = [] # The new lines of code for the file
|
194
|
+
# Read the file
|
195
|
+
File.open(file).each do |line|
|
196
|
+
new_line = line
|
197
|
+
|
198
|
+
if line.match("testrail_id")
|
199
|
+
if tag_device
|
200
|
+
type = get_example_device_type(line, test_cases)
|
201
|
+
if type
|
202
|
+
new_line = add_tag("device:", line, type)
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
if tag_priority
|
207
|
+
priority = get_example_priority(line, test_cases)
|
208
|
+
if priority
|
209
|
+
new_line = add_tag("priority:", new_line, priority.to_s, prepend_colon: false)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
213
|
+
changes << new_line
|
214
|
+
end
|
215
|
+
|
216
|
+
# Write the file out
|
217
|
+
File.open(file, "w") do |f|
|
218
|
+
changes.each do |line|
|
219
|
+
f.puts line
|
220
|
+
end
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
# This takes the file name associated with each testrail_id and posts it to the associated test rail ID on
|
226
|
+
# testrail.com.
|
227
|
+
# For example, if an rspec test example has a testrail_id of 123123 in file foo_spec.rb, it will update the test
|
228
|
+
# case on test rail and update the spec location field with foo_spec.rb.
|
229
|
+
# This will iterate over all the files in the regression_spec folder.
|
230
|
+
def self.update_automated_status(dryrun:false)
|
231
|
+
test_cases = TestRailOperations.get_test_rail_cases
|
232
|
+
regression_files = Dir["regression_spec/**/*_spec.rb"]
|
233
|
+
spec_files = regression_files + Dir["spec/**/*_spec.rb"]
|
234
|
+
# For keeping test cases that actually changed
|
235
|
+
changed_cases = {}
|
236
|
+
# parse all the files looking for examples
|
237
|
+
spec_files.each do |file|
|
238
|
+
# puts "Rspec file: #{file}"
|
239
|
+
File.open(file).each do |line|
|
240
|
+
if line.match("testrail_id")
|
241
|
+
testrail_ids = get_example_testrail_ids(line)
|
242
|
+
testrail_ids.each do |id|
|
243
|
+
# puts " id: #{id}"
|
244
|
+
tc = test_cases[id]
|
245
|
+
if tc
|
246
|
+
if file != tc.file
|
247
|
+
puts "\r\nID: #{id} - #{tc.title[0,20]}"
|
248
|
+
puts " Old File: #{tc.file}"
|
249
|
+
puts " New File: #{file}"
|
250
|
+
tc.file = file
|
251
|
+
changed_cases[id] = tc
|
252
|
+
end
|
253
|
+
|
254
|
+
# assuming it never becomes un-automated
|
255
|
+
unless tc.automated
|
256
|
+
puts " ID: #{id} Marking automated"
|
257
|
+
tc.automated = true
|
258
|
+
changed_cases[id] = tc
|
259
|
+
end
|
260
|
+
end
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
264
|
+
end
|
265
|
+
|
266
|
+
puts "\nTest Cases that will get modified"
|
267
|
+
trclient = TestRailOperations.get_test_rail_client
|
268
|
+
changed_cases.each do |id_key, tc_val|
|
269
|
+
puts "Test Case: id: #{id_key}, #{tc_val.file}" if tc_val.file
|
270
|
+
url = "update_case/#{id_key}"
|
271
|
+
data = { "custom_spec_location" => tc_val.file, "custom_automated" => tc_val.automated }
|
272
|
+
unless dryrun
|
273
|
+
trclient.send_post_retry(url, data)
|
274
|
+
end
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
# checks for duplicate test case ID's in the rspec tests
|
279
|
+
def self.check_duplicates
|
280
|
+
regression_files = Dir["regression_spec/**/*_spec.rb"]
|
281
|
+
spec_files = regression_files + Dir["spec/**/*_spec.rb"]
|
282
|
+
# For keeping a running list of found ID's
|
283
|
+
# Key is integer ID
|
284
|
+
# Value is the file
|
285
|
+
ids = {}
|
286
|
+
# parse all the files looking for examples
|
287
|
+
spec_files.each do |file|
|
288
|
+
# puts "Rspec file: #{file}"
|
289
|
+
File.open(file).each do |line|
|
290
|
+
if line.match("testrail_id")
|
291
|
+
testrail_ids = get_example_testrail_ids(line)
|
292
|
+
testrail_ids.each do |id|
|
293
|
+
# puts " id: #{id}"
|
294
|
+
if ids[id]
|
295
|
+
puts "Found duplicate: #{id}"
|
296
|
+
puts "Other File: #{ids[id]}"
|
297
|
+
puts "This File: #{file}"
|
298
|
+
else
|
299
|
+
ids[id] = file
|
300
|
+
end
|
301
|
+
end
|
302
|
+
end
|
303
|
+
end
|
304
|
+
end
|
305
|
+
end
|
306
|
+
|
307
|
+
# Parses the rspec files and reports how many test cases are skipped, and a percentage of
|
308
|
+
# the test cases that are executed.
|
309
|
+
# returns a hash of test cases, where
|
310
|
+
# the key is an integer of the test case ID (as found in test rail), and
|
311
|
+
# the value is an instance of TestCase, containing all the skip information and everything.
|
312
|
+
def self.parse_specs
|
313
|
+
|
314
|
+
rspec_examples = []
|
315
|
+
spec_files = Dir["regression_spec/**/*_spec.rb"]
|
316
|
+
example_count = 0
|
317
|
+
skip_count = 0
|
318
|
+
file_count = 0
|
319
|
+
# parse all the files looking for examples
|
320
|
+
spec_files.each do |file|
|
321
|
+
file_count += 1
|
322
|
+
parser = RSpecParser.new(file)
|
323
|
+
parser.parse
|
324
|
+
parser.test_cases.each do |tc|
|
325
|
+
tc.file = file
|
326
|
+
if tc.skip.count > 0
|
327
|
+
skip_count += 1
|
328
|
+
end
|
329
|
+
end
|
330
|
+
rspec_examples += parser.test_cases
|
331
|
+
# puts "%5d in %s" % [parser.test_cases.count, file]
|
332
|
+
example_count += parser.test_cases.count
|
333
|
+
end
|
334
|
+
puts "total files: #{file_count}"
|
335
|
+
puts "total examples: #{example_count}"
|
336
|
+
puts "total examples skipped: #{skip_count}"
|
337
|
+
puts "total executed: #{example_count - skip_count}"
|
338
|
+
puts "total coverage: %.2f %" % [(1.0 - (skip_count / example_count.to_f)) * 100]
|
339
|
+
|
340
|
+
result = {}
|
341
|
+
rspec_examples.each do |tc|
|
342
|
+
next if tc.id.nil?
|
343
|
+
tc.id.each do |id|
|
344
|
+
result[id.to_i] = tc
|
345
|
+
end
|
346
|
+
end
|
347
|
+
result
|
348
|
+
end
|
349
|
+
|
350
|
+
@browsers = { "none" => 0, "chrome" => 1, "firefox" => 2, "ie10" => 3, "ie11" => 4, "safari" => 5 }
|
351
|
+
def self.browsers_to_testrail(browser_array)
|
352
|
+
result = []
|
353
|
+
browser_array.each do |name|
|
354
|
+
if name == "allbrowsers"
|
355
|
+
result = [1, 2, 3, 5]
|
356
|
+
return result
|
357
|
+
else
|
358
|
+
result << @browsers[name]
|
359
|
+
end
|
360
|
+
end
|
361
|
+
result
|
362
|
+
end
|
363
|
+
|
364
|
+
# Parses all the test cases as found in the rspec files.
|
365
|
+
# then pushes information about teach test case up to testrail.
|
366
|
+
# This will update 3 fields about each test case:
|
367
|
+
# 1. the spec file location
|
368
|
+
# 2. if the test is skipped, and on which browser
|
369
|
+
# 3. if the test is automated (by default is true)
|
370
|
+
def self.push_to_testrail
|
371
|
+
rspec_examples = parse_specs
|
372
|
+
trclient = TestRailOperations.get_test_rail_client
|
373
|
+
|
374
|
+
# First set the fields to blank in test rail
|
375
|
+
test_cases = TestRailOperations.get_test_rail_cases
|
376
|
+
# test_cases.each do |key_id, val_tc|
|
377
|
+
# url = "update_case/#{key_id}"
|
378
|
+
# data = { "custom_spec_location" => nil, "custom_browser_skip" => [], "custom_automated" => false }
|
379
|
+
# trclient.send_post(url, data)
|
380
|
+
# end
|
381
|
+
|
382
|
+
# Then upload the information contain in our rspec files
|
383
|
+
rspec_examples.each do |id_key, tc_val|
|
384
|
+
if test_cases.key?(id_key)
|
385
|
+
url = "update_case/#{id_key}"
|
386
|
+
puts "updating test case: "
|
387
|
+
browser_skips = browsers_to_testrail(tc_val.skip)
|
388
|
+
data = { "custom_spec_location" => tc_val.file, "custom_browser_skip" => browser_skips, "custom_automated" => true }
|
389
|
+
trclient.send_post_retry(url, data)
|
390
|
+
end
|
391
|
+
end
|
392
|
+
end
|
393
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require "testrail_client"
|
2
|
+
|
3
|
+
# =================================================================================
|
4
|
+
#
|
5
|
+
# API's for Test Rail
|
6
|
+
#
|
7
|
+
# =================================================================================
|
8
|
+
|
9
|
+
module TestRail
|
10
|
+
class APIClient
|
11
|
+
def send_post_retry(uri, data)
|
12
|
+
# Gurock api's often deadlocks with errors like this:
|
13
|
+
# TestRail API returned HTTP 500 ("Deadlock found when trying to get lock; try restarting transaction")
|
14
|
+
# So if they say to retry, then that's what we will do
|
15
|
+
|
16
|
+
# Please note that the API is rate limited on TestRail Hosted and may throttle requests.
|
17
|
+
# TestRail might also return a 429 Too Many Requests response which you are expected to handle.
|
18
|
+
# Such a response also includes a Retry-After header indicating how many seconds to wait
|
19
|
+
# before you are allowed to submit the next request.
|
20
|
+
# http://docs.gurock.com/testrail-api2/introduction #Rate Limit
|
21
|
+
response = nil
|
22
|
+
# todo: use header [Retry-After] secs
|
23
|
+
# for HTTPTooManyRequests 429 error, retry post after either 10s, 30s, 90s or 270s.
|
24
|
+
exponential_backoff_seconds = 10
|
25
|
+
|
26
|
+
4.times do
|
27
|
+
begin
|
28
|
+
response = send_post(uri, data)
|
29
|
+
break
|
30
|
+
rescue TestRail::APIError => e
|
31
|
+
if e.message && e.message.match("HTTP 500.*Deadlock")
|
32
|
+
sleep 1
|
33
|
+
elsif e.message && e.message.match("HTTP 429")
|
34
|
+
puts "TestRail rate limited. retrying in #{exponential_backoff_seconds}"
|
35
|
+
sleep exponential_backoff_seconds
|
36
|
+
exponential_backoff_seconds *= 3
|
37
|
+
else
|
38
|
+
# Don't retry it, let the exception propagate
|
39
|
+
raise
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
response
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|