testrailtagging 0.3.6.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,393 @@
1
+ require_relative "testrail_operations"
2
+ require_relative "RSpecParser"
3
+
4
+ # =================================================================================
5
+ #
6
+ # Test case parsing and modifications
7
+ #
8
+ # =================================================================================
9
+
10
+ module TestCaseModifications
11
+ # Given a line of code extracted from an RSPEC example, it returns an array of numbers (integers)
12
+ # that correspond to the testrail_id's.
13
+ def self.get_example_testrail_ids(line)
14
+ # puts line
15
+ testrail_id = "testrail_id:"
16
+ length = testrail_id.size
17
+ index = line.index(testrail_id)
18
+
19
+ substr = line[(index + length)..-1]
20
+ comma = substr.index(",")
21
+
22
+ if comma
23
+ substr = substr[0, comma]
24
+ end
25
+ substr = substr.gsub(" do", "")
26
+ substr = substr.gsub("%w[", "")
27
+ substr = substr.gsub("]", "")
28
+
29
+ numbers = substr.split(" ")
30
+ result = []
31
+ numbers.each do |str|
32
+ result << str.to_i
33
+ end
34
+ result
35
+ end
36
+
37
+ # Gets a description of the minimum screen size given a line of text from the rspec.
38
+ # Given a line of code that was extracted from the test case contains a testrail_id tag,
39
+ # this function will extract the test rail ID, look it up in a list of test cases
40
+ # and return the type of screen size this minimally works on.
41
+ def self.get_example_device_type(line, test_cases)
42
+ ids = get_example_testrail_ids(line)
43
+ # puts ids
44
+ if (ids.size == 1)
45
+ id = ids[0]
46
+ tc = test_cases[id]
47
+ if tc
48
+ tc.screen_size
49
+ else
50
+ # It could be in some other test rail project
51
+ nil
52
+ end
53
+ else
54
+ # Multiple test rail id's specified for the example.
55
+ desktop_count = 0
56
+ tablet_count = 0
57
+ phone_count = 0
58
+ ids.each do |id|
59
+ tc = test_cases[id]
60
+ next if tc.nil?
61
+
62
+ case tc.screen_size
63
+ when "Desktop"
64
+ desktop_count += 1
65
+ when "Tablet"
66
+ tablet_count += 1
67
+ when "Phone"
68
+ phone_count += 1
69
+ end
70
+ end
71
+ # Always go with the smallest minimum size specified in the test rail id list
72
+ if phone_count != 0
73
+ "Phone"
74
+ elsif tablet_count != 0
75
+ "Tablet"
76
+ elsif desktop_count != 0
77
+ "Desktop"
78
+ else
79
+ "<-- ========================== ERROR ============================== -->"
80
+ end
81
+ end
82
+ end
83
+
84
+ # Finds the lowest priority for an array of test cases
85
+ # ids - array of test cases integer ID's.
86
+ # test_cases - array of TestCase instances
87
+ # returns the lower priority for a given set of test cases
88
+ def self.lowest_priority_of(ids, test_cases)
89
+ p0_count = 0
90
+ p1_count = 0
91
+ p2_count = 0
92
+ p3_count = 0
93
+ ids.each do |id|
94
+ tc = test_cases[id]
95
+ next if tc.nil?
96
+
97
+ case tc.priority
98
+ when 0
99
+ p0_count += 1
100
+ when 1
101
+ p1_count += 1
102
+ when 2
103
+ p2_count += 1
104
+ when 3
105
+ p3_count += 1
106
+ end
107
+ end
108
+
109
+ # Always go with the lowest priority specified in the test rail id list
110
+ if p0_count != 0
111
+ 0
112
+ elsif p1_count != 0
113
+ 1
114
+ elsif p2_count != 0
115
+ 2
116
+ elsif p3_count != 0
117
+ 3
118
+ else
119
+ "<-- ========================== ERROR ============================== -->"
120
+ end
121
+ end
122
+
123
+ # Get the priority of the test case given a line of text from the rspec.
124
+ # Given something like this:
125
+ # example "I make a course", testrail_id: %w[123456], priority: 1 do
126
+ # it will return a numeric value of the test priority
127
+ def self.get_example_priority(line, test_cases)
128
+ ids = get_example_testrail_ids(line)
129
+ if (ids.size == 1)
130
+ id = ids[0]
131
+ tc = test_cases[id]
132
+ if tc
133
+ tc.priority
134
+ else
135
+ # It could be in some other test rail project
136
+ nil
137
+ end
138
+ else
139
+ # Multiple test rail id's specified for the example.
140
+ lowest_priority_of(ids, test_cases)
141
+ end
142
+ end
143
+
144
+ # PreCondition: This is only called on examples that are verified to have test id's attached to them.
145
+ # Removes and adds in a tag to the string string parameter 'line'.
146
+ def self.add_tag(tag, line, type_string, prepend_colon:true)
147
+ tag_index = line.index(tag)
148
+ modified_line = "0xDEADBEAF"
149
+ if tag_index
150
+ # Found an existing tag:, remove the existing one first.
151
+ next_comma = line.index(",", tag_index)
152
+ if next_comma
153
+ modified_line = line[0, tag_index - 2] + line[next_comma..-1]
154
+ else
155
+ # It's at the end, Look for the 'do' keyword
156
+ last_do = line.index("do\n")
157
+ first_part = line[0, tag_index - 2]
158
+ second_part = line[last_do..-1]
159
+ modified_line = first_part + " " + second_part
160
+ end
161
+ line = modified_line
162
+ end
163
+
164
+ colon = ":" if prepend_colon
165
+
166
+ tag_index = line.index(tag)
167
+ unless tag_index
168
+ # None found, add one after testrail_id:
169
+ testrail_index = line.index("testrail_id:")
170
+ if testrail_index
171
+ next_comma = line.index(",", testrail_index)
172
+ if next_comma
173
+ modified_line = line.insert(next_comma, ", #{tag} #{colon}#{type_string.downcase}")
174
+ else
175
+ # It was at the end already.
176
+ last_bracket = line.index("]", testrail_index)
177
+ modified_line = line.insert(last_bracket + 1, ", #{tag} #{colon}#{type_string.downcase}")
178
+ end
179
+ end
180
+ end
181
+ modified_line
182
+ end
183
+
184
+ # Modifies the Rspec files to add metadata to each test example block.
185
+ # This will pull down data from testrail.com for each test case. It will then extract the priority
186
+ # and the desktop size that the test case can run on, and then add those as metadata to the
187
+ # example. This will modify all the rspec files in the regression_spec folder.
188
+ def self.add_tags_to_rspec_tests(tag_priority: true, tag_device: true)
189
+ test_cases = TestRailOperations.get_test_rail_cases
190
+ spec_files = Dir["regression_spec/**/*_spec.rb"]
191
+ spec_files.each do |file|
192
+ puts "Rspec file: #{file}"
193
+ changes = [] # The new lines of code for the file
194
+ # Read the file
195
+ File.open(file).each do |line|
196
+ new_line = line
197
+
198
+ if line.match("testrail_id")
199
+ if tag_device
200
+ type = get_example_device_type(line, test_cases)
201
+ if type
202
+ new_line = add_tag("device:", line, type)
203
+ end
204
+ end
205
+
206
+ if tag_priority
207
+ priority = get_example_priority(line, test_cases)
208
+ if priority
209
+ new_line = add_tag("priority:", new_line, priority.to_s, prepend_colon: false)
210
+ end
211
+ end
212
+ end
213
+ changes << new_line
214
+ end
215
+
216
+ # Write the file out
217
+ File.open(file, "w") do |f|
218
+ changes.each do |line|
219
+ f.puts line
220
+ end
221
+ end
222
+ end
223
+ end
224
+
225
+ # This takes the file name associated with each testrail_id and posts it to the associated test rail ID on
226
+ # testrail.com.
227
+ # For example, if an rspec test example has a testrail_id of 123123 in file foo_spec.rb, it will update the test
228
+ # case on test rail and update the spec location field with foo_spec.rb.
229
+ # This will iterate over all the files in the regression_spec folder.
230
+ def self.update_automated_status(dryrun:false)
231
+ test_cases = TestRailOperations.get_test_rail_cases
232
+ regression_files = Dir["regression_spec/**/*_spec.rb"]
233
+ spec_files = regression_files + Dir["spec/**/*_spec.rb"]
234
+ # For keeping test cases that actually changed
235
+ changed_cases = {}
236
+ # parse all the files looking for examples
237
+ spec_files.each do |file|
238
+ # puts "Rspec file: #{file}"
239
+ File.open(file).each do |line|
240
+ if line.match("testrail_id")
241
+ testrail_ids = get_example_testrail_ids(line)
242
+ testrail_ids.each do |id|
243
+ # puts " id: #{id}"
244
+ tc = test_cases[id]
245
+ if tc
246
+ if file != tc.file
247
+ puts "\r\nID: #{id} - #{tc.title[0,20]}"
248
+ puts " Old File: #{tc.file}"
249
+ puts " New File: #{file}"
250
+ tc.file = file
251
+ changed_cases[id] = tc
252
+ end
253
+
254
+ # assuming it never becomes un-automated
255
+ unless tc.automated
256
+ puts " ID: #{id} Marking automated"
257
+ tc.automated = true
258
+ changed_cases[id] = tc
259
+ end
260
+ end
261
+ end
262
+ end
263
+ end
264
+ end
265
+
266
+ puts "\nTest Cases that will get modified"
267
+ trclient = TestRailOperations.get_test_rail_client
268
+ changed_cases.each do |id_key, tc_val|
269
+ puts "Test Case: id: #{id_key}, #{tc_val.file}" if tc_val.file
270
+ url = "update_case/#{id_key}"
271
+ data = { "custom_spec_location" => tc_val.file, "custom_automated" => tc_val.automated }
272
+ unless dryrun
273
+ trclient.send_post_retry(url, data)
274
+ end
275
+ end
276
+ end
277
+
278
+ # checks for duplicate test case ID's in the rspec tests
279
+ def self.check_duplicates
280
+ regression_files = Dir["regression_spec/**/*_spec.rb"]
281
+ spec_files = regression_files + Dir["spec/**/*_spec.rb"]
282
+ # For keeping a running list of found ID's
283
+ # Key is integer ID
284
+ # Value is the file
285
+ ids = {}
286
+ # parse all the files looking for examples
287
+ spec_files.each do |file|
288
+ # puts "Rspec file: #{file}"
289
+ File.open(file).each do |line|
290
+ if line.match("testrail_id")
291
+ testrail_ids = get_example_testrail_ids(line)
292
+ testrail_ids.each do |id|
293
+ # puts " id: #{id}"
294
+ if ids[id]
295
+ puts "Found duplicate: #{id}"
296
+ puts "Other File: #{ids[id]}"
297
+ puts "This File: #{file}"
298
+ else
299
+ ids[id] = file
300
+ end
301
+ end
302
+ end
303
+ end
304
+ end
305
+ end
306
+
307
+ # Parses the rspec files and reports how many test cases are skipped, and a percentage of
308
+ # the test cases that are executed.
309
+ # returns a hash of test cases, where
310
+ # the key is an integer of the test case ID (as found in test rail), and
311
+ # the value is an instance of TestCase, containing all the skip information and everything.
312
+ def self.parse_specs
313
+
314
+ rspec_examples = []
315
+ spec_files = Dir["regression_spec/**/*_spec.rb"]
316
+ example_count = 0
317
+ skip_count = 0
318
+ file_count = 0
319
+ # parse all the files looking for examples
320
+ spec_files.each do |file|
321
+ file_count += 1
322
+ parser = RSpecParser.new(file)
323
+ parser.parse
324
+ parser.test_cases.each do |tc|
325
+ tc.file = file
326
+ if tc.skip.count > 0
327
+ skip_count += 1
328
+ end
329
+ end
330
+ rspec_examples += parser.test_cases
331
+ # puts "%5d in %s" % [parser.test_cases.count, file]
332
+ example_count += parser.test_cases.count
333
+ end
334
+ puts "total files: #{file_count}"
335
+ puts "total examples: #{example_count}"
336
+ puts "total examples skipped: #{skip_count}"
337
+ puts "total executed: #{example_count - skip_count}"
338
+ puts "total coverage: %.2f %" % [(1.0 - (skip_count / example_count.to_f)) * 100]
339
+
340
+ result = {}
341
+ rspec_examples.each do |tc|
342
+ next if tc.id.nil?
343
+ tc.id.each do |id|
344
+ result[id.to_i] = tc
345
+ end
346
+ end
347
+ result
348
+ end
349
+
350
+ @browsers = { "none" => 0, "chrome" => 1, "firefox" => 2, "ie10" => 3, "ie11" => 4, "safari" => 5 }
351
+ def self.browsers_to_testrail(browser_array)
352
+ result = []
353
+ browser_array.each do |name|
354
+ if name == "allbrowsers"
355
+ result = [1, 2, 3, 5]
356
+ return result
357
+ else
358
+ result << @browsers[name]
359
+ end
360
+ end
361
+ result
362
+ end
363
+
364
+ # Parses all the test cases as found in the rspec files.
365
+ # then pushes information about teach test case up to testrail.
366
+ # This will update 3 fields about each test case:
367
+ # 1. the spec file location
368
+ # 2. if the test is skipped, and on which browser
369
+ # 3. if the test is automated (by default is true)
370
+ def self.push_to_testrail
371
+ rspec_examples = parse_specs
372
+ trclient = TestRailOperations.get_test_rail_client
373
+
374
+ # First set the fields to blank in test rail
375
+ test_cases = TestRailOperations.get_test_rail_cases
376
+ # test_cases.each do |key_id, val_tc|
377
+ # url = "update_case/#{key_id}"
378
+ # data = { "custom_spec_location" => nil, "custom_browser_skip" => [], "custom_automated" => false }
379
+ # trclient.send_post(url, data)
380
+ # end
381
+
382
+ # Then upload the information contain in our rspec files
383
+ rspec_examples.each do |id_key, tc_val|
384
+ if test_cases.key?(id_key)
385
+ url = "update_case/#{id_key}"
386
+ puts "updating test case: "
387
+ browser_skips = browsers_to_testrail(tc_val.skip)
388
+ data = { "custom_spec_location" => tc_val.file, "custom_browser_skip" => browser_skips, "custom_automated" => true }
389
+ trclient.send_post_retry(url, data)
390
+ end
391
+ end
392
+ end
393
+ end
@@ -0,0 +1,46 @@
1
+ require "testrail_client"
2
+
3
+ # =================================================================================
4
+ #
5
+ # API's for Test Rail
6
+ #
7
+ # =================================================================================
8
+
9
+ module TestRail
10
+ class APIClient
11
+ def send_post_retry(uri, data)
12
+ # Gurock api's often deadlocks with errors like this:
13
+ # TestRail API returned HTTP 500 ("Deadlock found when trying to get lock; try restarting transaction")
14
+ # So if they say to retry, then that's what we will do
15
+
16
+ # Please note that the API is rate limited on TestRail Hosted and may throttle requests.
17
+ # TestRail might also return a 429 Too Many Requests response which you are expected to handle.
18
+ # Such a response also includes a Retry-After header indicating how many seconds to wait
19
+ # before you are allowed to submit the next request.
20
+ # http://docs.gurock.com/testrail-api2/introduction #Rate Limit
21
+ response = nil
22
+ # todo: use header [Retry-After] secs
23
+ # for HTTPTooManyRequests 429 error, retry post after either 10s, 30s, 90s or 270s.
24
+ exponential_backoff_seconds = 10
25
+
26
+ 4.times do
27
+ begin
28
+ response = send_post(uri, data)
29
+ break
30
+ rescue TestRail::APIError => e
31
+ if e.message && e.message.match("HTTP 500.*Deadlock")
32
+ sleep 1
33
+ elsif e.message && e.message.match("HTTP 429")
34
+ puts "TestRail rate limited. retrying in #{exponential_backoff_seconds}"
35
+ sleep exponential_backoff_seconds
36
+ exponential_backoff_seconds *= 3
37
+ else
38
+ # Don't retry it, let the exception propagate
39
+ raise
40
+ end
41
+ end
42
+ end
43
+ response
44
+ end
45
+ end
46
+ end