telly 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/bin/telly +1 -38
- data/lib/telly.rb +5 -288
- data/lib/telly/arg_parser.rb +64 -0
- data/lib/telly/test_rail.rb +109 -0
- data/lib/telly/test_rail_teller.rb +304 -0
- data/lib/telly/version.rb +5 -0
- metadata +24 -14
- data/lib/testrail.rb +0 -106
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NTdhZmFlMWNiNjY5NjhkOTMxZWM4ODk4MTUxOGViMjBkMjNiZjAwYw==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
MTkxMjViZTE4ODI4OWJmYmIzNmRkNjFiODFjZTFiYzg4Nzc2N2Q4MQ==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
YzVjYmY5OWIxZTIyOGUwMjk3YzU1MDJlMDYxNDFmNGNiMzNkOWUxOTNkZmE3
|
10
|
+
ODZkN2Q5Y2UyZjM0MzQ1Zjc1MTM1YTY3MzNhZDZhOWE4OTYyYmZiZjAyNzk0
|
11
|
+
NjlmNjlmYWJhMTBhZWQxZjY0MmY2MGExMTY1YzA3YzIwYmQwM2U=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
ZDc0MGI0ZDJmNzg3NDJiYzIwNTE2MWMwODlkYjIxNTlhYjhjYTk2MzQwMDQ0
|
14
|
+
Y2E5MGUyOWFmYTM2NTEyNWViMzBlMmE2NjM1NTRjNmQyYjQ2ZGQ4NTU4MzY2
|
15
|
+
Yjk4OTBkYTNkNDdiMjA4MjkyZDdjYWE3YTFjOTI1ZDM5YmZlMjQ=
|
data/bin/telly
CHANGED
@@ -16,42 +16,5 @@
|
|
16
16
|
# and a test status.""
|
17
17
|
|
18
18
|
require 'telly'
|
19
|
-
require 'optparse'
|
20
19
|
|
21
|
-
|
22
|
-
# @return [Void]
|
23
|
-
def parse_opts
|
24
|
-
options_hash = {}
|
25
|
-
|
26
|
-
optparse = OptionParser.new do|parser|
|
27
|
-
options_hash = {
|
28
|
-
testrun_id: nil,
|
29
|
-
junit_file: nil,
|
30
|
-
}
|
31
|
-
|
32
|
-
parser.on( '-t', '--testrun-id TESTRUN_ID', 'The testrun id' ) do |testrun_id|
|
33
|
-
options_hash[:testrun_id] = testrun_id
|
34
|
-
end
|
35
|
-
|
36
|
-
parser.on( '-j', '--junit-folder JUNIT_FILE', 'Beaker junit file' ) do |junit_file|
|
37
|
-
options_hash[:junit_file] = junit_file
|
38
|
-
end
|
39
|
-
|
40
|
-
parser.on( '-h', '--help', 'Display this screen' ) do
|
41
|
-
puts parser
|
42
|
-
exit
|
43
|
-
end
|
44
|
-
|
45
|
-
parser.parse!
|
46
|
-
|
47
|
-
if not options_hash[:testrun_id] or not options_hash[:junit_file]
|
48
|
-
puts "Error: Missing option(s)"
|
49
|
-
puts parser
|
50
|
-
exit
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
options_hash
|
55
|
-
end
|
56
|
-
|
57
|
-
Telly::main(parse_opts)
|
20
|
+
Telly::TestRailTeller.new.send_to_testrail
|
data/lib/telly.rb
CHANGED
@@ -1,297 +1,14 @@
|
|
1
|
-
|
1
|
+
require 'rubygems' unless defined?(Gem)
|
2
2
|
|
3
|
-
require 'nokogiri'
|
4
|
-
require 'yaml'
|
5
|
-
require 'pp'
|
6
|
-
require 'testrail'
|
7
3
|
|
8
|
-
|
9
|
-
# == telly.rb
|
10
|
-
# This module provides functions to add test results in testrail from a
|
11
|
-
# finished beaker run.
|
12
|
-
#
|
13
|
-
# It takes in a beaker junit file and a TestRail Testrun ID
|
14
|
-
#
|
15
|
-
# It matches the beaker tests with TestRail testcases by looking for the
|
16
|
-
# test case ID in the beaker script. The combination of a test run and a test case
|
17
|
-
# allows the script to add a result for a particular instance of a test case.
|
18
|
-
# In TestRail parlance, this is confusingly called a test.
|
19
|
-
# From the TestRail API docs:
|
20
|
-
# "In TestRail, tests are part of a test run and the test cases are part of the
|
21
|
-
# related test suite. So, when you create a new test run, TestRail creates a test
|
22
|
-
# for each test case found in the test suite of the run. You can therefore think
|
23
|
-
# of a test as an “instance” of a test case which can have test results, comments
|
24
|
-
# and a test status.""
|
25
4
|
module Telly
|
26
5
|
|
27
|
-
|
28
|
-
# script. It's meant to be caught and shown to the user as a warning
|
29
|
-
class MissingTestRailId < StandardError
|
30
|
-
end
|
31
|
-
|
32
|
-
TESTRAIL_URL = 'https://testrail.ops.puppetlabs.net/'
|
33
|
-
CREDENTIALS_FILE = '~/.testrail_credentials.yaml'
|
34
|
-
|
35
|
-
# Used for extracted the test case ID from beaker scripts
|
36
|
-
TESTCASE_ID_REGEX = /.*(?<jira_ticket>\w+-\d+).*[cC](?<testrun_id>\d+)/
|
37
|
-
|
38
|
-
# Testrail Status IDs
|
39
|
-
PASSED = 1
|
40
|
-
BLOCKED = 2
|
41
|
-
FAILED = 5
|
42
|
-
|
43
|
-
def do_stub_test(credentials)
|
44
|
-
api = get_testrail_api(credentials)
|
45
|
-
|
46
|
-
api.send_post
|
47
|
-
end
|
48
|
-
|
49
|
-
|
50
|
-
##################################
|
51
|
-
# Main
|
52
|
-
##################################
|
53
|
-
|
54
|
-
# Run the importer
|
55
|
-
#
|
56
|
-
# @param [Hash] An optparse object
|
57
|
-
#
|
58
|
-
# @return [Void]
|
59
|
-
#
|
60
|
-
# @example password = Telly::main(parse_opts)
|
61
|
-
def Telly.main(options)
|
62
|
-
# Get pass/fail/skips from junit file
|
63
|
-
results = load_junit_results(options[:junit_file])
|
64
|
-
|
65
|
-
puts "Run results:"
|
66
|
-
puts "#{results[:passes].length} Passing"
|
67
|
-
puts "#{results[:failures].length} Failing or Erroring"
|
68
|
-
puts "#{results[:skips].length} Skipped"
|
69
|
-
|
70
|
-
# Set results in testrail
|
71
|
-
bad_results = set_testrail_results(results, options[:junit_file], options[:testrun_id])
|
72
|
-
|
73
|
-
# Print error messages
|
74
|
-
if not bad_results.empty?
|
75
|
-
puts "Error: There were problems processing these test scripts:"
|
76
|
-
bad_results.each do |test_script, error|
|
77
|
-
puts "#{test_script}:\n\t#{error}"
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
|
83
|
-
##################################
|
84
|
-
# TestRail API
|
85
|
-
##################################
|
86
|
-
|
87
|
-
# Load testrail credentials from file
|
88
|
-
#
|
89
|
-
# @return [Hash] Contains testrail_username and testrail_password
|
90
|
-
#
|
91
|
-
# @example password = load_credentials()["testrail_password"]
|
92
|
-
def Telly.load_credentials(credentials_file)
|
6
|
+
%w( test_rail_teller test_rail version arg_parser ).each do |lib|
|
93
7
|
begin
|
94
|
-
|
95
|
-
rescue
|
96
|
-
|
97
|
-
puts "Create #{credentials_file} with the following:"
|
98
|
-
puts "testrail_username: your.username\ntestrail_password: yourpassword"
|
99
|
-
|
100
|
-
exit
|
101
|
-
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
|
106
|
-
# Returns a testrail API object that talks to testrail
|
107
|
-
#
|
108
|
-
# @param [Hash] credentials A hash containing at least two keys, testrail_username and testrail_password
|
109
|
-
#
|
110
|
-
# @return [TestRail::APIClient] The API object for talking to TestRail
|
111
|
-
#
|
112
|
-
# @example api = get_testrail_api(load_credentials)
|
113
|
-
def Telly.get_testrail_api(credentials)
|
114
|
-
client = TestRail::APIClient.new(TESTRAIL_URL)
|
115
|
-
client.user = credentials["testrail_username"]
|
116
|
-
client.password = credentials["testrail_password"]
|
117
|
-
|
118
|
-
return client
|
119
|
-
end
|
120
|
-
|
121
|
-
# Sets the results in testrail.
|
122
|
-
# Tests that have testrail API exceptions are kept track of in bad_results
|
123
|
-
#
|
124
|
-
# @param [Hash] results A hash of lists of xml objects from the junit output file.
|
125
|
-
# @param [String] junit_file The path to the junit xml file
|
126
|
-
# Needed for determining the path of the test file in add_failure, etc
|
127
|
-
# @param [String] testrun_id The TestRail test run ID
|
128
|
-
#
|
129
|
-
# @return [Void]
|
130
|
-
#
|
131
|
-
def Telly.set_testrail_results(results, junit_file, testrun_id)
|
132
|
-
credentials = load_credentials(CREDENTIALS_FILE)
|
133
|
-
api = get_testrail_api(credentials)
|
134
|
-
|
135
|
-
# Results that couldn't be set in testrail for some reason
|
136
|
-
bad_results = {}
|
137
|
-
|
138
|
-
# passes
|
139
|
-
results[:passes].each do |junit_result|
|
140
|
-
begin
|
141
|
-
submit_result(api, PASSED, junit_result, junit_file, testrun_id)
|
142
|
-
rescue MissingTestRailId, TestRail::APIError => e
|
143
|
-
bad_results[junit_result[:name]] = e.message
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
# Failures
|
148
|
-
results[:failures].each do |junit_result|
|
149
|
-
begin
|
150
|
-
submit_result(api, FAILED, junit_result, junit_file, testrun_id)
|
151
|
-
rescue MissingTestRailId, TestRail::APIError => e
|
152
|
-
bad_results[junit_result[:name]] = e.message
|
153
|
-
end
|
8
|
+
require "telly/#{lib}"
|
9
|
+
rescue LoadError
|
10
|
+
require File.expand_path(File.join(File.dirname(__FILE__), 'telly', lib))
|
154
11
|
end
|
155
|
-
|
156
|
-
# Skips
|
157
|
-
results[:skips].each do |junit_result|
|
158
|
-
begin
|
159
|
-
submit_result(api, BLOCKED, junit_result, junit_file, testrun_id)
|
160
|
-
rescue MissingTestRailId, TestRail::APIError => e
|
161
|
-
bad_results[junit_result[:name]] = e.message
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
return bad_results
|
166
|
-
end
|
167
|
-
|
168
|
-
# Submits a test result to TestRail
|
169
|
-
#
|
170
|
-
# @param [TestRail::APIClient] api TestRail API object
|
171
|
-
# @param [int] status The testrail status to set
|
172
|
-
# @param [Nokogiri::XML::Element] junit_result The nokogiri node that holds the junit result
|
173
|
-
# @param [String] junit_file Path to the junit file the test result originated from
|
174
|
-
# @param [String] testrun_id The testrun ID
|
175
|
-
#
|
176
|
-
# @return [Void]
|
177
|
-
#
|
178
|
-
# @raise [TestRail::APIError] When there is a problem with the API request, testrail raises
|
179
|
-
# this exception. Should be caught for error reporting
|
180
|
-
#
|
181
|
-
# @example submit_result(api, BLOCKED, junit_result, junit_file, testrun_id)
|
182
|
-
def Telly.submit_result(api, status, junit_result, junit_file, testrun_id)
|
183
|
-
test_file_path = beaker_test_path(junit_file, junit_result)
|
184
|
-
|
185
|
-
puts junit_result.class
|
186
|
-
testcase_id = testcase_id_from_beaker_script(test_file_path)
|
187
|
-
|
188
|
-
time_elapsed = make_testrail_time(junit_result[:time])
|
189
|
-
|
190
|
-
# Make appropriate comment for testrail
|
191
|
-
case status
|
192
|
-
when FAILED
|
193
|
-
error_message = junit_result.xpath('./failure').first[:message]
|
194
|
-
testrail_comment = "Failed with message:\n#{error_message}"
|
195
|
-
when BLOCKED
|
196
|
-
skip_message = junit_result.xpath('system-out').first.text
|
197
|
-
testrail_comment = "Skipped with message:\n#{skip_message}"
|
198
|
-
else
|
199
|
-
testrail_comment = "Passed"
|
200
|
-
end
|
201
|
-
|
202
|
-
puts "\nSetting result for test case: #{testcase_id}"
|
203
|
-
puts "Adding comment:\n#{testrail_comment}"
|
204
|
-
|
205
|
-
api.send_post("add_result_for_case/#{testrun_id}/#{testcase_id}",
|
206
|
-
{
|
207
|
-
status_id: status,
|
208
|
-
comment: testrail_comment,
|
209
|
-
elapsed: time_elapsed,
|
210
|
-
}
|
211
|
-
)
|
212
|
-
end
|
213
|
-
|
214
|
-
|
215
|
-
# Returns a string that testrail accepts as an elapsed time
|
216
|
-
# Input from beaker is a float in seconds, so it rounds it to the
|
217
|
-
# nearest second, and adds an 's' at the end
|
218
|
-
#
|
219
|
-
# Testrail throws an exception if it gets "0s", so it returns a
|
220
|
-
# minimum of "1s"
|
221
|
-
#
|
222
|
-
# @param [String] seconds_string A string that contains only a number, the elapsed time of a test
|
223
|
-
#
|
224
|
-
# @return [String] The elapsed time of the test run, rounded and with an 's' appended
|
225
|
-
#
|
226
|
-
# @example puts make_testrail_time("2.34") # "2s"
|
227
|
-
def Telly.make_testrail_time(seconds_string)
|
228
|
-
# If time is 0, make it 1
|
229
|
-
rounded_time = [seconds_string.to_f.round, 1].max
|
230
|
-
# Test duration
|
231
|
-
time_elapsed = "#{rounded_time}s"
|
232
|
-
|
233
|
-
return time_elapsed
|
234
|
-
end
|
235
|
-
|
236
|
-
|
237
|
-
##################################
|
238
|
-
# Junit and Beaker file functions
|
239
|
-
##################################
|
240
|
-
|
241
|
-
# Loads the results of a beaker run.
|
242
|
-
# Returns hash of failures, passes, and skips that each hold a list of
|
243
|
-
# junit xml objects
|
244
|
-
#
|
245
|
-
# @param [String] junit_file Path to a junit xml file
|
246
|
-
#
|
247
|
-
# @return [Hash] A hash containing xml objects for the failures, skips, and passes
|
248
|
-
#
|
249
|
-
# @example load_junit_results("~/junit/latest/beaker_junit.xml")
|
250
|
-
def Telly.load_junit_results(junit_file)
|
251
|
-
junit_doc = Nokogiri::XML(File.read(junit_file))
|
252
|
-
|
253
|
-
failures = junit_doc.xpath('//testcase[failure]')
|
254
|
-
skips = junit_doc.xpath('//testcase[skip]')
|
255
|
-
passes = junit_doc.xpath('//testcase[not(failure) and not(skip)]')
|
256
|
-
|
257
|
-
return {failures: failures, skips: skips, passes: passes}
|
258
|
-
end
|
259
|
-
|
260
|
-
|
261
|
-
# Extracts the test case id from the test script
|
262
|
-
#
|
263
|
-
# @param [String] beaker_file Path to a beaker test script
|
264
|
-
#
|
265
|
-
# @return [String] The test case ID
|
266
|
-
#
|
267
|
-
# @example testcase_id_from_beaker_script("~/tests/test_the_things.rb") # 1234
|
268
|
-
def Telly.testcase_id_from_beaker_script(beaker_file)
|
269
|
-
# Find first matching line
|
270
|
-
match = File.readlines(beaker_file).map { |line| line.match(TESTCASE_ID_REGEX) }.compact.first
|
271
|
-
|
272
|
-
raise MissingTestRailId, 'Testcase ID could not be found in file' if match.nil?
|
273
|
-
|
274
|
-
match[:testrun_id]
|
275
|
-
end
|
276
|
-
|
277
|
-
|
278
|
-
# Calculates the path to a beaker test file by combining the junit file path
|
279
|
-
# with the test name from the junit results.
|
280
|
-
# Makes the assumption that junit folder that beaker creates will always be
|
281
|
-
# 2 directories up from the beaker script base directory.
|
282
|
-
# TODO somewhat hacky, maybe a config/command line option
|
283
|
-
#
|
284
|
-
# @param [String] junit_file_path Path to a junit xml file
|
285
|
-
# @param [String] junit_result Path to a junit xml file
|
286
|
-
#
|
287
|
-
# @return [String] The path to the beaker script from the junit test result
|
288
|
-
#
|
289
|
-
# @example load_junit_results("~/junit/latest/beaker_junit.xml")
|
290
|
-
def Telly.beaker_test_path(junit_file_path, junit_result)
|
291
|
-
beaker_folder_path = junit_result[:classname]
|
292
|
-
test_filename = junit_result[:name]
|
293
|
-
|
294
|
-
File.join(File.dirname(junit_file_path), "../../", beaker_folder_path, test_filename)
|
295
12
|
end
|
296
13
|
|
297
14
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
|
3
|
+
module Telly
|
4
|
+
class ArgParser
|
5
|
+
|
6
|
+
VERSION_STRING =
|
7
|
+
"
|
8
|
+
__o_____
|
9
|
+
()/O\\___()
|
10
|
+
`-\\\\---' TELLY
|
11
|
+
%s!
|
12
|
+
__\\/__
|
13
|
+
| .... |
|
14
|
+
| .... |
|
15
|
+
------
|
16
|
+
"
|
17
|
+
|
18
|
+
# Parses the command line options
|
19
|
+
# @return [Void]
|
20
|
+
def self.parse_opts
|
21
|
+
options_hash = {}
|
22
|
+
|
23
|
+
optparse = OptionParser.new do|parser|
|
24
|
+
options_hash = {
|
25
|
+
testrun_id: nil,
|
26
|
+
junit_file: nil,
|
27
|
+
}
|
28
|
+
|
29
|
+
parser.on( '-t', '--testrun-id TESTRUN_ID', 'The testrun id' ) do |testrun_id|
|
30
|
+
options_hash[:testrun_id] = testrun_id
|
31
|
+
end
|
32
|
+
|
33
|
+
parser.on( '-j', '--junit-folder JUNIT_FILE', 'Beaker junit file' ) do |junit_file|
|
34
|
+
options_hash[:junit_file] = junit_file
|
35
|
+
end
|
36
|
+
|
37
|
+
parser.on( '-d', '--dry-run', 'Run without API connection to TestRail' ) do
|
38
|
+
options_hash[:dry_run] = true
|
39
|
+
end
|
40
|
+
|
41
|
+
parser.on( '-h', '--help', 'Display this screen' ) do
|
42
|
+
puts parser
|
43
|
+
exit
|
44
|
+
end
|
45
|
+
|
46
|
+
parser.on( '-v', '--version', 'Report the current version number' ) do
|
47
|
+
puts VERSION_STRING % Telly::Version::STRING
|
48
|
+
exit
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
parser.parse!
|
53
|
+
|
54
|
+
if not options_hash[:testrun_id] or not options_hash[:junit_file]
|
55
|
+
puts "Error: Missing option(s)"
|
56
|
+
puts parser
|
57
|
+
exit
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
options_hash
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
#
|
2
|
+
# TestRail API binding for Ruby (API v2, available since TestRail 3.0)
|
3
|
+
#
|
4
|
+
# Learn more:
|
5
|
+
#
|
6
|
+
# http://docs.gurock.com/testrail-api2/start
|
7
|
+
# http://docs.gurock.com/testrail-api2/accessing
|
8
|
+
#
|
9
|
+
# Copyright Gurock Software GmbH. See license.md for details.
|
10
|
+
#
|
11
|
+
|
12
|
+
require 'net/http'
|
13
|
+
require 'net/https'
|
14
|
+
require 'uri'
|
15
|
+
require 'json'
|
16
|
+
|
17
|
+
module Telly
|
18
|
+
module TestRail
|
19
|
+
class APIClient
|
20
|
+
@url = ''
|
21
|
+
@user = ''
|
22
|
+
@password = ''
|
23
|
+
|
24
|
+
attr_accessor :user
|
25
|
+
attr_accessor :password
|
26
|
+
|
27
|
+
def initialize(base_url)
|
28
|
+
if !base_url.match(/\/$/)
|
29
|
+
base_url += '/'
|
30
|
+
end
|
31
|
+
@url = base_url + 'index.php?/api/v2/'
|
32
|
+
end
|
33
|
+
|
34
|
+
#
|
35
|
+
# Send Get
|
36
|
+
#
|
37
|
+
# Issues a GET request (read) against the API and returns the result
|
38
|
+
# (as Ruby hash).
|
39
|
+
#
|
40
|
+
# Arguments:
|
41
|
+
#
|
42
|
+
# uri The API method to call including parameters
|
43
|
+
# (e.g. get_case/1)
|
44
|
+
#
|
45
|
+
def send_get(uri)
|
46
|
+
_send_request('GET', uri, nil)
|
47
|
+
end
|
48
|
+
|
49
|
+
#
|
50
|
+
# Send POST
|
51
|
+
#
|
52
|
+
# Issues a POST request (write) against the API and returns the result
|
53
|
+
# (as Ruby hash).
|
54
|
+
#
|
55
|
+
# Arguments:
|
56
|
+
#
|
57
|
+
# uri The API method to call including parameters
|
58
|
+
# (e.g. add_case/1)
|
59
|
+
# data The data to submit as part of the request (as
|
60
|
+
# Ruby hash, strings must be UTF-8 encoded)
|
61
|
+
#
|
62
|
+
def send_post(uri, data)
|
63
|
+
_send_request('POST', uri, data)
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
def _send_request(method, uri, data)
|
68
|
+
url = URI.parse(@url + uri)
|
69
|
+
if method == 'POST'
|
70
|
+
request = Net::HTTP::Post.new(url.path + '?' + url.query)
|
71
|
+
request.body = JSON.dump(data)
|
72
|
+
else
|
73
|
+
request = Net::HTTP::Get.new(url.path + '?' + url.query)
|
74
|
+
end
|
75
|
+
request.basic_auth(@user, @password)
|
76
|
+
request.add_field('Content-Type', 'application/json')
|
77
|
+
|
78
|
+
conn = Net::HTTP.new(url.host, url.port)
|
79
|
+
if url.scheme == 'https'
|
80
|
+
conn.use_ssl = true
|
81
|
+
conn.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
82
|
+
end
|
83
|
+
response = conn.request(request)
|
84
|
+
|
85
|
+
if response.body && !response.body.empty?
|
86
|
+
result = JSON.parse(response.body)
|
87
|
+
else
|
88
|
+
result = {}
|
89
|
+
end
|
90
|
+
|
91
|
+
if response.code != '200'
|
92
|
+
if result && result.key?('error')
|
93
|
+
error = '"' + result['error'] + '"'
|
94
|
+
else
|
95
|
+
error = 'No additional error message received'
|
96
|
+
end
|
97
|
+
raise APIError.new('TestRail API returned HTTP %s (%s)' %
|
98
|
+
[response.code, error])
|
99
|
+
end
|
100
|
+
|
101
|
+
result
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
class APIError < StandardError
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,304 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'nokogiri'
|
4
|
+
require 'yaml'
|
5
|
+
require 'pp'
|
6
|
+
|
7
|
+
|
8
|
+
# This module provides functions to add test results in testrail from a
|
9
|
+
# finished beaker run.
|
10
|
+
#
|
11
|
+
# It takes in a beaker junit file and a TestRail Testrun ID
|
12
|
+
#
|
13
|
+
# It matches the beaker tests with TestRail testcases by looking for the
|
14
|
+
# test case ID in the beaker script. The combination of a test run and a test case
|
15
|
+
# allows the script to add a result for a particular instance of a test case.
|
16
|
+
# In TestRail parlance, this is confusingly called a test.
|
17
|
+
# From the TestRail API docs:
|
18
|
+
# "In TestRail, tests are part of a test run and the test cases are part of the
|
19
|
+
# related test suite. So, when you create a new test run, TestRail creates a test
|
20
|
+
# for each test case found in the test suite of the run. You can therefore think
|
21
|
+
# of a test as an “instance” of a test case which can have test results, comments
|
22
|
+
# and a test status.""
|
23
|
+
module Telly
|
24
|
+
class TestRailTeller
|
25
|
+
|
26
|
+
# This exception is thrown when a testcase ID can't be found in a beaker
|
27
|
+
# script. It's meant to be caught and shown to the user as a warning
|
28
|
+
class MissingTestRailId < StandardError
|
29
|
+
end
|
30
|
+
|
31
|
+
TESTRAIL_URL = 'https://testrail.ops.puppetlabs.net/'
|
32
|
+
CREDENTIALS_FILE = '~/.testrail_credentials.yaml'
|
33
|
+
|
34
|
+
# Used for extracted the test case ID from beaker scripts
|
35
|
+
TESTCASE_ID_REGEX = /.*(?<jira_ticket>\w+-\d+).*[cC](?<testrun_id>\d+)/
|
36
|
+
|
37
|
+
# Testrail Status IDs
|
38
|
+
PASSED = 1
|
39
|
+
BLOCKED = 2
|
40
|
+
FAILED = 5
|
41
|
+
|
42
|
+
def do_stub_test(credentials)
|
43
|
+
api = get_testrail_api(credentials)
|
44
|
+
|
45
|
+
api.send_post
|
46
|
+
end
|
47
|
+
|
48
|
+
##################################
|
49
|
+
# send_to_testrail
|
50
|
+
##################################
|
51
|
+
|
52
|
+
# Run the importer
|
53
|
+
#
|
54
|
+
# @return [Void]
|
55
|
+
#
|
56
|
+
# @example password = Telly::main(parse_opts)
|
57
|
+
def send_to_testrail()
|
58
|
+
options = ArgParser.parse_opts
|
59
|
+
# Get pass/fail/skips from junit file
|
60
|
+
results = load_junit_results(options[:junit_file])
|
61
|
+
|
62
|
+
puts "Run results:"
|
63
|
+
puts "#{results[:passes].length} Passing"
|
64
|
+
puts "#{results[:failures].length} Failing or Erroring"
|
65
|
+
puts "#{results[:skips].length} Skipped"
|
66
|
+
|
67
|
+
# Set results in testrail
|
68
|
+
bad_results = set_testrail_results(results, options[:junit_file], options[:testrun_id], options[:dry_run])
|
69
|
+
|
70
|
+
# Print error messages
|
71
|
+
if not bad_results.empty?
|
72
|
+
puts "Error: There were problems processing these test scripts:"
|
73
|
+
bad_results.each do |test_script, error|
|
74
|
+
puts "#{test_script}:\n\t#{error}"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
|
80
|
+
##################################
|
81
|
+
# TestRail API
|
82
|
+
##################################
|
83
|
+
|
84
|
+
# Load testrail credentials from file
|
85
|
+
#
|
86
|
+
# @return [Hash] Contains testrail_username and testrail_password
|
87
|
+
#
|
88
|
+
# @example password = load_credentials()["testrail_password"]
|
89
|
+
def load_credentials(credentials_file)
|
90
|
+
begin
|
91
|
+
YAML.load_file(File.expand_path(credentials_file))
|
92
|
+
rescue
|
93
|
+
puts "Error: Could not find #{credentials_file}"
|
94
|
+
puts "Create #{credentials_file} with the following:"
|
95
|
+
puts "testrail_username: your.username\ntestrail_password: yourpassword"
|
96
|
+
|
97
|
+
exit
|
98
|
+
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
|
103
|
+
# Returns a testrail API object that talks to testrail
|
104
|
+
#
|
105
|
+
# @param [Hash] credentials A hash containing at least two keys, testrail_username and testrail_password
|
106
|
+
#
|
107
|
+
# @return [TestRail::APIClient] The API object for talking to TestRail
|
108
|
+
#
|
109
|
+
# @example api = get_testrail_api(load_credentials)
|
110
|
+
def get_testrail_api(credentials)
|
111
|
+
client = TestRail::APIClient.new(TESTRAIL_URL)
|
112
|
+
client.user = credentials["testrail_username"]
|
113
|
+
client.password = credentials["testrail_password"]
|
114
|
+
|
115
|
+
return client
|
116
|
+
end
|
117
|
+
|
118
|
+
# Sets the results in testrail.
|
119
|
+
# Tests that have testrail API exceptions are kept track of in bad_results
|
120
|
+
#
|
121
|
+
# @param [Hash] results A hash of lists of xml objects from the junit output file.
|
122
|
+
# @param [String] junit_file The path to the junit xml file
|
123
|
+
# Needed for determining the path of the test file in add_failure, etc
|
124
|
+
# @param [String] testrun_id The TestRail test run ID
|
125
|
+
# @param [Boolean] dry_run Do not create api connection when dry_run = true
|
126
|
+
#
|
127
|
+
# @return [Void]
|
128
|
+
#
|
129
|
+
def set_testrail_results(results, junit_file, testrun_id, dry_run = false)
|
130
|
+
if not dry_run
|
131
|
+
credentials = load_credentials(CREDENTIALS_FILE)
|
132
|
+
api = get_testrail_api(credentials)
|
133
|
+
else
|
134
|
+
api = get_testrail_api( { "testrail_username" => 'name', "testrail_password" => 'pass' } )
|
135
|
+
end
|
136
|
+
|
137
|
+
|
138
|
+
# Results that couldn't be set in testrail for some reason
|
139
|
+
bad_results = {}
|
140
|
+
|
141
|
+
# passes
|
142
|
+
results[:passes].each do |junit_result|
|
143
|
+
begin
|
144
|
+
submit_result(api, PASSED, junit_result, junit_file, testrun_id, dry_run)
|
145
|
+
rescue MissingTestRailId, TestRail::APIError => e
|
146
|
+
bad_results[junit_result[:name]] = e.message
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
# Failures
|
151
|
+
results[:failures].each do |junit_result|
|
152
|
+
begin
|
153
|
+
submit_result(api, FAILED, junit_result, junit_file, testrun_id, dry_run)
|
154
|
+
rescue MissingTestRailId, TestRail::APIError => e
|
155
|
+
bad_results[junit_result[:name]] = e.message
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
# Skips
|
160
|
+
results[:skips].each do |junit_result|
|
161
|
+
begin
|
162
|
+
submit_result(api, BLOCKED, junit_result, junit_file, testrun_id, dry_run)
|
163
|
+
rescue MissingTestRailId, TestRail::APIError => e
|
164
|
+
bad_results[junit_result[:name]] = e.message
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
return bad_results
|
169
|
+
end
|
170
|
+
|
171
|
+
# Submits a test result to TestRail
|
172
|
+
#
|
173
|
+
# @param [TestRail::APIClient] api TestRail API object
|
174
|
+
# @param [int] status The testrail status to set
|
175
|
+
# @param [Nokogiri::XML::Element] junit_result The nokogiri node that holds the junit result
|
176
|
+
# @param [String] junit_file Path to the junit file the test result originated from
|
177
|
+
# @param [String] testrun_id The testrun ID
|
178
|
+
# @param [Boolean] dry_run When true do not send results to TestRail
|
179
|
+
#
|
180
|
+
# @return [Void]
|
181
|
+
#
|
182
|
+
# @raise [TestRail::APIError] When there is a problem with the API request, testrail raises
|
183
|
+
# this exception. Should be caught for error reporting
|
184
|
+
#
|
185
|
+
# @example submit_result(api, BLOCKED, junit_result, junit_file, testrun_id)
|
186
|
+
def submit_result(api, status, junit_result, junit_file, testrun_id, dry_run = false)
|
187
|
+
test_file_path = beaker_test_path(junit_file, junit_result)
|
188
|
+
|
189
|
+
puts junit_result.class
|
190
|
+
testcase_id = testcase_id_from_beaker_script(test_file_path)
|
191
|
+
|
192
|
+
time_elapsed = make_testrail_time(junit_result[:time])
|
193
|
+
|
194
|
+
# Make appropriate comment for testrail
|
195
|
+
case status
|
196
|
+
when FAILED
|
197
|
+
error_message = junit_result.xpath('./failure').first[:message]
|
198
|
+
testrail_comment = "Failed with message:\n#{error_message}"
|
199
|
+
when BLOCKED
|
200
|
+
skip_message = junit_result.xpath('system-out').first.text
|
201
|
+
testrail_comment = "Skipped with message:\n#{skip_message}"
|
202
|
+
else
|
203
|
+
testrail_comment = "Passed"
|
204
|
+
end
|
205
|
+
|
206
|
+
puts "\nSetting result for test case: #{testcase_id}"
|
207
|
+
puts "Adding comment:\n#{testrail_comment}"
|
208
|
+
|
209
|
+
if not dry_run
|
210
|
+
api.send_post("add_result_for_case/#{testrun_id}/#{testcase_id}",
|
211
|
+
{
|
212
|
+
status_id: status,
|
213
|
+
comment: testrail_comment,
|
214
|
+
elapsed: time_elapsed,
|
215
|
+
}
|
216
|
+
)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
|
221
|
+
# Returns a string that testrail accepts as an elapsed time
|
222
|
+
# Input from beaker is a float in seconds, so it rounds it to the
|
223
|
+
# nearest second, and adds an 's' at the end
|
224
|
+
#
|
225
|
+
# Testrail throws an exception if it gets "0s", so it returns a
|
226
|
+
# minimum of "1s"
|
227
|
+
#
|
228
|
+
# @param [String] seconds_string A string that contains only a number, the elapsed time of a test
|
229
|
+
#
|
230
|
+
# @return [String] The elapsed time of the test run, rounded and with an 's' appended
|
231
|
+
#
|
232
|
+
# @example puts make_testrail_time("2.34") # "2s"
|
233
|
+
def make_testrail_time(seconds_string)
|
234
|
+
# If time is 0, make it 1
|
235
|
+
rounded_time = [seconds_string.to_f.round, 1].max
|
236
|
+
# Test duration
|
237
|
+
time_elapsed = "#{rounded_time}s"
|
238
|
+
|
239
|
+
return time_elapsed
|
240
|
+
end
|
241
|
+
|
242
|
+
|
243
|
+
##################################
|
244
|
+
# Junit and Beaker file functions
|
245
|
+
##################################
|
246
|
+
|
247
|
+
# Loads the results of a beaker run.
|
248
|
+
# Returns hash of failures, passes, and skips that each hold a list of
|
249
|
+
# junit xml objects
|
250
|
+
#
|
251
|
+
# @param [String] junit_file Path to a junit xml file
|
252
|
+
#
|
253
|
+
# @return [Hash] A hash containing xml objects for the failures, skips, and passes
|
254
|
+
#
|
255
|
+
# @example load_junit_results("~/junit/latest/beaker_junit.xml")
|
256
|
+
def load_junit_results(junit_file)
|
257
|
+
junit_doc = Nokogiri::XML(File.read(junit_file))
|
258
|
+
|
259
|
+
failures = junit_doc.xpath('//testcase[failure]')
|
260
|
+
skips = junit_doc.xpath('//testcase[skip]')
|
261
|
+
passes = junit_doc.xpath('//testcase[not(failure) and not(skip)]')
|
262
|
+
|
263
|
+
return {failures: failures, skips: skips, passes: passes}
|
264
|
+
end
|
265
|
+
|
266
|
+
|
267
|
+
# Extracts the test case id from the test script
|
268
|
+
#
|
269
|
+
# @param [String] beaker_file Path to a beaker test script
|
270
|
+
#
|
271
|
+
# @return [String] The test case ID
|
272
|
+
#
|
273
|
+
# @example testcase_id_from_beaker_script("~/tests/test_the_things.rb") # 1234
|
274
|
+
def testcase_id_from_beaker_script(beaker_file)
|
275
|
+
# Find first matching line
|
276
|
+
match = File.readlines(beaker_file).map { |line| line.match(TESTCASE_ID_REGEX) }.compact.first
|
277
|
+
|
278
|
+
raise MissingTestRailId, 'Testcase ID could not be found in file' if match.nil?
|
279
|
+
|
280
|
+
match[:testrun_id]
|
281
|
+
end
|
282
|
+
|
283
|
+
|
284
|
+
# Calculates the path to a beaker test file by combining the junit file path
|
285
|
+
# with the test name from the junit results.
|
286
|
+
# Makes the assumption that junit folder that beaker creates will always be
|
287
|
+
# 2 directories up from the beaker script base directory.
|
288
|
+
# TODO somewhat hacky, maybe a config/command line option
|
289
|
+
#
|
290
|
+
# @param [String] junit_file_path Path to a junit xml file
|
291
|
+
# @param [String] junit_result Path to a junit xml file
|
292
|
+
#
|
293
|
+
# @return [String] The path to the beaker script from the junit test result
|
294
|
+
#
|
295
|
+
# @example load_junit_results("~/junit/latest/beaker_junit.xml")
|
296
|
+
def beaker_test_path(junit_file_path, junit_result)
|
297
|
+
beaker_folder_path = junit_result[:classname]
|
298
|
+
test_filename = junit_result[:name]
|
299
|
+
|
300
|
+
File.join(File.dirname(junit_file_path), "../../../", beaker_folder_path, test_filename)
|
301
|
+
end
|
302
|
+
|
303
|
+
end
|
304
|
+
end
|
metadata
CHANGED
@@ -1,8 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: telly
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.2.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Joe Pinsonault
|
@@ -15,7 +14,6 @@ dependencies:
|
|
15
14
|
- !ruby/object:Gem::Dependency
|
16
15
|
name: webmock
|
17
16
|
requirement: !ruby/object:Gem::Requirement
|
18
|
-
none: false
|
19
17
|
requirements:
|
20
18
|
- - ! '>='
|
21
19
|
- !ruby/object:Gem::Version
|
@@ -23,7 +21,6 @@ dependencies:
|
|
23
21
|
type: :runtime
|
24
22
|
prerelease: false
|
25
23
|
version_requirements: !ruby/object:Gem::Requirement
|
26
|
-
none: false
|
27
24
|
requirements:
|
28
25
|
- - ! '>='
|
29
26
|
- !ruby/object:Gem::Version
|
@@ -31,7 +28,6 @@ dependencies:
|
|
31
28
|
- !ruby/object:Gem::Dependency
|
32
29
|
name: nokogiri
|
33
30
|
requirement: !ruby/object:Gem::Requirement
|
34
|
-
none: false
|
35
31
|
requirements:
|
36
32
|
- - ! '>='
|
37
33
|
- !ruby/object:Gem::Version
|
@@ -39,7 +35,6 @@ dependencies:
|
|
39
35
|
type: :runtime
|
40
36
|
prerelease: false
|
41
37
|
version_requirements: !ruby/object:Gem::Requirement
|
42
|
-
none: false
|
43
38
|
requirements:
|
44
39
|
- - ! '>='
|
45
40
|
- !ruby/object:Gem::Version
|
@@ -47,7 +42,6 @@ dependencies:
|
|
47
42
|
- !ruby/object:Gem::Dependency
|
48
43
|
name: rspec
|
49
44
|
requirement: !ruby/object:Gem::Requirement
|
50
|
-
none: false
|
51
45
|
requirements:
|
52
46
|
- - ! '>='
|
53
47
|
- !ruby/object:Gem::Version
|
@@ -55,7 +49,20 @@ dependencies:
|
|
55
49
|
type: :development
|
56
50
|
prerelease: false
|
57
51
|
version_requirements: !ruby/object:Gem::Requirement
|
58
|
-
|
52
|
+
requirements:
|
53
|
+
- - ! '>='
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rake
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ! '>='
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
59
66
|
requirements:
|
60
67
|
- - ! '>='
|
61
68
|
- !ruby/object:Gem::Version
|
@@ -67,32 +74,35 @@ executables:
|
|
67
74
|
extensions: []
|
68
75
|
extra_rdoc_files: []
|
69
76
|
files:
|
70
|
-
- lib/telly.rb
|
71
|
-
- lib/testrail.rb
|
72
77
|
- bin/telly
|
78
|
+
- lib/telly.rb
|
79
|
+
- lib/telly/arg_parser.rb
|
80
|
+
- lib/telly/test_rail.rb
|
81
|
+
- lib/telly/test_rail_teller.rb
|
82
|
+
- lib/telly/version.rb
|
73
83
|
homepage: http://rubygems.org/gems/telly
|
74
84
|
licenses:
|
75
85
|
- Apache
|
86
|
+
metadata: {}
|
76
87
|
post_install_message:
|
77
88
|
rdoc_options: []
|
78
89
|
require_paths:
|
79
90
|
- lib
|
80
91
|
required_ruby_version: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
92
|
requirements:
|
83
93
|
- - ! '>='
|
84
94
|
- !ruby/object:Gem::Version
|
85
95
|
version: '0'
|
86
96
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
|
-
none: false
|
88
97
|
requirements:
|
89
98
|
- - ! '>='
|
90
99
|
- !ruby/object:Gem::Version
|
91
100
|
version: '0'
|
92
101
|
requirements: []
|
93
102
|
rubyforge_project:
|
94
|
-
rubygems_version:
|
103
|
+
rubygems_version: 2.4.6
|
95
104
|
signing_key:
|
96
|
-
specification_version:
|
105
|
+
specification_version: 4
|
97
106
|
summary: Imports beaker results into TestRail test run results
|
98
107
|
test_files: []
|
108
|
+
has_rdoc:
|
data/lib/testrail.rb
DELETED
@@ -1,106 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# TestRail API binding for Ruby (API v2, available since TestRail 3.0)
|
3
|
-
#
|
4
|
-
# Learn more:
|
5
|
-
#
|
6
|
-
# http://docs.gurock.com/testrail-api2/start
|
7
|
-
# http://docs.gurock.com/testrail-api2/accessing
|
8
|
-
#
|
9
|
-
# Copyright Gurock Software GmbH. See license.md for details.
|
10
|
-
#
|
11
|
-
|
12
|
-
require 'net/http'
|
13
|
-
require 'net/https'
|
14
|
-
require 'uri'
|
15
|
-
require 'json'
|
16
|
-
|
17
|
-
module TestRail
|
18
|
-
class APIClient
|
19
|
-
@url = ''
|
20
|
-
@user = ''
|
21
|
-
@password = ''
|
22
|
-
|
23
|
-
attr_accessor :user
|
24
|
-
attr_accessor :password
|
25
|
-
|
26
|
-
def initialize(base_url)
|
27
|
-
if !base_url.match(/\/$/)
|
28
|
-
base_url += '/'
|
29
|
-
end
|
30
|
-
@url = base_url + 'index.php?/api/v2/'
|
31
|
-
end
|
32
|
-
|
33
|
-
#
|
34
|
-
# Send Get
|
35
|
-
#
|
36
|
-
# Issues a GET request (read) against the API and returns the result
|
37
|
-
# (as Ruby hash).
|
38
|
-
#
|
39
|
-
# Arguments:
|
40
|
-
#
|
41
|
-
# uri The API method to call including parameters
|
42
|
-
# (e.g. get_case/1)
|
43
|
-
#
|
44
|
-
def send_get(uri)
|
45
|
-
_send_request('GET', uri, nil)
|
46
|
-
end
|
47
|
-
|
48
|
-
#
|
49
|
-
# Send POST
|
50
|
-
#
|
51
|
-
# Issues a POST request (write) against the API and returns the result
|
52
|
-
# (as Ruby hash).
|
53
|
-
#
|
54
|
-
# Arguments:
|
55
|
-
#
|
56
|
-
# uri The API method to call including parameters
|
57
|
-
# (e.g. add_case/1)
|
58
|
-
# data The data to submit as part of the request (as
|
59
|
-
# Ruby hash, strings must be UTF-8 encoded)
|
60
|
-
#
|
61
|
-
def send_post(uri, data)
|
62
|
-
_send_request('POST', uri, data)
|
63
|
-
end
|
64
|
-
|
65
|
-
private
|
66
|
-
def _send_request(method, uri, data)
|
67
|
-
url = URI.parse(@url + uri)
|
68
|
-
if method == 'POST'
|
69
|
-
request = Net::HTTP::Post.new(url.path + '?' + url.query)
|
70
|
-
request.body = JSON.dump(data)
|
71
|
-
else
|
72
|
-
request = Net::HTTP::Get.new(url.path + '?' + url.query)
|
73
|
-
end
|
74
|
-
request.basic_auth(@user, @password)
|
75
|
-
request.add_field('Content-Type', 'application/json')
|
76
|
-
|
77
|
-
conn = Net::HTTP.new(url.host, url.port)
|
78
|
-
if url.scheme == 'https'
|
79
|
-
conn.use_ssl = true
|
80
|
-
conn.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
81
|
-
end
|
82
|
-
response = conn.request(request)
|
83
|
-
|
84
|
-
if response.body && !response.body.empty?
|
85
|
-
result = JSON.parse(response.body)
|
86
|
-
else
|
87
|
-
result = {}
|
88
|
-
end
|
89
|
-
|
90
|
-
if response.code != '200'
|
91
|
-
if result && result.key?('error')
|
92
|
-
error = '"' + result['error'] + '"'
|
93
|
-
else
|
94
|
-
error = 'No additional error message received'
|
95
|
-
end
|
96
|
-
raise APIError.new('TestRail API returned HTTP %s (%s)' %
|
97
|
-
[response.code, error])
|
98
|
-
end
|
99
|
-
|
100
|
-
result
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
class APIError < StandardError
|
105
|
-
end
|
106
|
-
end
|