telly 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/bin/telly +1 -38
- data/lib/telly.rb +5 -288
- data/lib/telly/arg_parser.rb +64 -0
- data/lib/telly/test_rail.rb +109 -0
- data/lib/telly/test_rail_teller.rb +304 -0
- data/lib/telly/version.rb +5 -0
- metadata +24 -14
- data/lib/testrail.rb +0 -106
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NTdhZmFlMWNiNjY5NjhkOTMxZWM4ODk4MTUxOGViMjBkMjNiZjAwYw==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
MTkxMjViZTE4ODI4OWJmYmIzNmRkNjFiODFjZTFiYzg4Nzc2N2Q4MQ==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
YzVjYmY5OWIxZTIyOGUwMjk3YzU1MDJlMDYxNDFmNGNiMzNkOWUxOTNkZmE3
|
10
|
+
ODZkN2Q5Y2UyZjM0MzQ1Zjc1MTM1YTY3MzNhZDZhOWE4OTYyYmZiZjAyNzk0
|
11
|
+
NjlmNjlmYWJhMTBhZWQxZjY0MmY2MGExMTY1YzA3YzIwYmQwM2U=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
ZDc0MGI0ZDJmNzg3NDJiYzIwNTE2MWMwODlkYjIxNTlhYjhjYTk2MzQwMDQ0
|
14
|
+
Y2E5MGUyOWFmYTM2NTEyNWViMzBlMmE2NjM1NTRjNmQyYjQ2ZGQ4NTU4MzY2
|
15
|
+
Yjk4OTBkYTNkNDdiMjA4MjkyZDdjYWE3YTFjOTI1ZDM5YmZlMjQ=
|
data/bin/telly
CHANGED
@@ -16,42 +16,5 @@
|
|
16
16
|
# and a test status.""
|
17
17
|
|
18
18
|
require 'telly'
|
19
|
-
require 'optparse'
|
20
19
|
|
21
|
-
|
22
|
-
# @return [Void]
|
23
|
-
def parse_opts
|
24
|
-
options_hash = {}
|
25
|
-
|
26
|
-
optparse = OptionParser.new do|parser|
|
27
|
-
options_hash = {
|
28
|
-
testrun_id: nil,
|
29
|
-
junit_file: nil,
|
30
|
-
}
|
31
|
-
|
32
|
-
parser.on( '-t', '--testrun-id TESTRUN_ID', 'The testrun id' ) do |testrun_id|
|
33
|
-
options_hash[:testrun_id] = testrun_id
|
34
|
-
end
|
35
|
-
|
36
|
-
parser.on( '-j', '--junit-folder JUNIT_FILE', 'Beaker junit file' ) do |junit_file|
|
37
|
-
options_hash[:junit_file] = junit_file
|
38
|
-
end
|
39
|
-
|
40
|
-
parser.on( '-h', '--help', 'Display this screen' ) do
|
41
|
-
puts parser
|
42
|
-
exit
|
43
|
-
end
|
44
|
-
|
45
|
-
parser.parse!
|
46
|
-
|
47
|
-
if not options_hash[:testrun_id] or not options_hash[:junit_file]
|
48
|
-
puts "Error: Missing option(s)"
|
49
|
-
puts parser
|
50
|
-
exit
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
options_hash
|
55
|
-
end
|
56
|
-
|
57
|
-
Telly::main(parse_opts)
|
20
|
+
Telly::TestRailTeller.new.send_to_testrail
|
data/lib/telly.rb
CHANGED
@@ -1,297 +1,14 @@
|
|
1
|
-
|
1
|
+
require 'rubygems' unless defined?(Gem)
|
2
2
|
|
3
|
-
require 'nokogiri'
|
4
|
-
require 'yaml'
|
5
|
-
require 'pp'
|
6
|
-
require 'testrail'
|
7
3
|
|
8
|
-
|
9
|
-
# == telly.rb
|
10
|
-
# This module provides functions to add test results in testrail from a
|
11
|
-
# finished beaker run.
|
12
|
-
#
|
13
|
-
# It takes in a beaker junit file and a TestRail Testrun ID
|
14
|
-
#
|
15
|
-
# It matches the beaker tests with TestRail testcases by looking for the
|
16
|
-
# test case ID in the beaker script. The combination of a test run and a test case
|
17
|
-
# allows the script to add a result for a particular instance of a test case.
|
18
|
-
# In TestRail parlance, this is confusingly called a test.
|
19
|
-
# From the TestRail API docs:
|
20
|
-
# "In TestRail, tests are part of a test run and the test cases are part of the
|
21
|
-
# related test suite. So, when you create a new test run, TestRail creates a test
|
22
|
-
# for each test case found in the test suite of the run. You can therefore think
|
23
|
-
# of a test as an “instance” of a test case which can have test results, comments
|
24
|
-
# and a test status.""
|
25
4
|
module Telly
|
26
5
|
|
27
|
-
|
28
|
-
# script. It's meant to be caught and shown to the user as a warning
|
29
|
-
class MissingTestRailId < StandardError
|
30
|
-
end
|
31
|
-
|
32
|
-
TESTRAIL_URL = 'https://testrail.ops.puppetlabs.net/'
|
33
|
-
CREDENTIALS_FILE = '~/.testrail_credentials.yaml'
|
34
|
-
|
35
|
-
# Used for extracted the test case ID from beaker scripts
|
36
|
-
TESTCASE_ID_REGEX = /.*(?<jira_ticket>\w+-\d+).*[cC](?<testrun_id>\d+)/
|
37
|
-
|
38
|
-
# Testrail Status IDs
|
39
|
-
PASSED = 1
|
40
|
-
BLOCKED = 2
|
41
|
-
FAILED = 5
|
42
|
-
|
43
|
-
def do_stub_test(credentials)
|
44
|
-
api = get_testrail_api(credentials)
|
45
|
-
|
46
|
-
api.send_post
|
47
|
-
end
|
48
|
-
|
49
|
-
|
50
|
-
##################################
|
51
|
-
# Main
|
52
|
-
##################################
|
53
|
-
|
54
|
-
# Run the importer
|
55
|
-
#
|
56
|
-
# @param [Hash] An optparse object
|
57
|
-
#
|
58
|
-
# @return [Void]
|
59
|
-
#
|
60
|
-
# @example password = Telly::main(parse_opts)
|
61
|
-
def Telly.main(options)
|
62
|
-
# Get pass/fail/skips from junit file
|
63
|
-
results = load_junit_results(options[:junit_file])
|
64
|
-
|
65
|
-
puts "Run results:"
|
66
|
-
puts "#{results[:passes].length} Passing"
|
67
|
-
puts "#{results[:failures].length} Failing or Erroring"
|
68
|
-
puts "#{results[:skips].length} Skipped"
|
69
|
-
|
70
|
-
# Set results in testrail
|
71
|
-
bad_results = set_testrail_results(results, options[:junit_file], options[:testrun_id])
|
72
|
-
|
73
|
-
# Print error messages
|
74
|
-
if not bad_results.empty?
|
75
|
-
puts "Error: There were problems processing these test scripts:"
|
76
|
-
bad_results.each do |test_script, error|
|
77
|
-
puts "#{test_script}:\n\t#{error}"
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
|
83
|
-
##################################
|
84
|
-
# TestRail API
|
85
|
-
##################################
|
86
|
-
|
87
|
-
# Load testrail credentials from file
|
88
|
-
#
|
89
|
-
# @return [Hash] Contains testrail_username and testrail_password
|
90
|
-
#
|
91
|
-
# @example password = load_credentials()["testrail_password"]
|
92
|
-
def Telly.load_credentials(credentials_file)
|
6
|
+
%w( test_rail_teller test_rail version arg_parser ).each do |lib|
|
93
7
|
begin
|
94
|
-
|
95
|
-
rescue
|
96
|
-
|
97
|
-
puts "Create #{credentials_file} with the following:"
|
98
|
-
puts "testrail_username: your.username\ntestrail_password: yourpassword"
|
99
|
-
|
100
|
-
exit
|
101
|
-
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
|
106
|
-
# Returns a testrail API object that talks to testrail
|
107
|
-
#
|
108
|
-
# @param [Hash] credentials A hash containing at least two keys, testrail_username and testrail_password
|
109
|
-
#
|
110
|
-
# @return [TestRail::APIClient] The API object for talking to TestRail
|
111
|
-
#
|
112
|
-
# @example api = get_testrail_api(load_credentials)
|
113
|
-
def Telly.get_testrail_api(credentials)
|
114
|
-
client = TestRail::APIClient.new(TESTRAIL_URL)
|
115
|
-
client.user = credentials["testrail_username"]
|
116
|
-
client.password = credentials["testrail_password"]
|
117
|
-
|
118
|
-
return client
|
119
|
-
end
|
120
|
-
|
121
|
-
# Sets the results in testrail.
|
122
|
-
# Tests that have testrail API exceptions are kept track of in bad_results
|
123
|
-
#
|
124
|
-
# @param [Hash] results A hash of lists of xml objects from the junit output file.
|
125
|
-
# @param [String] junit_file The path to the junit xml file
|
126
|
-
# Needed for determining the path of the test file in add_failure, etc
|
127
|
-
# @param [String] testrun_id The TestRail test run ID
|
128
|
-
#
|
129
|
-
# @return [Void]
|
130
|
-
#
|
131
|
-
def Telly.set_testrail_results(results, junit_file, testrun_id)
|
132
|
-
credentials = load_credentials(CREDENTIALS_FILE)
|
133
|
-
api = get_testrail_api(credentials)
|
134
|
-
|
135
|
-
# Results that couldn't be set in testrail for some reason
|
136
|
-
bad_results = {}
|
137
|
-
|
138
|
-
# passes
|
139
|
-
results[:passes].each do |junit_result|
|
140
|
-
begin
|
141
|
-
submit_result(api, PASSED, junit_result, junit_file, testrun_id)
|
142
|
-
rescue MissingTestRailId, TestRail::APIError => e
|
143
|
-
bad_results[junit_result[:name]] = e.message
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
# Failures
|
148
|
-
results[:failures].each do |junit_result|
|
149
|
-
begin
|
150
|
-
submit_result(api, FAILED, junit_result, junit_file, testrun_id)
|
151
|
-
rescue MissingTestRailId, TestRail::APIError => e
|
152
|
-
bad_results[junit_result[:name]] = e.message
|
153
|
-
end
|
8
|
+
require "telly/#{lib}"
|
9
|
+
rescue LoadError
|
10
|
+
require File.expand_path(File.join(File.dirname(__FILE__), 'telly', lib))
|
154
11
|
end
|
155
|
-
|
156
|
-
# Skips
|
157
|
-
results[:skips].each do |junit_result|
|
158
|
-
begin
|
159
|
-
submit_result(api, BLOCKED, junit_result, junit_file, testrun_id)
|
160
|
-
rescue MissingTestRailId, TestRail::APIError => e
|
161
|
-
bad_results[junit_result[:name]] = e.message
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
return bad_results
|
166
|
-
end
|
167
|
-
|
168
|
-
# Submits a test result to TestRail
|
169
|
-
#
|
170
|
-
# @param [TestRail::APIClient] api TestRail API object
|
171
|
-
# @param [int] status The testrail status to set
|
172
|
-
# @param [Nokogiri::XML::Element] junit_result The nokogiri node that holds the junit result
|
173
|
-
# @param [String] junit_file Path to the junit file the test result originated from
|
174
|
-
# @param [String] testrun_id The testrun ID
|
175
|
-
#
|
176
|
-
# @return [Void]
|
177
|
-
#
|
178
|
-
# @raise [TestRail::APIError] When there is a problem with the API request, testrail raises
|
179
|
-
# this exception. Should be caught for error reporting
|
180
|
-
#
|
181
|
-
# @example submit_result(api, BLOCKED, junit_result, junit_file, testrun_id)
|
182
|
-
def Telly.submit_result(api, status, junit_result, junit_file, testrun_id)
|
183
|
-
test_file_path = beaker_test_path(junit_file, junit_result)
|
184
|
-
|
185
|
-
puts junit_result.class
|
186
|
-
testcase_id = testcase_id_from_beaker_script(test_file_path)
|
187
|
-
|
188
|
-
time_elapsed = make_testrail_time(junit_result[:time])
|
189
|
-
|
190
|
-
# Make appropriate comment for testrail
|
191
|
-
case status
|
192
|
-
when FAILED
|
193
|
-
error_message = junit_result.xpath('./failure').first[:message]
|
194
|
-
testrail_comment = "Failed with message:\n#{error_message}"
|
195
|
-
when BLOCKED
|
196
|
-
skip_message = junit_result.xpath('system-out').first.text
|
197
|
-
testrail_comment = "Skipped with message:\n#{skip_message}"
|
198
|
-
else
|
199
|
-
testrail_comment = "Passed"
|
200
|
-
end
|
201
|
-
|
202
|
-
puts "\nSetting result for test case: #{testcase_id}"
|
203
|
-
puts "Adding comment:\n#{testrail_comment}"
|
204
|
-
|
205
|
-
api.send_post("add_result_for_case/#{testrun_id}/#{testcase_id}",
|
206
|
-
{
|
207
|
-
status_id: status,
|
208
|
-
comment: testrail_comment,
|
209
|
-
elapsed: time_elapsed,
|
210
|
-
}
|
211
|
-
)
|
212
|
-
end
|
213
|
-
|
214
|
-
|
215
|
-
# Returns a string that testrail accepts as an elapsed time
|
216
|
-
# Input from beaker is a float in seconds, so it rounds it to the
|
217
|
-
# nearest second, and adds an 's' at the end
|
218
|
-
#
|
219
|
-
# Testrail throws an exception if it gets "0s", so it returns a
|
220
|
-
# minimum of "1s"
|
221
|
-
#
|
222
|
-
# @param [String] seconds_string A string that contains only a number, the elapsed time of a test
|
223
|
-
#
|
224
|
-
# @return [String] The elapsed time of the test run, rounded and with an 's' appended
|
225
|
-
#
|
226
|
-
# @example puts make_testrail_time("2.34") # "2s"
|
227
|
-
def Telly.make_testrail_time(seconds_string)
|
228
|
-
# If time is 0, make it 1
|
229
|
-
rounded_time = [seconds_string.to_f.round, 1].max
|
230
|
-
# Test duration
|
231
|
-
time_elapsed = "#{rounded_time}s"
|
232
|
-
|
233
|
-
return time_elapsed
|
234
|
-
end
|
235
|
-
|
236
|
-
|
237
|
-
##################################
|
238
|
-
# Junit and Beaker file functions
|
239
|
-
##################################
|
240
|
-
|
241
|
-
# Loads the results of a beaker run.
|
242
|
-
# Returns hash of failures, passes, and skips that each hold a list of
|
243
|
-
# junit xml objects
|
244
|
-
#
|
245
|
-
# @param [String] junit_file Path to a junit xml file
|
246
|
-
#
|
247
|
-
# @return [Hash] A hash containing xml objects for the failures, skips, and passes
|
248
|
-
#
|
249
|
-
# @example load_junit_results("~/junit/latest/beaker_junit.xml")
|
250
|
-
def Telly.load_junit_results(junit_file)
|
251
|
-
junit_doc = Nokogiri::XML(File.read(junit_file))
|
252
|
-
|
253
|
-
failures = junit_doc.xpath('//testcase[failure]')
|
254
|
-
skips = junit_doc.xpath('//testcase[skip]')
|
255
|
-
passes = junit_doc.xpath('//testcase[not(failure) and not(skip)]')
|
256
|
-
|
257
|
-
return {failures: failures, skips: skips, passes: passes}
|
258
|
-
end
|
259
|
-
|
260
|
-
|
261
|
-
# Extracts the test case id from the test script
|
262
|
-
#
|
263
|
-
# @param [String] beaker_file Path to a beaker test script
|
264
|
-
#
|
265
|
-
# @return [String] The test case ID
|
266
|
-
#
|
267
|
-
# @example testcase_id_from_beaker_script("~/tests/test_the_things.rb") # 1234
|
268
|
-
def Telly.testcase_id_from_beaker_script(beaker_file)
|
269
|
-
# Find first matching line
|
270
|
-
match = File.readlines(beaker_file).map { |line| line.match(TESTCASE_ID_REGEX) }.compact.first
|
271
|
-
|
272
|
-
raise MissingTestRailId, 'Testcase ID could not be found in file' if match.nil?
|
273
|
-
|
274
|
-
match[:testrun_id]
|
275
|
-
end
|
276
|
-
|
277
|
-
|
278
|
-
# Calculates the path to a beaker test file by combining the junit file path
|
279
|
-
# with the test name from the junit results.
|
280
|
-
# Makes the assumption that junit folder that beaker creates will always be
|
281
|
-
# 2 directories up from the beaker script base directory.
|
282
|
-
# TODO somewhat hacky, maybe a config/command line option
|
283
|
-
#
|
284
|
-
# @param [String] junit_file_path Path to a junit xml file
|
285
|
-
# @param [String] junit_result Path to a junit xml file
|
286
|
-
#
|
287
|
-
# @return [String] The path to the beaker script from the junit test result
|
288
|
-
#
|
289
|
-
# @example load_junit_results("~/junit/latest/beaker_junit.xml")
|
290
|
-
def Telly.beaker_test_path(junit_file_path, junit_result)
|
291
|
-
beaker_folder_path = junit_result[:classname]
|
292
|
-
test_filename = junit_result[:name]
|
293
|
-
|
294
|
-
File.join(File.dirname(junit_file_path), "../../", beaker_folder_path, test_filename)
|
295
12
|
end
|
296
13
|
|
297
14
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
|
3
|
+
module Telly
|
4
|
+
class ArgParser
|
5
|
+
|
6
|
+
VERSION_STRING =
|
7
|
+
"
|
8
|
+
__o_____
|
9
|
+
()/O\\___()
|
10
|
+
`-\\\\---' TELLY
|
11
|
+
%s!
|
12
|
+
__\\/__
|
13
|
+
| .... |
|
14
|
+
| .... |
|
15
|
+
------
|
16
|
+
"
|
17
|
+
|
18
|
+
# Parses the command line options
|
19
|
+
# @return [Void]
|
20
|
+
def self.parse_opts
|
21
|
+
options_hash = {}
|
22
|
+
|
23
|
+
optparse = OptionParser.new do|parser|
|
24
|
+
options_hash = {
|
25
|
+
testrun_id: nil,
|
26
|
+
junit_file: nil,
|
27
|
+
}
|
28
|
+
|
29
|
+
parser.on( '-t', '--testrun-id TESTRUN_ID', 'The testrun id' ) do |testrun_id|
|
30
|
+
options_hash[:testrun_id] = testrun_id
|
31
|
+
end
|
32
|
+
|
33
|
+
parser.on( '-j', '--junit-folder JUNIT_FILE', 'Beaker junit file' ) do |junit_file|
|
34
|
+
options_hash[:junit_file] = junit_file
|
35
|
+
end
|
36
|
+
|
37
|
+
parser.on( '-d', '--dry-run', 'Run without API connection to TestRail' ) do
|
38
|
+
options_hash[:dry_run] = true
|
39
|
+
end
|
40
|
+
|
41
|
+
parser.on( '-h', '--help', 'Display this screen' ) do
|
42
|
+
puts parser
|
43
|
+
exit
|
44
|
+
end
|
45
|
+
|
46
|
+
parser.on( '-v', '--version', 'Report the current version number' ) do
|
47
|
+
puts VERSION_STRING % Telly::Version::STRING
|
48
|
+
exit
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
parser.parse!
|
53
|
+
|
54
|
+
if not options_hash[:testrun_id] or not options_hash[:junit_file]
|
55
|
+
puts "Error: Missing option(s)"
|
56
|
+
puts parser
|
57
|
+
exit
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
options_hash
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
#
|
2
|
+
# TestRail API binding for Ruby (API v2, available since TestRail 3.0)
|
3
|
+
#
|
4
|
+
# Learn more:
|
5
|
+
#
|
6
|
+
# http://docs.gurock.com/testrail-api2/start
|
7
|
+
# http://docs.gurock.com/testrail-api2/accessing
|
8
|
+
#
|
9
|
+
# Copyright Gurock Software GmbH. See license.md for details.
|
10
|
+
#
|
11
|
+
|
12
|
+
require 'net/http'
|
13
|
+
require 'net/https'
|
14
|
+
require 'uri'
|
15
|
+
require 'json'
|
16
|
+
|
17
|
+
module Telly
|
18
|
+
module TestRail
|
19
|
+
class APIClient
|
20
|
+
@url = ''
|
21
|
+
@user = ''
|
22
|
+
@password = ''
|
23
|
+
|
24
|
+
attr_accessor :user
|
25
|
+
attr_accessor :password
|
26
|
+
|
27
|
+
def initialize(base_url)
|
28
|
+
if !base_url.match(/\/$/)
|
29
|
+
base_url += '/'
|
30
|
+
end
|
31
|
+
@url = base_url + 'index.php?/api/v2/'
|
32
|
+
end
|
33
|
+
|
34
|
+
#
|
35
|
+
# Send Get
|
36
|
+
#
|
37
|
+
# Issues a GET request (read) against the API and returns the result
|
38
|
+
# (as Ruby hash).
|
39
|
+
#
|
40
|
+
# Arguments:
|
41
|
+
#
|
42
|
+
# uri The API method to call including parameters
|
43
|
+
# (e.g. get_case/1)
|
44
|
+
#
|
45
|
+
def send_get(uri)
|
46
|
+
_send_request('GET', uri, nil)
|
47
|
+
end
|
48
|
+
|
49
|
+
#
|
50
|
+
# Send POST
|
51
|
+
#
|
52
|
+
# Issues a POST request (write) against the API and returns the result
|
53
|
+
# (as Ruby hash).
|
54
|
+
#
|
55
|
+
# Arguments:
|
56
|
+
#
|
57
|
+
# uri The API method to call including parameters
|
58
|
+
# (e.g. add_case/1)
|
59
|
+
# data The data to submit as part of the request (as
|
60
|
+
# Ruby hash, strings must be UTF-8 encoded)
|
61
|
+
#
|
62
|
+
def send_post(uri, data)
|
63
|
+
_send_request('POST', uri, data)
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
def _send_request(method, uri, data)
|
68
|
+
url = URI.parse(@url + uri)
|
69
|
+
if method == 'POST'
|
70
|
+
request = Net::HTTP::Post.new(url.path + '?' + url.query)
|
71
|
+
request.body = JSON.dump(data)
|
72
|
+
else
|
73
|
+
request = Net::HTTP::Get.new(url.path + '?' + url.query)
|
74
|
+
end
|
75
|
+
request.basic_auth(@user, @password)
|
76
|
+
request.add_field('Content-Type', 'application/json')
|
77
|
+
|
78
|
+
conn = Net::HTTP.new(url.host, url.port)
|
79
|
+
if url.scheme == 'https'
|
80
|
+
conn.use_ssl = true
|
81
|
+
conn.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
82
|
+
end
|
83
|
+
response = conn.request(request)
|
84
|
+
|
85
|
+
if response.body && !response.body.empty?
|
86
|
+
result = JSON.parse(response.body)
|
87
|
+
else
|
88
|
+
result = {}
|
89
|
+
end
|
90
|
+
|
91
|
+
if response.code != '200'
|
92
|
+
if result && result.key?('error')
|
93
|
+
error = '"' + result['error'] + '"'
|
94
|
+
else
|
95
|
+
error = 'No additional error message received'
|
96
|
+
end
|
97
|
+
raise APIError.new('TestRail API returned HTTP %s (%s)' %
|
98
|
+
[response.code, error])
|
99
|
+
end
|
100
|
+
|
101
|
+
result
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
class APIError < StandardError
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,304 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'nokogiri'
|
4
|
+
require 'yaml'
|
5
|
+
require 'pp'
|
6
|
+
|
7
|
+
|
8
|
+
# This module provides functions to add test results in testrail from a
|
9
|
+
# finished beaker run.
|
10
|
+
#
|
11
|
+
# It takes in a beaker junit file and a TestRail Testrun ID
|
12
|
+
#
|
13
|
+
# It matches the beaker tests with TestRail testcases by looking for the
|
14
|
+
# test case ID in the beaker script. The combination of a test run and a test case
|
15
|
+
# allows the script to add a result for a particular instance of a test case.
|
16
|
+
# In TestRail parlance, this is confusingly called a test.
|
17
|
+
# From the TestRail API docs:
|
18
|
+
# "In TestRail, tests are part of a test run and the test cases are part of the
|
19
|
+
# related test suite. So, when you create a new test run, TestRail creates a test
|
20
|
+
# for each test case found in the test suite of the run. You can therefore think
|
21
|
+
# of a test as an “instance” of a test case which can have test results, comments
|
22
|
+
# and a test status.""
|
23
|
+
module Telly
|
24
|
+
class TestRailTeller
|
25
|
+
|
26
|
+
# This exception is thrown when a testcase ID can't be found in a beaker
|
27
|
+
# script. It's meant to be caught and shown to the user as a warning
|
28
|
+
class MissingTestRailId < StandardError
|
29
|
+
end
|
30
|
+
|
31
|
+
TESTRAIL_URL = 'https://testrail.ops.puppetlabs.net/'
|
32
|
+
CREDENTIALS_FILE = '~/.testrail_credentials.yaml'
|
33
|
+
|
34
|
+
# Used for extracted the test case ID from beaker scripts
|
35
|
+
TESTCASE_ID_REGEX = /.*(?<jira_ticket>\w+-\d+).*[cC](?<testrun_id>\d+)/
|
36
|
+
|
37
|
+
# Testrail Status IDs
|
38
|
+
PASSED = 1
|
39
|
+
BLOCKED = 2
|
40
|
+
FAILED = 5
|
41
|
+
|
42
|
+
def do_stub_test(credentials)
|
43
|
+
api = get_testrail_api(credentials)
|
44
|
+
|
45
|
+
api.send_post
|
46
|
+
end
|
47
|
+
|
48
|
+
##################################
|
49
|
+
# send_to_testrail
|
50
|
+
##################################
|
51
|
+
|
52
|
+
# Run the importer
|
53
|
+
#
|
54
|
+
# @return [Void]
|
55
|
+
#
|
56
|
+
# @example password = Telly::main(parse_opts)
|
57
|
+
def send_to_testrail()
|
58
|
+
options = ArgParser.parse_opts
|
59
|
+
# Get pass/fail/skips from junit file
|
60
|
+
results = load_junit_results(options[:junit_file])
|
61
|
+
|
62
|
+
puts "Run results:"
|
63
|
+
puts "#{results[:passes].length} Passing"
|
64
|
+
puts "#{results[:failures].length} Failing or Erroring"
|
65
|
+
puts "#{results[:skips].length} Skipped"
|
66
|
+
|
67
|
+
# Set results in testrail
|
68
|
+
bad_results = set_testrail_results(results, options[:junit_file], options[:testrun_id], options[:dry_run])
|
69
|
+
|
70
|
+
# Print error messages
|
71
|
+
if not bad_results.empty?
|
72
|
+
puts "Error: There were problems processing these test scripts:"
|
73
|
+
bad_results.each do |test_script, error|
|
74
|
+
puts "#{test_script}:\n\t#{error}"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
|
80
|
+
##################################
|
81
|
+
# TestRail API
|
82
|
+
##################################
|
83
|
+
|
84
|
+
# Load testrail credentials from file
|
85
|
+
#
|
86
|
+
# @return [Hash] Contains testrail_username and testrail_password
|
87
|
+
#
|
88
|
+
# @example password = load_credentials()["testrail_password"]
|
89
|
+
def load_credentials(credentials_file)
|
90
|
+
begin
|
91
|
+
YAML.load_file(File.expand_path(credentials_file))
|
92
|
+
rescue
|
93
|
+
puts "Error: Could not find #{credentials_file}"
|
94
|
+
puts "Create #{credentials_file} with the following:"
|
95
|
+
puts "testrail_username: your.username\ntestrail_password: yourpassword"
|
96
|
+
|
97
|
+
exit
|
98
|
+
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
|
103
|
+
# Returns a testrail API object that talks to testrail
|
104
|
+
#
|
105
|
+
# @param [Hash] credentials A hash containing at least two keys, testrail_username and testrail_password
|
106
|
+
#
|
107
|
+
# @return [TestRail::APIClient] The API object for talking to TestRail
|
108
|
+
#
|
109
|
+
# @example api = get_testrail_api(load_credentials)
|
110
|
+
def get_testrail_api(credentials)
|
111
|
+
client = TestRail::APIClient.new(TESTRAIL_URL)
|
112
|
+
client.user = credentials["testrail_username"]
|
113
|
+
client.password = credentials["testrail_password"]
|
114
|
+
|
115
|
+
return client
|
116
|
+
end
|
117
|
+
|
118
|
+
# Sets the results in testrail.
|
119
|
+
# Tests that have testrail API exceptions are kept track of in bad_results
|
120
|
+
#
|
121
|
+
# @param [Hash] results A hash of lists of xml objects from the junit output file.
|
122
|
+
# @param [String] junit_file The path to the junit xml file
|
123
|
+
# Needed for determining the path of the test file in add_failure, etc
|
124
|
+
# @param [String] testrun_id The TestRail test run ID
|
125
|
+
# @param [Boolean] dry_run Do not create api connection when dry_run = true
|
126
|
+
#
|
127
|
+
# @return [Void]
|
128
|
+
#
|
129
|
+
def set_testrail_results(results, junit_file, testrun_id, dry_run = false)
|
130
|
+
if not dry_run
|
131
|
+
credentials = load_credentials(CREDENTIALS_FILE)
|
132
|
+
api = get_testrail_api(credentials)
|
133
|
+
else
|
134
|
+
api = get_testrail_api( { "testrail_username" => 'name', "testrail_password" => 'pass' } )
|
135
|
+
end
|
136
|
+
|
137
|
+
|
138
|
+
# Results that couldn't be set in testrail for some reason
|
139
|
+
bad_results = {}
|
140
|
+
|
141
|
+
# passes
|
142
|
+
results[:passes].each do |junit_result|
|
143
|
+
begin
|
144
|
+
submit_result(api, PASSED, junit_result, junit_file, testrun_id, dry_run)
|
145
|
+
rescue MissingTestRailId, TestRail::APIError => e
|
146
|
+
bad_results[junit_result[:name]] = e.message
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
# Failures
|
151
|
+
results[:failures].each do |junit_result|
|
152
|
+
begin
|
153
|
+
submit_result(api, FAILED, junit_result, junit_file, testrun_id, dry_run)
|
154
|
+
rescue MissingTestRailId, TestRail::APIError => e
|
155
|
+
bad_results[junit_result[:name]] = e.message
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
# Skips
|
160
|
+
results[:skips].each do |junit_result|
|
161
|
+
begin
|
162
|
+
submit_result(api, BLOCKED, junit_result, junit_file, testrun_id, dry_run)
|
163
|
+
rescue MissingTestRailId, TestRail::APIError => e
|
164
|
+
bad_results[junit_result[:name]] = e.message
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
return bad_results
|
169
|
+
end
|
170
|
+
|
171
|
+
# Submits a test result to TestRail
|
172
|
+
#
|
173
|
+
# @param [TestRail::APIClient] api TestRail API object
|
174
|
+
# @param [int] status The testrail status to set
|
175
|
+
# @param [Nokogiri::XML::Element] junit_result The nokogiri node that holds the junit result
|
176
|
+
# @param [String] junit_file Path to the junit file the test result originated from
|
177
|
+
# @param [String] testrun_id The testrun ID
|
178
|
+
# @param [Boolean] dry_run When true do not send results to TestRail
|
179
|
+
#
|
180
|
+
# @return [Void]
|
181
|
+
#
|
182
|
+
# @raise [TestRail::APIError] When there is a problem with the API request, testrail raises
|
183
|
+
# this exception. Should be caught for error reporting
|
184
|
+
#
|
185
|
+
# @example submit_result(api, BLOCKED, junit_result, junit_file, testrun_id)
|
186
|
+
def submit_result(api, status, junit_result, junit_file, testrun_id, dry_run = false)
|
187
|
+
test_file_path = beaker_test_path(junit_file, junit_result)
|
188
|
+
|
189
|
+
puts junit_result.class
|
190
|
+
testcase_id = testcase_id_from_beaker_script(test_file_path)
|
191
|
+
|
192
|
+
time_elapsed = make_testrail_time(junit_result[:time])
|
193
|
+
|
194
|
+
# Make appropriate comment for testrail
|
195
|
+
case status
|
196
|
+
when FAILED
|
197
|
+
error_message = junit_result.xpath('./failure').first[:message]
|
198
|
+
testrail_comment = "Failed with message:\n#{error_message}"
|
199
|
+
when BLOCKED
|
200
|
+
skip_message = junit_result.xpath('system-out').first.text
|
201
|
+
testrail_comment = "Skipped with message:\n#{skip_message}"
|
202
|
+
else
|
203
|
+
testrail_comment = "Passed"
|
204
|
+
end
|
205
|
+
|
206
|
+
puts "\nSetting result for test case: #{testcase_id}"
|
207
|
+
puts "Adding comment:\n#{testrail_comment}"
|
208
|
+
|
209
|
+
if not dry_run
|
210
|
+
api.send_post("add_result_for_case/#{testrun_id}/#{testcase_id}",
|
211
|
+
{
|
212
|
+
status_id: status,
|
213
|
+
comment: testrail_comment,
|
214
|
+
elapsed: time_elapsed,
|
215
|
+
}
|
216
|
+
)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
|
221
|
+
# Returns a string that testrail accepts as an elapsed time
|
222
|
+
# Input from beaker is a float in seconds, so it rounds it to the
|
223
|
+
# nearest second, and adds an 's' at the end
|
224
|
+
#
|
225
|
+
# Testrail throws an exception if it gets "0s", so it returns a
|
226
|
+
# minimum of "1s"
|
227
|
+
#
|
228
|
+
# @param [String] seconds_string A string that contains only a number, the elapsed time of a test
|
229
|
+
#
|
230
|
+
# @return [String] The elapsed time of the test run, rounded and with an 's' appended
|
231
|
+
#
|
232
|
+
# @example puts make_testrail_time("2.34") # "2s"
|
233
|
+
def make_testrail_time(seconds_string)
|
234
|
+
# If time is 0, make it 1
|
235
|
+
rounded_time = [seconds_string.to_f.round, 1].max
|
236
|
+
# Test duration
|
237
|
+
time_elapsed = "#{rounded_time}s"
|
238
|
+
|
239
|
+
return time_elapsed
|
240
|
+
end
|
241
|
+
|
242
|
+
|
243
|
+
##################################
|
244
|
+
# Junit and Beaker file functions
|
245
|
+
##################################
|
246
|
+
|
247
|
+
# Loads the results of a beaker run.
|
248
|
+
# Returns hash of failures, passes, and skips that each hold a list of
|
249
|
+
# junit xml objects
|
250
|
+
#
|
251
|
+
# @param [String] junit_file Path to a junit xml file
|
252
|
+
#
|
253
|
+
# @return [Hash] A hash containing xml objects for the failures, skips, and passes
|
254
|
+
#
|
255
|
+
# @example load_junit_results("~/junit/latest/beaker_junit.xml")
|
256
|
+
def load_junit_results(junit_file)
|
257
|
+
junit_doc = Nokogiri::XML(File.read(junit_file))
|
258
|
+
|
259
|
+
failures = junit_doc.xpath('//testcase[failure]')
|
260
|
+
skips = junit_doc.xpath('//testcase[skip]')
|
261
|
+
passes = junit_doc.xpath('//testcase[not(failure) and not(skip)]')
|
262
|
+
|
263
|
+
return {failures: failures, skips: skips, passes: passes}
|
264
|
+
end
|
265
|
+
|
266
|
+
|
267
|
+
# Extracts the test case id from the test script
|
268
|
+
#
|
269
|
+
# @param [String] beaker_file Path to a beaker test script
|
270
|
+
#
|
271
|
+
# @return [String] The test case ID
|
272
|
+
#
|
273
|
+
# @example testcase_id_from_beaker_script("~/tests/test_the_things.rb") # 1234
|
274
|
+
def testcase_id_from_beaker_script(beaker_file)
|
275
|
+
# Find first matching line
|
276
|
+
match = File.readlines(beaker_file).map { |line| line.match(TESTCASE_ID_REGEX) }.compact.first
|
277
|
+
|
278
|
+
raise MissingTestRailId, 'Testcase ID could not be found in file' if match.nil?
|
279
|
+
|
280
|
+
match[:testrun_id]
|
281
|
+
end
|
282
|
+
|
283
|
+
|
284
|
+
# Calculates the path to a beaker test file by combining the junit file path
|
285
|
+
# with the test name from the junit results.
|
286
|
+
# Makes the assumption that junit folder that beaker creates will always be
|
287
|
+
# 2 directories up from the beaker script base directory.
|
288
|
+
# TODO somewhat hacky, maybe a config/command line option
|
289
|
+
#
|
290
|
+
# @param [String] junit_file_path Path to a junit xml file
|
291
|
+
# @param [String] junit_result Path to a junit xml file
|
292
|
+
#
|
293
|
+
# @return [String] The path to the beaker script from the junit test result
|
294
|
+
#
|
295
|
+
# @example load_junit_results("~/junit/latest/beaker_junit.xml")
|
296
|
+
def beaker_test_path(junit_file_path, junit_result)
|
297
|
+
beaker_folder_path = junit_result[:classname]
|
298
|
+
test_filename = junit_result[:name]
|
299
|
+
|
300
|
+
File.join(File.dirname(junit_file_path), "../../../", beaker_folder_path, test_filename)
|
301
|
+
end
|
302
|
+
|
303
|
+
end
|
304
|
+
end
|
metadata
CHANGED
@@ -1,8 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: telly
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.2.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Joe Pinsonault
|
@@ -15,7 +14,6 @@ dependencies:
|
|
15
14
|
- !ruby/object:Gem::Dependency
|
16
15
|
name: webmock
|
17
16
|
requirement: !ruby/object:Gem::Requirement
|
18
|
-
none: false
|
19
17
|
requirements:
|
20
18
|
- - ! '>='
|
21
19
|
- !ruby/object:Gem::Version
|
@@ -23,7 +21,6 @@ dependencies:
|
|
23
21
|
type: :runtime
|
24
22
|
prerelease: false
|
25
23
|
version_requirements: !ruby/object:Gem::Requirement
|
26
|
-
none: false
|
27
24
|
requirements:
|
28
25
|
- - ! '>='
|
29
26
|
- !ruby/object:Gem::Version
|
@@ -31,7 +28,6 @@ dependencies:
|
|
31
28
|
- !ruby/object:Gem::Dependency
|
32
29
|
name: nokogiri
|
33
30
|
requirement: !ruby/object:Gem::Requirement
|
34
|
-
none: false
|
35
31
|
requirements:
|
36
32
|
- - ! '>='
|
37
33
|
- !ruby/object:Gem::Version
|
@@ -39,7 +35,6 @@ dependencies:
|
|
39
35
|
type: :runtime
|
40
36
|
prerelease: false
|
41
37
|
version_requirements: !ruby/object:Gem::Requirement
|
42
|
-
none: false
|
43
38
|
requirements:
|
44
39
|
- - ! '>='
|
45
40
|
- !ruby/object:Gem::Version
|
@@ -47,7 +42,6 @@ dependencies:
|
|
47
42
|
- !ruby/object:Gem::Dependency
|
48
43
|
name: rspec
|
49
44
|
requirement: !ruby/object:Gem::Requirement
|
50
|
-
none: false
|
51
45
|
requirements:
|
52
46
|
- - ! '>='
|
53
47
|
- !ruby/object:Gem::Version
|
@@ -55,7 +49,20 @@ dependencies:
|
|
55
49
|
type: :development
|
56
50
|
prerelease: false
|
57
51
|
version_requirements: !ruby/object:Gem::Requirement
|
58
|
-
|
52
|
+
requirements:
|
53
|
+
- - ! '>='
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rake
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ! '>='
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
59
66
|
requirements:
|
60
67
|
- - ! '>='
|
61
68
|
- !ruby/object:Gem::Version
|
@@ -67,32 +74,35 @@ executables:
|
|
67
74
|
extensions: []
|
68
75
|
extra_rdoc_files: []
|
69
76
|
files:
|
70
|
-
- lib/telly.rb
|
71
|
-
- lib/testrail.rb
|
72
77
|
- bin/telly
|
78
|
+
- lib/telly.rb
|
79
|
+
- lib/telly/arg_parser.rb
|
80
|
+
- lib/telly/test_rail.rb
|
81
|
+
- lib/telly/test_rail_teller.rb
|
82
|
+
- lib/telly/version.rb
|
73
83
|
homepage: http://rubygems.org/gems/telly
|
74
84
|
licenses:
|
75
85
|
- Apache
|
86
|
+
metadata: {}
|
76
87
|
post_install_message:
|
77
88
|
rdoc_options: []
|
78
89
|
require_paths:
|
79
90
|
- lib
|
80
91
|
required_ruby_version: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
92
|
requirements:
|
83
93
|
- - ! '>='
|
84
94
|
- !ruby/object:Gem::Version
|
85
95
|
version: '0'
|
86
96
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
|
-
none: false
|
88
97
|
requirements:
|
89
98
|
- - ! '>='
|
90
99
|
- !ruby/object:Gem::Version
|
91
100
|
version: '0'
|
92
101
|
requirements: []
|
93
102
|
rubyforge_project:
|
94
|
-
rubygems_version:
|
103
|
+
rubygems_version: 2.4.6
|
95
104
|
signing_key:
|
96
|
-
specification_version:
|
105
|
+
specification_version: 4
|
97
106
|
summary: Imports beaker results into TestRail test run results
|
98
107
|
test_files: []
|
108
|
+
has_rdoc:
|
data/lib/testrail.rb
DELETED
@@ -1,106 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# TestRail API binding for Ruby (API v2, available since TestRail 3.0)
|
3
|
-
#
|
4
|
-
# Learn more:
|
5
|
-
#
|
6
|
-
# http://docs.gurock.com/testrail-api2/start
|
7
|
-
# http://docs.gurock.com/testrail-api2/accessing
|
8
|
-
#
|
9
|
-
# Copyright Gurock Software GmbH. See license.md for details.
|
10
|
-
#
|
11
|
-
|
12
|
-
require 'net/http'
|
13
|
-
require 'net/https'
|
14
|
-
require 'uri'
|
15
|
-
require 'json'
|
16
|
-
|
17
|
-
module TestRail
|
18
|
-
class APIClient
|
19
|
-
@url = ''
|
20
|
-
@user = ''
|
21
|
-
@password = ''
|
22
|
-
|
23
|
-
attr_accessor :user
|
24
|
-
attr_accessor :password
|
25
|
-
|
26
|
-
def initialize(base_url)
|
27
|
-
if !base_url.match(/\/$/)
|
28
|
-
base_url += '/'
|
29
|
-
end
|
30
|
-
@url = base_url + 'index.php?/api/v2/'
|
31
|
-
end
|
32
|
-
|
33
|
-
#
|
34
|
-
# Send Get
|
35
|
-
#
|
36
|
-
# Issues a GET request (read) against the API and returns the result
|
37
|
-
# (as Ruby hash).
|
38
|
-
#
|
39
|
-
# Arguments:
|
40
|
-
#
|
41
|
-
# uri The API method to call including parameters
|
42
|
-
# (e.g. get_case/1)
|
43
|
-
#
|
44
|
-
def send_get(uri)
|
45
|
-
_send_request('GET', uri, nil)
|
46
|
-
end
|
47
|
-
|
48
|
-
#
|
49
|
-
# Send POST
|
50
|
-
#
|
51
|
-
# Issues a POST request (write) against the API and returns the result
|
52
|
-
# (as Ruby hash).
|
53
|
-
#
|
54
|
-
# Arguments:
|
55
|
-
#
|
56
|
-
# uri The API method to call including parameters
|
57
|
-
# (e.g. add_case/1)
|
58
|
-
# data The data to submit as part of the request (as
|
59
|
-
# Ruby hash, strings must be UTF-8 encoded)
|
60
|
-
#
|
61
|
-
def send_post(uri, data)
|
62
|
-
_send_request('POST', uri, data)
|
63
|
-
end
|
64
|
-
|
65
|
-
private
|
66
|
-
def _send_request(method, uri, data)
|
67
|
-
url = URI.parse(@url + uri)
|
68
|
-
if method == 'POST'
|
69
|
-
request = Net::HTTP::Post.new(url.path + '?' + url.query)
|
70
|
-
request.body = JSON.dump(data)
|
71
|
-
else
|
72
|
-
request = Net::HTTP::Get.new(url.path + '?' + url.query)
|
73
|
-
end
|
74
|
-
request.basic_auth(@user, @password)
|
75
|
-
request.add_field('Content-Type', 'application/json')
|
76
|
-
|
77
|
-
conn = Net::HTTP.new(url.host, url.port)
|
78
|
-
if url.scheme == 'https'
|
79
|
-
conn.use_ssl = true
|
80
|
-
conn.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
81
|
-
end
|
82
|
-
response = conn.request(request)
|
83
|
-
|
84
|
-
if response.body && !response.body.empty?
|
85
|
-
result = JSON.parse(response.body)
|
86
|
-
else
|
87
|
-
result = {}
|
88
|
-
end
|
89
|
-
|
90
|
-
if response.code != '200'
|
91
|
-
if result && result.key?('error')
|
92
|
-
error = '"' + result['error'] + '"'
|
93
|
-
else
|
94
|
-
error = 'No additional error message received'
|
95
|
-
end
|
96
|
-
raise APIError.new('TestRail API returned HTTP %s (%s)' %
|
97
|
-
[response.code, error])
|
98
|
-
end
|
99
|
-
|
100
|
-
result
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
class APIError < StandardError
|
105
|
-
end
|
106
|
-
end
|