res 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +96 -0
- data/lib/res.rb +45 -0
- data/lib/res/config.rb +59 -0
- data/lib/res/formatters/ruby_cucumber.rb +284 -0
- data/lib/res/ir.rb +81 -0
- data/lib/res/mappings.rb +23 -0
- data/lib/res/parsers/junit.rb +65 -0
- data/lib/res/reporters/hive.rb +42 -0
- data/lib/res/reporters/test_rail.rb +150 -0
- data/lib/res/reporters/testmine.rb +48 -0
- metadata +113 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 127358ee22371eb6201add12abb4a4aec0534ef7
|
4
|
+
data.tar.gz: 7e4bdc7f8c10691fe03569bf3a8762cde6aaada9
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 934947a0fb56156732515d456b454819146b83ce8e3b7346a2bba55faafe47b54b716093a22aaf2201a3dda3f5fdf1cd7010ad812fab92c30e16a70dc55d89e4
|
7
|
+
data.tar.gz: 0507cb3031cb9301ebc6281d4b45ce9316ad3877b1a4ea53d69351bbaf121abf3674691d5f6d039e8299e9f55bce40799d9baf74038476b77d314c8870063fe3
|
data/README.md
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
# Res
|
2
|
+
|
3
|
+
Res is a collection of formatting tools for making test
|
4
|
+
result reporting easier and more consistent. The concept is that
|
5
|
+
rather than writing formatters over and over again for different testing
|
6
|
+
tools and result repositories, we standardise on one format for test
|
7
|
+
results (an Intermediate Representation) and write a single formatter
|
8
|
+
and submitter for each runner and repository.
|
9
|
+
|
10
|
+
Example: Reporting a testing tool output to testrail
|
11
|
+
|
12
|
+
Formatter ---------> Intermediate --------> Reporter
|
13
|
+
or Parser Representation (TestRail)
|
14
|
+
|
15
|
+
## Frontends
|
16
|
+
|
17
|
+
There are three different types of front end:
|
18
|
+
|
19
|
+
1. Formatters -- libraries that can be loaded into a test runner to generate IR output
|
20
|
+
2. Parsers -- scripts that parse result files into IR after a test runner has completed
|
21
|
+
3. Native -- custom test runners that use IR as their native format
|
22
|
+
|
23
|
+
IR is a json formatted file that captures a hierarchical definition of
|
24
|
+
a test suite alongside results and metadata.
|
25
|
+
|
26
|
+
## Formatters and Parsers
|
27
|
+
You can dump a Res IR results file using a cucumber formatter or parse xunit output into res format.
|
28
|
+
|
29
|
+
## Cucumber
|
30
|
+
|
31
|
+
cucumber -f pretty -f Res::Formatters::RubyCucumber -o './cucumber.res'
|
32
|
+
Note: This cucumber formatter works for cucumber version < 2.0
|
33
|
+
|
34
|
+
## Junit
|
35
|
+
|
36
|
+
./bin/res.rb --junit '/path/to/xunit_result.xml'
|
37
|
+
Note: The Res output of the xunit parser is saved in the current directory
|
38
|
+
|
39
|
+
## Reporters
|
40
|
+
|
41
|
+
There are a number of backend reporters, for taking IR and producing a report from
|
42
|
+
it, or submitting the IR results to a test repository or test management
|
43
|
+
tool.
|
44
|
+
|
45
|
+
If you have a Res IR file, you can submit using a reporter:
|
46
|
+
|
47
|
+
./bin/res.rb --res '/path/to/file.res' --submit REPORTER [... options]
|
48
|
+
|
49
|
+
### Hive
|
50
|
+
|
51
|
+
Hive CI uses a Res reporter for result submission, the api arguments look like this:
|
52
|
+
|
53
|
+
Res.submit_results(
|
54
|
+
:reporter => :hive,
|
55
|
+
:ir => 'path/to/file.res',
|
56
|
+
:url => 'http://hive.local',
|
57
|
+
:job_id => 10723
|
58
|
+
)
|
59
|
+
|
60
|
+
### TestRail
|
61
|
+
|
62
|
+
./bin/res.rb --res '/path/to/file.res' --submit testrail --config-file '/path/to/.test_rail.yaml'
|
63
|
+
|
64
|
+
Our TestRail reporter currently be used to sync a suite with TestRail, and
|
65
|
+
to submit test results against a test run. You will need to create a
|
66
|
+
config file in your project in order for our reporter to know where to sync
|
67
|
+
the tests and put the results.
|
68
|
+
|
69
|
+
Your config file should be called .test_rail.yaml, and should look like this:
|
70
|
+
|
71
|
+
namespace: 'NameSpace'
|
72
|
+
user: 'user@example.com'
|
73
|
+
password: 'passw0rd'
|
74
|
+
project: 'MyProjectName'
|
75
|
+
suite: 'MySuite'
|
76
|
+
run_name: 'RunName' or run_id: '1234'
|
77
|
+
|
78
|
+
### Testmine
|
79
|
+
|
80
|
+
The Testmine reporter is very similar to the TestRail reporter, but doesn't
|
81
|
+
require you to sync your test definitions before you submit a run.
|
82
|
+
|
83
|
+
You need to create a project file in the root of your test code, the file
|
84
|
+
should be called .testmine.yaml, and should look like this:
|
85
|
+
|
86
|
+
testmine_url: 'mytestmineinstance.bbc.co.uk'
|
87
|
+
authentication: 'authenticationkey'
|
88
|
+
project: 'MyProjectName'
|
89
|
+
component: 'Android acceptance'
|
90
|
+
suite: 'MySuiteName'
|
91
|
+
|
92
|
+
## License
|
93
|
+
|
94
|
+
*Res* is available to everyone under the terms of the MIT open source licence. Take a look at the LICENSE file in the code.
|
95
|
+
|
96
|
+
Copyright (c) 2015 BBC
|
data/lib/res.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'res/ir'
|
2
|
+
|
3
|
+
# Res API
|
4
|
+
module Res
|
5
|
+
|
6
|
+
# Report Res IR to a test repository or similar
|
7
|
+
def self.submit_results(args)
|
8
|
+
reporter_class = Res.reporter_class(args[:reporter])
|
9
|
+
reporter = reporter_class.new( args )
|
10
|
+
|
11
|
+
ir = Res::IR.load(args[:ir])
|
12
|
+
|
13
|
+
reporter.submit_results( ir, args )
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.reporter_class(type)
|
17
|
+
case type
|
18
|
+
when :test_rail
|
19
|
+
require 'res/reporters/test_rail'
|
20
|
+
Res::Reporters::TestRail
|
21
|
+
when :hive
|
22
|
+
require 'res/reporters/hive'
|
23
|
+
Res::Reporters::Hive
|
24
|
+
when :testmine
|
25
|
+
require 'res/reporters/testmine'
|
26
|
+
Res::Reporters::Testmine
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.parse_results(args)
|
31
|
+
parser_class = Res.parser_class(args[:parser])
|
32
|
+
parser_class.new(args[:file])
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.parser_class(type)
|
36
|
+
case type
|
37
|
+
when :junit
|
38
|
+
require 'res/parsers/junit'
|
39
|
+
Res::Parsers::Junit
|
40
|
+
else
|
41
|
+
raise "#{type} parser not Implemented"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
data/lib/res/config.rb
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
|
4
|
+
module Res
|
5
|
+
class Config
|
6
|
+
attr_accessor :struct, :required, :optional, :prepend
|
7
|
+
|
8
|
+
def initialize(items, args = {})
|
9
|
+
@required = *items
|
10
|
+
@optional = args[:optional] || []
|
11
|
+
@prepend = args[:pre_env] || ''
|
12
|
+
items = required + optional
|
13
|
+
@struct = Struct.new(*items).new
|
14
|
+
end
|
15
|
+
|
16
|
+
# Load in config -- this can come from three places:
|
17
|
+
# 1. Arguments passed to the initializer
|
18
|
+
# 2. From environment variables
|
19
|
+
# 3. From a config file
|
20
|
+
def process( args = {} )
|
21
|
+
|
22
|
+
config_from_file = {}
|
23
|
+
if args[:config_file]
|
24
|
+
config_from_file = load_from_config(args[:config_file])
|
25
|
+
args.delete(:config_file)
|
26
|
+
end
|
27
|
+
|
28
|
+
missing = []
|
29
|
+
struct.members.each do |item|
|
30
|
+
struct[item] = args[item] || ENV[(prepend + item.to_s).upcase] || config_from_file[item] || nil
|
31
|
+
missing << item if (struct[item].nil? && required.include?(item))
|
32
|
+
end
|
33
|
+
raise "Missing configuration: " + missing.join( ", ") if missing.any?
|
34
|
+
end
|
35
|
+
|
36
|
+
def load_from_config(config_file)
|
37
|
+
config = {}
|
38
|
+
if File.exists?(config_file)
|
39
|
+
config = Res::Config.symbolize_keys(YAML::load(File.open(config_file)))
|
40
|
+
else
|
41
|
+
raise "Couldn't find config file '#{config_file}'"
|
42
|
+
end
|
43
|
+
config
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.symbolize_keys(hash)
|
47
|
+
symbolized_hash = {}
|
48
|
+
hash.each do |key, value|
|
49
|
+
symbolized_hash[key.to_sym] = value
|
50
|
+
end
|
51
|
+
symbolized_hash
|
52
|
+
end
|
53
|
+
|
54
|
+
def method_missing(m, *args, &block)
|
55
|
+
struct.send(m,*args, &block)
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,284 @@
|
|
1
|
+
# Formatter for ruby cucumber
|
2
|
+
|
3
|
+
require 'fileutils'
|
4
|
+
require 'res/ir'
|
5
|
+
require 'cucumber/formatter/io'
|
6
|
+
|
7
|
+
module Res
|
8
|
+
module Formatters
|
9
|
+
class RubyCucumber
|
10
|
+
include FileUtils
|
11
|
+
include ::Cucumber::Formatter::Io
|
12
|
+
|
13
|
+
def initialize(runtime, path_or_io, options)
|
14
|
+
cucumber_version = %x(cucumber --version)
|
15
|
+
@cucumber_version = cucumber_version.gsub("\n","")
|
16
|
+
|
17
|
+
@runtime = runtime
|
18
|
+
@io = ensure_io(path_or_io, "reporter")
|
19
|
+
@options = options
|
20
|
+
@exceptions = []
|
21
|
+
@indent = 0
|
22
|
+
@prefixes = options[:prefixes] || {}
|
23
|
+
@delayed_messages = []
|
24
|
+
@_start_time = Time.now
|
25
|
+
end
|
26
|
+
|
27
|
+
def before_features(features)
|
28
|
+
@_features = []
|
29
|
+
end
|
30
|
+
|
31
|
+
# Once everything has run -- whack it in a ResultIR object and
|
32
|
+
# dump it as json
|
33
|
+
def after_features(features)
|
34
|
+
results = @_features
|
35
|
+
ir = ::Res::IR.new( :started => @_start_time,
|
36
|
+
:finished => Time.now(),
|
37
|
+
:results => results,
|
38
|
+
:type => 'Cucumber' )
|
39
|
+
@io.puts ir.json
|
40
|
+
end
|
41
|
+
|
42
|
+
def before_feature(feature)
|
43
|
+
@_feature = {}
|
44
|
+
@_context = {}
|
45
|
+
@_feature[:started] = Time.now()
|
46
|
+
begin
|
47
|
+
if @cucumber_version.to_f < 1.3.to_f
|
48
|
+
uri = feature.file.to_s
|
49
|
+
else
|
50
|
+
uri = feature.location.to_s
|
51
|
+
end
|
52
|
+
|
53
|
+
hash = RubyCucumber.split_uri( uri )
|
54
|
+
@_feature[:file] = hash[:file]
|
55
|
+
@_feature[:line] = hash[:line]
|
56
|
+
@_feature[:urn] = hash[:urn]
|
57
|
+
rescue
|
58
|
+
@_feature[:uri] = 'unknown'
|
59
|
+
end
|
60
|
+
@_features << @_feature
|
61
|
+
@_context = @_feature
|
62
|
+
end
|
63
|
+
|
64
|
+
def comment_line(comment_line)
|
65
|
+
@_context[:comments] = [] if !@_context[:comments]
|
66
|
+
@_context[:comments] << comment_line
|
67
|
+
end
|
68
|
+
|
69
|
+
def after_tags(tags)
|
70
|
+
end
|
71
|
+
|
72
|
+
def tag_name(tag_name)
|
73
|
+
@_context[:tags] = [] if !@_context[:tag]
|
74
|
+
# Strip @ from tags
|
75
|
+
@_context[:tags] << tag_name[1..-1]
|
76
|
+
end
|
77
|
+
|
78
|
+
# { :type => 'Feature',
|
79
|
+
# :name => 'Feature name',
|
80
|
+
# :description => "As a blah\nAs a blah\n" }
|
81
|
+
def feature_name(keyword, name)
|
82
|
+
@_feature[:type] = "Cucumber::" + keyword.gsub(/\s+/, "")
|
83
|
+
|
84
|
+
lines = name.split("\n")
|
85
|
+
lines = lines.collect { |l| l.strip }
|
86
|
+
|
87
|
+
@_feature[:name] = lines.shift
|
88
|
+
@_feature[:description] = lines.join("\n")
|
89
|
+
end
|
90
|
+
|
91
|
+
def after_feature(feature)
|
92
|
+
@_feature[:finished] = Time.now()
|
93
|
+
end
|
94
|
+
|
95
|
+
def before_feature_element(feature_element)
|
96
|
+
|
97
|
+
@_feature_element = {}
|
98
|
+
@_context = {}
|
99
|
+
@_feature_element[:started] = Time.now
|
100
|
+
begin
|
101
|
+
if @cucumber_version.to_f < 1.3.to_f
|
102
|
+
uri = feature_element.file_colon_line
|
103
|
+
else
|
104
|
+
uri = feature_element.location.to_s
|
105
|
+
end
|
106
|
+
hash = RubyCucumber.split_uri( uri )
|
107
|
+
@_feature_element[:file] = hash[:file]
|
108
|
+
@_feature_element[:line] = hash[:line]
|
109
|
+
@_feature_element[:urn] = hash[:urn]
|
110
|
+
rescue => e
|
111
|
+
@_feature_element[:error] = e.message
|
112
|
+
@_feature_element[:file] = 'unknown'
|
113
|
+
end
|
114
|
+
|
115
|
+
@_feature[:children] = [] if ! @_feature[:children]
|
116
|
+
|
117
|
+
@_feature[:children] << @_feature_element
|
118
|
+
@_context = @_feature_element
|
119
|
+
end
|
120
|
+
|
121
|
+
# After a scenario
|
122
|
+
def after_feature_element(feature_element)
|
123
|
+
@_context = {}
|
124
|
+
|
125
|
+
if feature_element.respond_to? :status
|
126
|
+
@_feature_element[:status] = feature_element.status
|
127
|
+
end
|
128
|
+
@_feature_element[:finished] = Time.now
|
129
|
+
end
|
130
|
+
|
131
|
+
def before_background(background)
|
132
|
+
#@_context[:background] = background
|
133
|
+
end
|
134
|
+
|
135
|
+
def after_background(background)
|
136
|
+
end
|
137
|
+
|
138
|
+
def background_name(keyword, name, file_colon_line, source_indent)
|
139
|
+
end
|
140
|
+
|
141
|
+
# def before_examples_array(examples_array)
|
142
|
+
# @indent = 4
|
143
|
+
# @io.puts
|
144
|
+
# @visiting_first_example_name = true
|
145
|
+
# end
|
146
|
+
#
|
147
|
+
|
148
|
+
def examples_name(keyword, name)
|
149
|
+
# @io.puts unless @visiting_first_example_name
|
150
|
+
# @visiting_first_example_name = false
|
151
|
+
# names = name.strip.empty? ? [name.strip] : name.split("\n")
|
152
|
+
# @io.puts(" #{keyword}: #{names[0]}")
|
153
|
+
# names[1..-1].each {|s| @io.puts " #{s}" } unless names.empty?
|
154
|
+
# @io.flush
|
155
|
+
# @indent = 6
|
156
|
+
# @scenario_indent = 6
|
157
|
+
end
|
158
|
+
#
|
159
|
+
|
160
|
+
def scenario_name(keyword, name, file_colon_line, source_indent)
|
161
|
+
@_context[:type] = "Cucumber::" + keyword.gsub(/\s+/, "")
|
162
|
+
@_context[:name] = name || ''
|
163
|
+
end
|
164
|
+
|
165
|
+
def before_step(step)
|
166
|
+
@_step = {}
|
167
|
+
|
168
|
+
# Background steps can appear totally divorced from scenerios (feature
|
169
|
+
# elements). Need to make sure we're not including them as children
|
170
|
+
# to scenario that don't exist
|
171
|
+
return if @_feature_element && @_feature_element[:finished]
|
172
|
+
|
173
|
+
@_feature_element = {} if !@_feature_element
|
174
|
+
@_feature_element[:children] = [] if !@_feature_element[:children]
|
175
|
+
@_feature_element[:children] << @_step
|
176
|
+
@_context = @_step
|
177
|
+
end
|
178
|
+
|
179
|
+
# def before_step_result(keyword, step_match, multiline_arg, status, exception, source_indent, background, file_colon_line)
|
180
|
+
# @hide_this_step = false
|
181
|
+
# if exception
|
182
|
+
# if @exceptions.include?(exception)
|
183
|
+
# @hide_this_step = true
|
184
|
+
# return
|
185
|
+
# end
|
186
|
+
# @exceptions << exception
|
187
|
+
# end
|
188
|
+
# if status != :failed && @in_background ^ background
|
189
|
+
# @hide_this_step = true
|
190
|
+
# return
|
191
|
+
# end
|
192
|
+
# @status = status
|
193
|
+
# end
|
194
|
+
|
195
|
+
# Argument list changed after cucumber 1.4, hence the *args
|
196
|
+
def step_name(keyword, step_match, status, source_indent, background, *args)
|
197
|
+
|
198
|
+
file_colon_line = args[0] if args[0]
|
199
|
+
|
200
|
+
@_step[:type] = "Cucumber::Step"
|
201
|
+
name = keyword + step_match.format_args(lambda{|param| %{#{param}}})
|
202
|
+
@_step[:name] = name
|
203
|
+
@_step[:status] = status
|
204
|
+
#@_step[:background] = background
|
205
|
+
@_step[:type] = "Cucumber::Step"
|
206
|
+
|
207
|
+
end
|
208
|
+
|
209
|
+
# def doc_string(string)
|
210
|
+
# return if @options[:no_multiline] || @hide_this_step
|
211
|
+
# s = %{"""\n#{string}\n"""}.indent(@indent)
|
212
|
+
# s = s.split("\n").map{|l| l =~ /^\s+$/ ? '' : l}.join("\n")
|
213
|
+
# @io.puts(format_string(s, @current_step.status))
|
214
|
+
# @io.flush
|
215
|
+
# end
|
216
|
+
#
|
217
|
+
|
218
|
+
def exception(exception, status)
|
219
|
+
@_context[:message] = exception.to_s
|
220
|
+
end
|
221
|
+
|
222
|
+
def before_multiline_arg(multiline_arg)
|
223
|
+
# return if @options[:no_multiline] || @hide_this_step
|
224
|
+
# @table = multiline_arg
|
225
|
+
end
|
226
|
+
|
227
|
+
def after_multiline_arg(multiline_arg)
|
228
|
+
@_context[:args] = multiline_arg.to_s.gsub(/\e\[(\d+)m/, '')
|
229
|
+
@_table = nil
|
230
|
+
end
|
231
|
+
|
232
|
+
# Before a scenario outline is encountered
|
233
|
+
def before_outline_table(outline_table)
|
234
|
+
# Scenario outlines appear as children like normal scenarios,
|
235
|
+
# but really we just want to construct normal-looking children
|
236
|
+
# from them
|
237
|
+
@_outlines = @_feature_element[:children]
|
238
|
+
@_table = []
|
239
|
+
end
|
240
|
+
|
241
|
+
def after_outline_table(outline_table)
|
242
|
+
headings = @_table.shift
|
243
|
+
description = @_outlines.collect{ |o| o[:name] }.join("\n") + "\n" + headings[:name]
|
244
|
+
@_feature_element[:children] = @_table
|
245
|
+
@_feature_element[:description] = description
|
246
|
+
end
|
247
|
+
|
248
|
+
def before_table_row(table_row)
|
249
|
+
@_current_table_row = { :type => 'Cucumber::ScenarioOutline::Example' }
|
250
|
+
@_table = [] if !@_table
|
251
|
+
end
|
252
|
+
|
253
|
+
def after_table_row(table_row)
|
254
|
+
if table_row.class == Cucumber::Ast::OutlineTable::ExampleRow
|
255
|
+
@_current_table_row[:name] = table_row.name
|
256
|
+
if table_row.exception
|
257
|
+
@_current_table_row[:message] = table_row.exception.to_s
|
258
|
+
end
|
259
|
+
if table_row.scenario_outline
|
260
|
+
@_current_table_row[:status] = table_row.status
|
261
|
+
end
|
262
|
+
@_current_table_row[:line] = table_row.line
|
263
|
+
@_current_table_row[:urn] = @_feature_element[:file] + ":" + table_row.line.to_s
|
264
|
+
@_table << @_current_table_row
|
265
|
+
end
|
266
|
+
end
|
267
|
+
|
268
|
+
def after_table_cell(cell)
|
269
|
+
end
|
270
|
+
|
271
|
+
def table_cell_value(value, status)
|
272
|
+
@_current_table_row[:children] = [] if !@_current_table_row[:children]
|
273
|
+
@_current_table_row[:children] << { :type => "Cucumber::ScenarioOutline::Parameter",
|
274
|
+
:name => value, :status => status }
|
275
|
+
end
|
276
|
+
|
277
|
+
def self.split_uri(uri)
|
278
|
+
strings = uri.rpartition(/:/)
|
279
|
+
{ :file => strings[0], :line => strings[2].to_i, :urn => uri }
|
280
|
+
end
|
281
|
+
|
282
|
+
end
|
283
|
+
end
|
284
|
+
end
|
data/lib/res/ir.rb
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Res
|
4
|
+
class IR
|
5
|
+
attr_accessor :hash, :results, :type, :start_time, :end_time, :world
|
6
|
+
attr_accessor :project, :suite, :target, :hive_job_id
|
7
|
+
|
8
|
+
def self.load(file)
|
9
|
+
f = File.open file
|
10
|
+
hash = JSON.load(f, nil, :symbolize_names => true )
|
11
|
+
Res::IR.new( hash )
|
12
|
+
end
|
13
|
+
|
14
|
+
# Expects hash of:
|
15
|
+
# :results => { }
|
16
|
+
# :type => test_runner
|
17
|
+
# :start_time => Time the tests started
|
18
|
+
# :end_time => Time they completed
|
19
|
+
def initialize( options = {} )
|
20
|
+
@results = options[:results] or raise "No results data"
|
21
|
+
@type = options[:type] or raise "No type provided (e.g. 'Cucumber')"
|
22
|
+
@started = options[:started] or raise "Need to provide a start time"
|
23
|
+
@finished = options[:finished] or raise "Need to provide an end time"
|
24
|
+
end
|
25
|
+
|
26
|
+
# Dump as json
|
27
|
+
def json
|
28
|
+
hash = {
|
29
|
+
:started => @started,
|
30
|
+
:finished => @finished,
|
31
|
+
:results => @results,
|
32
|
+
:type => @type }
|
33
|
+
|
34
|
+
# Merge in the world information if it's available
|
35
|
+
hash[:world] = world if world
|
36
|
+
hash[:hive_job_id] = hive_job_id if hive_job_id
|
37
|
+
|
38
|
+
JSON.pretty_generate( hash )
|
39
|
+
end
|
40
|
+
|
41
|
+
# Pluck out the actual test nodes from the contexts
|
42
|
+
def tests
|
43
|
+
IR.find_tests(results).flatten
|
44
|
+
end
|
45
|
+
|
46
|
+
def count(status)
|
47
|
+
tests.count { |t| t[:status].to_sym == status.to_sym }
|
48
|
+
end
|
49
|
+
|
50
|
+
# Returns a simple array of test information
|
51
|
+
# [ { :name => 'test1', :urn => 'file/tests.t:32', :status => 'passed', :time => 12.04 },
|
52
|
+
# { :name => 'test2', :urn => 'file/tests.t:36', :status => 'failed', :time => } ]
|
53
|
+
def flat_format
|
54
|
+
self.tests.collect do |t|
|
55
|
+
{ :name => t[:name],
|
56
|
+
:urn => t[:urn],
|
57
|
+
:status => t[:status] }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
# Recursive function for retrieving test nodes
|
64
|
+
def self.find_tests(nodes)
|
65
|
+
tests = []
|
66
|
+
nodes.each do |n|
|
67
|
+
if IR.is_a_test?(n)
|
68
|
+
tests << n
|
69
|
+
elsif n[:children]
|
70
|
+
tests << IR.find_tests(n[:children])
|
71
|
+
end
|
72
|
+
end
|
73
|
+
tests
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.is_a_test?(node)
|
77
|
+
!node[:status].nil?
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
end
|
data/lib/res/mappings.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
module Res
|
2
|
+
class Mappings
|
3
|
+
|
4
|
+
attr_accessor :context, :case, :type
|
5
|
+
|
6
|
+
def initialize(type)
|
7
|
+
@type = type
|
8
|
+
set_type
|
9
|
+
end
|
10
|
+
|
11
|
+
def set_type
|
12
|
+
case @type
|
13
|
+
when "Junit"
|
14
|
+
@context = ["JUnit::testsuite", "JUnit::testsuites"]
|
15
|
+
@case = ["JUnit::testcase"]
|
16
|
+
when "Cucumber"
|
17
|
+
@context = ["Cucumber::Feature", "Cucumber::ScenarioOutline"]
|
18
|
+
@case = ["Cucumber::Scenario"]
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'ox'
|
2
|
+
require 'json'
|
3
|
+
require 'res/ir'
|
4
|
+
|
5
|
+
module Res
|
6
|
+
module Parsers
|
7
|
+
class Junit
|
8
|
+
attr_accessor :io
|
9
|
+
|
10
|
+
def initialize(junit_xml)
|
11
|
+
file = File.open(junit_xml, "rb")
|
12
|
+
begin
|
13
|
+
junit = Ox.parse(file.read)
|
14
|
+
rescue Ox::ParseError => e
|
15
|
+
raise "Invalid xunit XML format. Error: #{e}"
|
16
|
+
end
|
17
|
+
file.close
|
18
|
+
result = Array.new
|
19
|
+
result = attach_suite(junit)
|
20
|
+
ir = ::Res::IR.new( :type => 'Junit',
|
21
|
+
:started => "",
|
22
|
+
:finished => Time.now(),
|
23
|
+
:results => result
|
24
|
+
)
|
25
|
+
@io = File.open("./junit.res", "w")
|
26
|
+
@io.puts ir.json
|
27
|
+
@io.close
|
28
|
+
end
|
29
|
+
|
30
|
+
def attach_cases(node)
|
31
|
+
testcase = Hash.new
|
32
|
+
testcase["type"] = "JUnit::" + node.value
|
33
|
+
testcase["name"] = node.attributes[:name]
|
34
|
+
testcase["classname"] = node.attributes[:classname] if testcase["classname"] != nil
|
35
|
+
testcase["duration"] = node.attributes[:time]
|
36
|
+
testcase["status"] = "passed"
|
37
|
+
if node.nodes[0] != nil
|
38
|
+
testcase["status"] = "failed" if node.nodes[0].value == "failure" or node.nodes[0].value == "error"
|
39
|
+
testcase["status"] = "notrun" if node.nodes[0].value == "skipped"
|
40
|
+
end
|
41
|
+
testcase
|
42
|
+
end
|
43
|
+
|
44
|
+
def attach_suite(component)
|
45
|
+
suite = Array.new
|
46
|
+
index = 0
|
47
|
+
component.nodes.each do |node|
|
48
|
+
if node.value == "testcase"
|
49
|
+
suite[index] = Hash.new
|
50
|
+
suite[index] = attach_cases(node)
|
51
|
+
else
|
52
|
+
suite[index] = Hash.new
|
53
|
+
suite[index]["type"] = "JUnit::" + node.value
|
54
|
+
suite[index]["name"] = node.attributes[:name]
|
55
|
+
suite[index]["classname"] = node.attributes[:classname] if suite[index]["classname"] != nil
|
56
|
+
suite[index]["children"] = attach_suite(node)
|
57
|
+
end # if
|
58
|
+
index += 1
|
59
|
+
end # each
|
60
|
+
suite
|
61
|
+
end # def attach_suite
|
62
|
+
end # class JUnit
|
63
|
+
end # class Parsers
|
64
|
+
end # class Res
|
65
|
+
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'hive/messages'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module Res
|
5
|
+
module Reporters
|
6
|
+
class Hive
|
7
|
+
|
8
|
+
def initialize( args = {} )
|
9
|
+
|
10
|
+
if args[:url]
|
11
|
+
::Hive::Messages.configure do |config|
|
12
|
+
config.base_path = args[:url]
|
13
|
+
config.pem_file = args[:cert]
|
14
|
+
config.ssl_verify_mode = args[:ssl_verify_mode]
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
def submit_results(ir, args)
|
21
|
+
|
22
|
+
# Still include count summaries for backward compatability
|
23
|
+
params = { :failed_count => ir.count(:failed),
|
24
|
+
:passed_count => ir.count(:passed),
|
25
|
+
:errored_count => ir.count(:errored),
|
26
|
+
:running_count => ir.count(:running),
|
27
|
+
:result_details => ir.flat_format.to_json }
|
28
|
+
|
29
|
+
hive_job = ::Hive::Messages::Job.new(:job_id => args[:job_id])
|
30
|
+
begin
|
31
|
+
update_response = hive_job.update_results(params)
|
32
|
+
#log "Hive results update response: #{update_response}"
|
33
|
+
rescue
|
34
|
+
#log "Hive::Messages couldn't hit the hive, retrying once more"
|
35
|
+
update_response = hive_job.update_results(params)
|
36
|
+
#log "Hive results update response: #{update_response}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,150 @@
|
|
1
|
+
require 'test_rail'
|
2
|
+
require 'res/ir'
|
3
|
+
require 'res/mappings'
|
4
|
+
require 'res/config'
|
5
|
+
|
6
|
+
module Res
|
7
|
+
module Reporters
|
8
|
+
class TestRail
|
9
|
+
|
10
|
+
attr_accessor :ir, :case_status, :config, :project, :suite
|
11
|
+
def initialize(args)
|
12
|
+
@url = args[:url]
|
13
|
+
@config = Res::Config.new([:user, :password, :namespace, :project, :suite], :optional => [:run_id, :run_name], :pre_env => 'test_rail')
|
14
|
+
config.process(args)
|
15
|
+
|
16
|
+
@case_status = {}
|
17
|
+
@io = File.new("./.test_rail_reporter.log","w+")
|
18
|
+
|
19
|
+
test_rail_project = config.project
|
20
|
+
@suite_name = config.suite
|
21
|
+
|
22
|
+
begin
|
23
|
+
@project = tr.find_project(:name => test_rail_project)
|
24
|
+
rescue
|
25
|
+
@io.puts "Project: #{test_rail_project} could not be found"
|
26
|
+
@io.puts " Please create a project first to contain test suite #{@suite_name}"
|
27
|
+
@io.close
|
28
|
+
return "Project #{test_rail_project} could not be found"
|
29
|
+
end
|
30
|
+
end # initialize
|
31
|
+
|
32
|
+
# Creates a new suite within testrail
|
33
|
+
def sync_tests(args)
|
34
|
+
@ir = args[:ir]
|
35
|
+
@mappings = Res::Mappings.new(@ir.type)
|
36
|
+
|
37
|
+
suite = @project.find_or_create_suite(:name => @suite_name, :id => @project.id)
|
38
|
+
@io.puts "Syncing Suite"
|
39
|
+
|
40
|
+
i = 0
|
41
|
+
while i < @ir.results.count
|
42
|
+
section = suite.find_or_create_section(:project_id => @project.id, :suite_id => suite.id, :name => @ir.results[i][:name])
|
43
|
+
create_suite(@ir.results[i], @project.id, suite, section)
|
44
|
+
i += 1
|
45
|
+
end # while
|
46
|
+
@synced = true
|
47
|
+
@io.puts "> Sync Successful"
|
48
|
+
end
|
49
|
+
|
50
|
+
# Submits run against suite
|
51
|
+
# Either creates a new run using run_name or use existing run_id
|
52
|
+
def submit_results(args = {})
|
53
|
+
sync_tests(args) if !@synced
|
54
|
+
suite = @project.find_suite(:name => @suite_name)
|
55
|
+
|
56
|
+
run_name = @config.run_name || args[:run_name] || nil
|
57
|
+
run_id = @config.run_id || args[:run_id] || nil
|
58
|
+
|
59
|
+
if !run_name.nil?
|
60
|
+
run_name = @config.run_name
|
61
|
+
@io.puts "> Created new run with name #{run_name}"
|
62
|
+
run_id = @tr.add_run( :project_id => @project.id, :name => run_name, :description => args[:run_description], :suite_id => suite.id ).id
|
63
|
+
@io.puts "> Created new run: #{run_id}"
|
64
|
+
|
65
|
+
elsif !run_id.nil?
|
66
|
+
run_id = @config.run_id
|
67
|
+
begin
|
68
|
+
run = @tr.get_run(:run_id => @config.run_id)
|
69
|
+
@io.puts "> Found run with id #{run_id}"
|
70
|
+
rescue
|
71
|
+
@io.puts "> Couldn't find run with id #{run_id}"
|
72
|
+
@io.close
|
73
|
+
return "Couldn't find run with id #{run_id}"
|
74
|
+
end
|
75
|
+
|
76
|
+
else
|
77
|
+
@io.puts "> run_name and run_id are either nil or not specified"
|
78
|
+
@io.close
|
79
|
+
return "run_name and run_id are either nil or not specified"
|
80
|
+
|
81
|
+
end
|
82
|
+
|
83
|
+
i = 0
|
84
|
+
if @case_status.empty?
|
85
|
+
while i < @ir.results.count
|
86
|
+
begin
|
87
|
+
section = suite.find_section(:name => @ir.results[i][:name])
|
88
|
+
rescue
|
89
|
+
@io.puts "> Couldn't find section with name #{@ir.results[i][:name]}"
|
90
|
+
puts "Couldn't find section with name #{@ir.results[i][:name]}"
|
91
|
+
end
|
92
|
+
case_details(@ir.results[i], section)
|
93
|
+
i += 1
|
94
|
+
end # while
|
95
|
+
end # ifa
|
96
|
+
add_case_status(run_id)
|
97
|
+
@io.puts "> Added the test case status"
|
98
|
+
@io.puts "> Submit Successful"
|
99
|
+
@io.close
|
100
|
+
return "Submitted to Test Rail"
|
101
|
+
end
|
102
|
+
|
103
|
+
def tr
|
104
|
+
@tr = ::TestRail::API.new( :user => @config.user,
|
105
|
+
:password => @config.password,
|
106
|
+
:namespace => @config.namespace)
|
107
|
+
end
|
108
|
+
|
109
|
+
# Add status to each testcase
|
110
|
+
def add_case_status(run_id)
|
111
|
+
@case_status.each { |case_id, status|
|
112
|
+
@tr.add_result_for_case(:run_id => run_id, :case_id => case_id, :status => status)
|
113
|
+
}
|
114
|
+
end
|
115
|
+
|
116
|
+
# Add section or cases within parent section
|
117
|
+
def create_suite(ir, project_id, suite, parent)
|
118
|
+
ir[:children].each do |child|
|
119
|
+
if @mappings.context.include?(child[:type])
|
120
|
+
section = parent.find_or_create_section(:project_id => project_id, :suite_id => suite.id, :name => child[:name])
|
121
|
+
create_suite(child, project_id, suite, section) if child[:children].count > 0
|
122
|
+
|
123
|
+
elsif @mappings.case.include?(child[:type])
|
124
|
+
parent = suite if parent.nil?
|
125
|
+
tcase = parent.find_or_create_test_case(:title => child[:name])
|
126
|
+
@case_status[:"#{tcase.id}"] = child[:status]
|
127
|
+
|
128
|
+
else
|
129
|
+
# To be added. Ex: steps
|
130
|
+
end # if
|
131
|
+
end # each
|
132
|
+
end # create_suite
|
133
|
+
|
134
|
+
def case_details(ir, section)
|
135
|
+
ir[:children].each do |child|
|
136
|
+
if @mappings.context.include?(child[:type])
|
137
|
+
section = section.find_section(:name => child[:name])
|
138
|
+
case_details(child, section)
|
139
|
+
|
140
|
+
elsif @mappings.case.include?(child[:type])
|
141
|
+
tcase = section.find_test_case(:title => child[:name]) if !section.nil?
|
142
|
+
@case_status[:"#{tcase.id}"] = child[:status] if !tcase.nil?
|
143
|
+
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
end # TestRail
|
149
|
+
end # Reporters
|
150
|
+
end # Res
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'res/config'
|
2
|
+
|
3
|
+
module Res
|
4
|
+
module Reporters
|
5
|
+
class Testmine
|
6
|
+
|
7
|
+
attr_accessor :url, :config
|
8
|
+
|
9
|
+
def initialize(args)
|
10
|
+
@url = args[:url]
|
11
|
+
@config = Res::Config.new([:project, :component, :suite, :version, :url, :target],
|
12
|
+
:optional => [:hive_job_id],
|
13
|
+
:pre_env => 'TESTMINE_')
|
14
|
+
config.process(args)
|
15
|
+
end
|
16
|
+
|
17
|
+
def submit_results(ir, args)
|
18
|
+
# Set missing project information
|
19
|
+
ir.project = config.project
|
20
|
+
ir.suite = config.suite
|
21
|
+
ir.target = config.target
|
22
|
+
ir.hive_job_id = config.hive_job_id
|
23
|
+
|
24
|
+
# Load world information into json hash
|
25
|
+
ir.world = {
|
26
|
+
:project => @config.project,
|
27
|
+
:component => @config.component,
|
28
|
+
:version => @config.version,
|
29
|
+
}
|
30
|
+
|
31
|
+
# Submit to testmine
|
32
|
+
uri = URI.parse(config.url)
|
33
|
+
net = Net::HTTP.new(uri.host, uri.port)
|
34
|
+
request = Net::HTTP::Post.new("/api/v1/submit")
|
35
|
+
request.set_form_data({"data" => ir.to_json})
|
36
|
+
net.read_timeout = 60
|
37
|
+
net.open_timeout = 10
|
38
|
+
|
39
|
+
response = net.start do |http|
|
40
|
+
http.request(request)
|
41
|
+
end
|
42
|
+
|
43
|
+
response.read_body
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
metadata
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: res
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- BBC
|
8
|
+
- David Buckhurst
|
9
|
+
- Asim Khan
|
10
|
+
autorequire:
|
11
|
+
bindir: bin
|
12
|
+
cert_chain: []
|
13
|
+
date: 2015-10-09 00:00:00.000000000 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: json
|
17
|
+
requirement: !ruby/object:Gem::Requirement
|
18
|
+
requirements:
|
19
|
+
- - "~>"
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '1.8'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
requirements:
|
26
|
+
- - "~>"
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
version: '1.8'
|
29
|
+
- !ruby/object:Gem::Dependency
|
30
|
+
name: test_rail-api
|
31
|
+
requirement: !ruby/object:Gem::Requirement
|
32
|
+
requirements:
|
33
|
+
- - "~>"
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: '0.4'
|
36
|
+
type: :runtime
|
37
|
+
prerelease: false
|
38
|
+
version_requirements: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - "~>"
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '0.4'
|
43
|
+
- !ruby/object:Gem::Dependency
|
44
|
+
name: ox
|
45
|
+
requirement: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - "~>"
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '2.2'
|
50
|
+
type: :runtime
|
51
|
+
prerelease: false
|
52
|
+
version_requirements: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - "~>"
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: '2.2'
|
57
|
+
- !ruby/object:Gem::Dependency
|
58
|
+
name: rspec
|
59
|
+
requirement: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - "~>"
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '3.2'
|
64
|
+
type: :development
|
65
|
+
prerelease: false
|
66
|
+
version_requirements: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - "~>"
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '3.2'
|
71
|
+
description: Formatters, parsers, and submitters for test results
|
72
|
+
email:
|
73
|
+
- david.buckhurst@bbc.co.uk
|
74
|
+
- asim.khan.ext@bbc.co.uk
|
75
|
+
executables: []
|
76
|
+
extensions: []
|
77
|
+
extra_rdoc_files: []
|
78
|
+
files:
|
79
|
+
- README.md
|
80
|
+
- lib/res.rb
|
81
|
+
- lib/res/config.rb
|
82
|
+
- lib/res/formatters/ruby_cucumber.rb
|
83
|
+
- lib/res/ir.rb
|
84
|
+
- lib/res/mappings.rb
|
85
|
+
- lib/res/parsers/junit.rb
|
86
|
+
- lib/res/reporters/hive.rb
|
87
|
+
- lib/res/reporters/test_rail.rb
|
88
|
+
- lib/res/reporters/testmine.rb
|
89
|
+
homepage: https://github.com/bbc/res
|
90
|
+
licenses:
|
91
|
+
- MIT
|
92
|
+
metadata: {}
|
93
|
+
post_install_message:
|
94
|
+
rdoc_options: []
|
95
|
+
require_paths:
|
96
|
+
- lib
|
97
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
98
|
+
requirements:
|
99
|
+
- - ">="
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
103
|
+
requirements:
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '0'
|
107
|
+
requirements: []
|
108
|
+
rubyforge_project:
|
109
|
+
rubygems_version: 2.4.8
|
110
|
+
signing_key:
|
111
|
+
specification_version: 4
|
112
|
+
summary: Test Result report libraries
|
113
|
+
test_files: []
|