moto 0.8.8 → 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cli.rb +28 -31
- data/lib/parser.rb +29 -13
- data/lib/reporting/listeners/base.rb +5 -5
- data/lib/reporting/listeners/console.rb +0 -4
- data/lib/reporting/listeners/junit_xml.rb +1 -1
- data/lib/reporting/listeners/webui.rb +153 -63
- data/lib/reporting/listeners/webui_deprecated.rb +127 -0
- data/lib/reporting/test_reporter.rb +10 -6
- data/lib/runner/test_generator.rb +19 -17
- data/lib/runner/test_provider.rb +12 -11
- data/lib/runner/test_runner.rb +6 -6
- data/lib/runner/thread_context.rb +1 -1
- data/lib/test/base.rb +9 -3
- data/lib/test/metadata.rb +77 -0
- data/lib/version.rb +1 -1
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4207f74793c80ef13675cb7eb98f8125e7f251ab
|
4
|
+
data.tar.gz: f5be5e7a42a02407a03db6a2622d9ffc28af7635
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 925d4a8275fbe9a7c865282c3678161771cd647ae4d770d339984b14083e24885b2a66da7abad143a7d87e34777b925083e137cb5e1a4d0dc05b5eb4652277bf
|
7
|
+
data.tar.gz: 2cd6b32278b60a09f4dbcc65ad5f58d5e8da892e0faa1cd2b4e6082d038cf71f10f8338051bd58975cda7ca96050367ea7bcffd2c17e5a340e9fa914d5c05a2c
|
data/lib/cli.rb
CHANGED
@@ -18,6 +18,7 @@ require_relative './runner/test_runner'
|
|
18
18
|
require_relative './runner/thread_context'
|
19
19
|
require_relative './runner/test_generator'
|
20
20
|
require_relative './test/base'
|
21
|
+
require_relative './test/metadata'
|
21
22
|
require_relative './version'
|
22
23
|
require_relative './reporting/listeners/base'
|
23
24
|
require_relative './reporting/listeners/console'
|
@@ -35,14 +36,16 @@ module Moto
|
|
35
36
|
class Cli
|
36
37
|
def self.run(argv)
|
37
38
|
|
38
|
-
|
39
|
+
tests_metadata = []
|
39
40
|
directories = argv[:tests]
|
40
41
|
tags = argv[:tags]
|
41
42
|
filters = argv[:filters]
|
42
43
|
|
43
44
|
if directories
|
44
45
|
directories.each do |directory|
|
45
|
-
|
46
|
+
Dir.glob("#{MotoApp::DIR}/tests/#{directory}/**/*.rb").each do |test_path|
|
47
|
+
tests_metadata << Moto::Test::Metadata.new(test_path)
|
48
|
+
end
|
46
49
|
end
|
47
50
|
end
|
48
51
|
|
@@ -50,51 +53,40 @@ module Moto
|
|
50
53
|
tests_total = Dir.glob("#{MotoApp::DIR}/tests/**/*.rb")
|
51
54
|
tests_total.each do |test_path|
|
52
55
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
if matches
|
57
|
-
test_tags = matches.to_a[2].gsub(/\s*/, '').split(',')
|
58
|
-
test_paths_absolute << test_path unless (tags & test_tags).empty?
|
59
|
-
end
|
56
|
+
metadata = Moto::Test::Metadata.new(test_path)
|
57
|
+
tests_metadata << metadata unless (tags & metadata.tags).empty?
|
60
58
|
|
61
59
|
end
|
62
60
|
end
|
63
61
|
|
64
62
|
# Make sure there are no repetitions in gathered set
|
65
|
-
|
63
|
+
tests_metadata.uniq! { |metadata| metadata.test_path }
|
66
64
|
|
67
65
|
# Tests to be removed due to filtering will be gathered in this array
|
68
66
|
# [].delete(item) cannot be used since it interferes with [].each
|
69
|
-
|
67
|
+
unfit_metadata = []
|
70
68
|
|
71
69
|
# Filter tests by provied tags
|
70
|
+
# - test must contain ALL tags specified with -f param
|
71
|
+
# - test may contain other tags
|
72
72
|
if filters
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
if (filters & test_tags).empty?
|
82
|
-
# Test doesn't contain any tags to be filtered upon
|
83
|
-
filtered_test_paths << test_path
|
84
|
-
end
|
85
|
-
|
86
|
-
else
|
87
|
-
# Test has no tags at all
|
88
|
-
filtered_test_paths << test_path
|
73
|
+
tests_metadata.each do |metadata|
|
74
|
+
|
75
|
+
# If test has no tags at all and filters are set it should be automatically removed
|
76
|
+
if metadata.tags.empty?
|
77
|
+
unfit_metadata << metadata
|
78
|
+
# Otherwise check provided tags and filters for compatibility
|
79
|
+
elsif (metadata.tags & filters).length != filters.length
|
80
|
+
unfit_metadata << metadata
|
89
81
|
end
|
90
82
|
|
91
83
|
end
|
92
84
|
end
|
93
85
|
|
94
|
-
|
86
|
+
tests_metadata -= unfit_metadata
|
95
87
|
|
96
88
|
#TODO Display criteria used
|
97
|
-
if
|
89
|
+
if tests_metadata.empty?
|
98
90
|
puts 'No tests found for given arguments.'
|
99
91
|
Kernel.exit(-1)
|
100
92
|
end
|
@@ -107,9 +99,14 @@ module Moto
|
|
107
99
|
initializer.init
|
108
100
|
end
|
109
101
|
|
110
|
-
|
102
|
+
run_params = {}
|
103
|
+
run_params[:run_name] = argv[:run_name]
|
104
|
+
run_params[:suite_name] = argv[:suite_name]
|
105
|
+
run_params[:assignee] = argv[:assignee]
|
106
|
+
|
107
|
+
test_reporter = Moto::Reporting::TestReporter.new(argv[:listeners], run_params)
|
111
108
|
|
112
|
-
runner = Moto::Runner::TestRunner.new(
|
109
|
+
runner = Moto::Runner::TestRunner.new(tests_metadata, test_reporter, argv[:stop_on])
|
113
110
|
runner.run
|
114
111
|
end
|
115
112
|
|
data/lib/parser.rb
CHANGED
@@ -34,9 +34,11 @@ module Moto
|
|
34
34
|
|
35
35
|
# Default options
|
36
36
|
options = {}
|
37
|
-
options[:listeners]
|
38
|
-
options[:
|
39
|
-
options[:
|
37
|
+
options[:listeners] = []
|
38
|
+
options[:run_name] = nil
|
39
|
+
options[:suite_name] = nil
|
40
|
+
options[:assignee] = nil
|
41
|
+
options[:stop_on] = {error: false, fail: false, skip: false}
|
40
42
|
|
41
43
|
# Parse arguments
|
42
44
|
OptionParser.new do |opts|
|
@@ -45,17 +47,20 @@ module Moto
|
|
45
47
|
opts.on('-f', '--filters Filters', Array) { |v| options[:filters] = v }
|
46
48
|
opts.on('-l', '--listeners Listeners', Array) { |v| options[:listeners] = v }
|
47
49
|
opts.on('-e', '--environment Environment') { |v| options[:environment] = v }
|
48
|
-
opts.on('-
|
50
|
+
opts.on('-r', '--run RunName') { |v| options[:run_name] = v }
|
51
|
+
opts.on('-s', '--suite SuiteName') { |v| options[:suite_name] = v }
|
52
|
+
opts.on('-a', '--assignee Assignee') { |v| options[:assignee] = v }
|
49
53
|
opts.on('-c', '--config Config') { |v| options[:config_name] = v }
|
50
54
|
opts.on('--stop-on-error') { options[:stop_on][:error] = true }
|
51
55
|
opts.on('--stop-on-fail') { options[:stop_on][:fail] = true }
|
52
56
|
opts.on('--stop-on-skip') { options[:stop_on][:skip] = true }
|
53
57
|
end.parse!
|
54
58
|
|
55
|
-
if options[:
|
56
|
-
options[:
|
59
|
+
if options[:run_name].nil?
|
60
|
+
options[:run_name] = evaluate_name(options[:tags], options[:tests], options[:filters])
|
57
61
|
end
|
58
62
|
|
63
|
+
|
59
64
|
if options[:environment]
|
60
65
|
Moto::Lib::Config.environment = options[:environment]
|
61
66
|
Moto::Lib::Config.load_configuration(options[:config_name] ? options[:config_name] : 'moto')
|
@@ -111,25 +116,36 @@ module Moto
|
|
111
116
|
Moto (#{Moto::VERSION}) CLI Help:
|
112
117
|
moto --version Display current version
|
113
118
|
|
114
|
-
|
119
|
+
MOTO RUN:
|
115
120
|
-t, --tests Tests to be executed.
|
116
121
|
-g, --tags Tags of tests to be executed.
|
117
122
|
Use # MOTO_TAGS: TAGNAME in test to assign tag.
|
118
123
|
-f, --filters Tags that filter tests passed via -t parameter.
|
119
|
-
Only tests in appropriate directory, having
|
120
|
-
Use # MOTO_TAGS:
|
121
|
-
|
122
|
-
|
123
|
-
One reporter that is always used: Moto::Reporting::Listeners::KernelCode
|
124
|
+
Only tests in appropriate directory, having all of the specified tags will be executed.
|
125
|
+
Use # MOTO_TAGS: TAGNAME1 in test to assign tag.
|
126
|
+
|
127
|
+
|
124
128
|
-e, --environment Mandatory environment. Environment constants and tests parametrized in certain way depend on this.
|
125
129
|
-c, --config Name of the config, without extension, to be loaded from MotoApp/config/CONFIG_NAME.rb
|
126
130
|
Default: moto (which loads: MotoApp/config/moto.rb)
|
131
|
+
|
132
|
+
|
133
|
+
-l, --listeners Reporters to be used.
|
134
|
+
Defaults are Moto::Reporting::Listeners::ConsoleDots, Moto::Reporting::Listeners::JunitXml
|
135
|
+
One reporter that is always used: Moto::Reporting::Listeners::KernelCode
|
136
|
+
-s, --suitename Name of the test suite to which should aggregate the results of current test run.
|
137
|
+
Required when specifying MotoWebUI as one of the listeners.
|
138
|
+
-r, --runname Name of the test run to which everything will be reported when using MotoWebUI.
|
139
|
+
Default: Value of -g or -t depending on which one was specified.
|
140
|
+
-a, --assignee ID of a person responsible for current test run.
|
141
|
+
Can have a default value set in config/webui section.
|
142
|
+
|
127
143
|
--stop-on-error Moto will stop test execution when an error is encountered in test results
|
128
144
|
--stop-on-fail Moto will stop test execution when a failure is encountered in test results
|
129
145
|
--stop-on-skip Moto will stop test execution when a skip is encountered in test results
|
130
146
|
|
131
147
|
|
132
|
-
|
148
|
+
MOTO GENERATE:
|
133
149
|
-t, --test Path and name of the test to be created.
|
134
150
|
Examples:
|
135
151
|
-ttest_name will create MotoApp/tests/test_name/test_name.rb
|
@@ -5,11 +5,11 @@ module Moto
|
|
5
5
|
# Base class for listeners that report results of testing 'outside' of the application.
|
6
6
|
class Base
|
7
7
|
|
8
|
-
attr_reader :
|
8
|
+
attr_reader :run_params
|
9
9
|
|
10
|
-
# @param [String]
|
11
|
-
def initialize(
|
12
|
-
@
|
10
|
+
# @param [String] run_params Described in detail in [Moto::Reporting::TestReporter]
|
11
|
+
def initialize(run_params)
|
12
|
+
@run_params = run_params
|
13
13
|
end
|
14
14
|
|
15
15
|
# Invoked when whole batch of tests starts
|
@@ -25,7 +25,7 @@ module Moto
|
|
25
25
|
|
26
26
|
# Abstract
|
27
27
|
# Invoked when a single test is started
|
28
|
-
def start_test(test_status)
|
28
|
+
def start_test(test_status, test_metadata)
|
29
29
|
# Abstract
|
30
30
|
end
|
31
31
|
|
@@ -13,7 +13,7 @@ module Moto
|
|
13
13
|
errors: run_status.tests_error.length,
|
14
14
|
failures: run_status.tests_failed.length,
|
15
15
|
skipped: run_status.tests_skipped.length,
|
16
|
-
name:
|
16
|
+
name: run_params[:run_name],
|
17
17
|
tests: run_status.tests_all.length,
|
18
18
|
time: run_status.duration,
|
19
19
|
timestamp: Time.at(run_status.time_start)
|
@@ -1,103 +1,177 @@
|
|
1
1
|
require 'rest-client'
|
2
2
|
require 'sys/uname'
|
3
|
+
require 'uri'
|
3
4
|
|
4
5
|
module Moto
|
5
6
|
module Reporting
|
6
7
|
module Listeners
|
7
8
|
class Webui < Base
|
8
9
|
|
9
|
-
REST_MAX_TRIES =
|
10
|
+
REST_MAX_TRIES = 1
|
10
11
|
REST_TIMEOUT = 15
|
11
12
|
|
12
|
-
def
|
13
|
+
def initialize(run_params)
|
14
|
+
super
|
15
|
+
|
16
|
+
if run_params[:suite_name].nil?
|
17
|
+
raise 'ERROR: Please specify suite name (-s SUITE_NAME) when using MotoWebUI as one of the listeners.'
|
18
|
+
end
|
19
|
+
|
20
|
+
if run_params[:assignee]
|
21
|
+
@assignee = run_params[:assignee]
|
22
|
+
else
|
23
|
+
@assignee = config[:default_assignee]
|
24
|
+
end
|
13
25
|
|
14
|
-
@
|
26
|
+
@tests = {}
|
27
|
+
@url = "#{config[:url]}/api"
|
15
28
|
@send_log_on_pass = config[:send_log_on_pass]
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
def start_run
|
33
|
+
|
34
|
+
# Create Suite, if it did exist already nothing new will be created and existing data will be sent in the response
|
35
|
+
url_suites = "#{@url}/suites"
|
36
|
+
suite_data = {name: run_params[:suite_name]}.to_json
|
16
37
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
cnt_error: nil,
|
24
|
-
cnt_skipped: nil,
|
25
|
-
user: Sys::Uname.sysname.downcase.include?('windows') ? ENV['USERNAME'] : ENV['LOGNAME'],
|
26
|
-
host: Sys::Uname.nodename,
|
27
|
-
pid: Process.pid
|
38
|
+
response = try {
|
39
|
+
RestClient::Request.execute(method: :post,
|
40
|
+
url: URI.escape(url_suites),
|
41
|
+
payload: suite_data,
|
42
|
+
timeout: REST_TIMEOUT,
|
43
|
+
headers: {content_type: :json, accept: :json})
|
28
44
|
}
|
45
|
+
suite = JSON.parse(response, symbolize_names: true)
|
29
46
|
|
30
|
-
|
31
|
-
|
47
|
+
# Store ID of current Suite
|
48
|
+
@suite_id = suite[:id]
|
49
|
+
|
50
|
+
# Prepare data for new TestRun
|
51
|
+
url_runs = "#{@url}/suites/#{@suite_id}/runs"
|
52
|
+
run_data = {
|
53
|
+
name: run_params[:run_name],
|
54
|
+
start_time: Time.now
|
32
55
|
}
|
33
56
|
|
34
|
-
@
|
35
|
-
|
57
|
+
if @assignee
|
58
|
+
run_data[:tester_id] = @assignee
|
59
|
+
end
|
60
|
+
|
61
|
+
run_data = run_data.to_json
|
62
|
+
|
63
|
+
# Create new TestRun based on prepared data
|
64
|
+
response = try {
|
65
|
+
RestClient::Request.execute(method: :post,
|
66
|
+
url: URI.escape(url_runs),
|
67
|
+
payload: run_data,
|
68
|
+
timeout: REST_TIMEOUT,
|
69
|
+
headers: {content_type: :json, accept: :json})
|
70
|
+
}
|
71
|
+
|
72
|
+
@run = JSON.parse(response, symbolize_names: true)
|
36
73
|
end
|
37
74
|
|
75
|
+
|
38
76
|
def end_run(run_status)
|
39
|
-
# PUT http://sandbox.dev:3000/api/runs/1
|
40
|
-
data = {
|
41
|
-
result: run_status.result,
|
42
|
-
cnt_all: run_status.tests_all.length,
|
43
|
-
cnt_passed: run_status.tests_passed.length,
|
44
|
-
cnt_failure: run_status.tests_failed.length,
|
45
|
-
cnt_error: run_status.tests_error.length,
|
46
|
-
cnt_skipped: run_status.tests_skipped.length,
|
47
|
-
duration: run_status.duration
|
48
|
-
}
|
49
77
|
|
50
|
-
|
51
|
-
|
78
|
+
url_run = "#{@url}/suites/#{@suite_id}/runs/#{@run[:id]}"
|
79
|
+
run_data = {
|
80
|
+
duration: (Time.now.to_f - run_status.time_start).to_i
|
81
|
+
}.to_json
|
82
|
+
|
83
|
+
response = try {
|
84
|
+
RestClient::Request.execute(method: :put,
|
85
|
+
url: URI.escape(url_run),
|
86
|
+
payload: run_data,
|
87
|
+
timeout: REST_TIMEOUT,
|
88
|
+
headers: {content_type: :json, accept: :json})
|
52
89
|
}
|
53
|
-
@run = JSON.parse(
|
90
|
+
@run = JSON.parse(response, symbolize_names: true)
|
54
91
|
end
|
55
92
|
|
56
|
-
def start_test(test_status)
|
57
|
-
# POST http://sandbox.dev:3000/api/tests/create
|
58
|
-
data = {
|
59
|
-
name: test_status.name,
|
60
|
-
class_name: test_status.test_class_name,
|
61
|
-
log: nil,
|
62
|
-
run_id: @run['id'],
|
63
|
-
env: test_status.env,
|
64
|
-
parameters: test_status.params.to_s,
|
65
|
-
result: :running,
|
66
|
-
error: nil,
|
67
|
-
failures: nil
|
68
|
-
}
|
69
93
|
|
70
|
-
|
71
|
-
|
94
|
+
def start_test(test_status, test_metadata)
|
95
|
+
|
96
|
+
# Prepare data for new Test
|
97
|
+
url_tests = "#{@url}/suites/#{@suite_id}/runs/#{@run[:id]}/tests"
|
98
|
+
test_data = {
|
99
|
+
name: test_status.display_name, #test_status.test_class_name
|
100
|
+
start_time: Time.now
|
72
101
|
}
|
73
|
-
@tests[test_status.name] = JSON.parse(result)
|
74
|
-
end
|
75
102
|
|
76
|
-
|
103
|
+
if test_metadata.ticket_url
|
104
|
+
test_data[:ticket_url] = test_metadata.ticket_url
|
105
|
+
end
|
77
106
|
|
78
|
-
|
79
|
-
|
80
|
-
full_log = nil
|
81
|
-
else
|
82
|
-
full_log = File.read(test_status.log_path)
|
107
|
+
if test_metadata.tags
|
108
|
+
test_data[:tags] = test_metadata.tags.join(',')
|
83
109
|
end
|
84
110
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
111
|
+
test_data = test_data.to_json
|
112
|
+
|
113
|
+
# Create new Test based on prepared data
|
114
|
+
response = try {
|
115
|
+
RestClient::Request.execute(method: :post,
|
116
|
+
url: URI.escape(url_tests),
|
117
|
+
payload: test_data,
|
118
|
+
timeout: REST_TIMEOUT,
|
119
|
+
headers: {content_type: :json, accept: :json})
|
91
120
|
}
|
92
121
|
|
93
|
-
|
94
|
-
|
122
|
+
test = JSON.parse(response, symbolize_names: true)
|
123
|
+
|
124
|
+
# Store Test in a hash with its name as key so later it can be accessed and server side ID can be retrieved
|
125
|
+
@tests[test[:name]] = test
|
126
|
+
end
|
127
|
+
|
128
|
+
|
129
|
+
def end_test(test_status)
|
130
|
+
|
131
|
+
url_test = "#{@url}/suites/#{@suite_id}/runs/#{@run[:id]}/tests/#{@tests[test_status.display_name][:id]}"
|
132
|
+
test_data = {
|
133
|
+
duration: (Time.now.to_f - test_status.time_start).to_i,
|
134
|
+
error_message: test_status.results.last.code == Moto::Test::Result::ERROR ? nil : test_status.results.last.message,
|
135
|
+
fail_message: test_failures(test_status),
|
136
|
+
result_id: webui_result_id(test_status.results.last.code),
|
137
|
+
}.to_json
|
138
|
+
|
139
|
+
# Create new Test based on prepared data
|
140
|
+
response = try {
|
141
|
+
RestClient::Request.execute(method: :put,
|
142
|
+
url: URI.escape(url_test),
|
143
|
+
payload: test_data,
|
144
|
+
timeout: REST_TIMEOUT,
|
145
|
+
headers: {content_type: :json, accept: :json})
|
95
146
|
}
|
96
|
-
|
147
|
+
|
148
|
+
test = JSON.parse(response, symbolize_names: true)
|
149
|
+
|
150
|
+
@tests[test_status.name] = test
|
151
|
+
|
152
|
+
# Add Log to already existing Test
|
153
|
+
if (test_status.results.last.code == Moto::Test::Result::PASSED && @send_log_on_pass) || test_status.results.last.code != Moto::Test::Result::PASSED
|
154
|
+
|
155
|
+
url_log = "#{url_test}/logs"
|
156
|
+
log_data = { text: File.read(test_status.log_path) }.to_json
|
157
|
+
|
158
|
+
response = try {
|
159
|
+
RestClient::Request.execute(method: :post,
|
160
|
+
url: URI.escape(url_log),
|
161
|
+
payload: log_data,
|
162
|
+
timeout: REST_TIMEOUT,
|
163
|
+
headers: {content_type: :json, accept: :json})
|
164
|
+
}
|
165
|
+
response
|
166
|
+
end
|
97
167
|
end
|
98
168
|
|
99
169
|
# @return [String] string with messages of all failures in a test
|
100
170
|
def test_failures(test_status)
|
171
|
+
if test_status.results.last.failures.empty?
|
172
|
+
return nil
|
173
|
+
end
|
174
|
+
|
101
175
|
test_status.results.last.failures.join("\n\t")
|
102
176
|
end
|
103
177
|
|
@@ -121,6 +195,22 @@ module Moto
|
|
121
195
|
end
|
122
196
|
private :config
|
123
197
|
|
198
|
+
def webui_result_id(code)
|
199
|
+
case code
|
200
|
+
when Moto::Test::Result::RUNNING
|
201
|
+
1
|
202
|
+
when Moto::Test::Result::PASSED
|
203
|
+
2
|
204
|
+
when Moto::Test::Result::FAILURE
|
205
|
+
3
|
206
|
+
when Moto::Test::Result::ERROR
|
207
|
+
4
|
208
|
+
when Moto::Test::Result::SKIPPED
|
209
|
+
5
|
210
|
+
end
|
211
|
+
end
|
212
|
+
private :webui_result_id
|
213
|
+
|
124
214
|
end
|
125
215
|
end
|
126
216
|
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
require 'rest-client'
|
2
|
+
require 'sys/uname'
|
3
|
+
|
4
|
+
module Moto
|
5
|
+
module Reporting
|
6
|
+
module Listeners
|
7
|
+
class WebuiDeprecated < Base
|
8
|
+
|
9
|
+
REST_MAX_TRIES = 3
|
10
|
+
REST_TIMEOUT = 15
|
11
|
+
|
12
|
+
def start_run
|
13
|
+
|
14
|
+
@url = config[:url]
|
15
|
+
@send_log_on_pass = config[:send_log_on_pass]
|
16
|
+
|
17
|
+
data = {
|
18
|
+
name: run_params[:name],
|
19
|
+
result: :running,
|
20
|
+
cnt_all: nil,
|
21
|
+
cnt_passed: nil,
|
22
|
+
cnt_failure: nil,
|
23
|
+
cnt_error: nil,
|
24
|
+
cnt_skipped: nil,
|
25
|
+
user: Sys::Uname.sysname.downcase.include?('windows') ? ENV['USERNAME'] : ENV['LOGNAME'],
|
26
|
+
host: Sys::Uname.nodename,
|
27
|
+
pid: Process.pid
|
28
|
+
}
|
29
|
+
|
30
|
+
result = try {
|
31
|
+
RestClient::Request.execute(method: :post, url: "#{@url}/api/runs", payload: data.to_json, timeout: REST_TIMEOUT, headers: {content_type: :json, accept: :json})
|
32
|
+
}
|
33
|
+
|
34
|
+
@run = JSON.parse(result)
|
35
|
+
@tests = {}
|
36
|
+
end
|
37
|
+
|
38
|
+
def end_run(run_status)
|
39
|
+
# PUT http://sandbox.dev:3000/api/runs/1
|
40
|
+
data = {
|
41
|
+
result: run_status.result,
|
42
|
+
cnt_all: run_status.tests_all.length,
|
43
|
+
cnt_passed: run_status.tests_passed.length,
|
44
|
+
cnt_failure: run_status.tests_failed.length,
|
45
|
+
cnt_error: run_status.tests_error.length,
|
46
|
+
cnt_skipped: run_status.tests_skipped.length,
|
47
|
+
duration: run_status.duration
|
48
|
+
}
|
49
|
+
|
50
|
+
result = try {
|
51
|
+
RestClient::Request.execute(method: :put, url: "#{@url}/api/runs/#{@run['id']}", payload: data.to_json, timeout: REST_TIMEOUT, headers: {content_type: :json, accept: :json})
|
52
|
+
}
|
53
|
+
@run = JSON.parse(result)
|
54
|
+
end
|
55
|
+
|
56
|
+
def start_test(test_status, test_metadata)
|
57
|
+
# POST http://sandbox.dev:3000/api/tests/create
|
58
|
+
data = {
|
59
|
+
name: test_status.name,
|
60
|
+
class_name: test_status.test_class_name,
|
61
|
+
log: nil,
|
62
|
+
run_id: @run['id'],
|
63
|
+
env: test_status.env,
|
64
|
+
parameters: test_status.params.to_s,
|
65
|
+
result: :running,
|
66
|
+
error: nil,
|
67
|
+
failures: nil
|
68
|
+
}
|
69
|
+
|
70
|
+
result = try {
|
71
|
+
RestClient::Request.execute(method: :post, url: "#{@url}/api/tests", payload: data.to_json, timeout: REST_TIMEOUT, headers: {content_type: :json, accept: :json})
|
72
|
+
}
|
73
|
+
@tests[test_status.name] = JSON.parse(result)
|
74
|
+
end
|
75
|
+
|
76
|
+
def end_test(test_status)
|
77
|
+
|
78
|
+
# don't send the log if the test has passed and appropriate flag is set to false
|
79
|
+
if test_status.results.last.code == Moto::Test::Result::PASSED && !@send_log_on_pass
|
80
|
+
full_log = nil
|
81
|
+
else
|
82
|
+
full_log = File.read(test_status.log_path)
|
83
|
+
end
|
84
|
+
|
85
|
+
data = {
|
86
|
+
log: full_log,
|
87
|
+
result: test_status.results.last.code,
|
88
|
+
error: test_status.results.last.code == Moto::Test::Result::ERROR ? nil : test_status.results.last.message,
|
89
|
+
failures: test_failures(test_status),
|
90
|
+
duration: test_status.duration
|
91
|
+
}
|
92
|
+
|
93
|
+
result = try {
|
94
|
+
RestClient::Request.execute(method: :put, url: "#{@url}/api/tests/#{@tests[test_status.name]['id']}", payload: data.to_json, timeout: REST_TIMEOUT, headers: {content_type: :json, accept: :json})
|
95
|
+
}
|
96
|
+
@tests[test_status.name] = JSON.parse(result)
|
97
|
+
end
|
98
|
+
|
99
|
+
# @return [String] string with messages of all failures in a test
|
100
|
+
def test_failures(test_status)
|
101
|
+
test_status.results.last.failures.join("\n\t")
|
102
|
+
end
|
103
|
+
|
104
|
+
# Tries to execute, without an error, block of code passed to the function.
|
105
|
+
# @param block Block of code to be executed up to MAX_REST_TRIES
|
106
|
+
def try(&block)
|
107
|
+
|
108
|
+
tries = REST_MAX_TRIES
|
109
|
+
|
110
|
+
begin
|
111
|
+
yield
|
112
|
+
rescue
|
113
|
+
tries -= 1
|
114
|
+
tries > 0 ? retry : raise
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
# @return [Hash] Hash with config for WebUI
|
119
|
+
def config
|
120
|
+
Moto::Lib::Config.moto[:test_reporter][:listeners][:webui]
|
121
|
+
end
|
122
|
+
private :config
|
123
|
+
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -11,8 +11,11 @@ module Moto
|
|
11
11
|
|
12
12
|
# @param [Array] listeners An array of strings, which represent qualified names of classes (listeners) that will be instantiated.
|
13
13
|
# empty array is passed then :default_listeners will be taken from config
|
14
|
-
# @param
|
15
|
-
|
14
|
+
# @param[Hash] run_params Variables specified by the user when parametrizing current moto run
|
15
|
+
# suite_name: String Name of the test suite
|
16
|
+
# run_name: String Name of the test run, may be custom made or automatically generated
|
17
|
+
# assignee: ID of person responsible for test run
|
18
|
+
def initialize(listeners, run_params)
|
16
19
|
|
17
20
|
if listeners.empty?
|
18
21
|
config[:default_listeners].each do |listener_class_name|
|
@@ -25,7 +28,7 @@ module Moto
|
|
25
28
|
end
|
26
29
|
|
27
30
|
@listeners = []
|
28
|
-
@
|
31
|
+
@run_params = run_params
|
29
32
|
listeners.each { |l| add_listener(l) }
|
30
33
|
end
|
31
34
|
|
@@ -33,7 +36,7 @@ module Moto
|
|
33
36
|
# All listeners on the list will have events reported to them.
|
34
37
|
# @param [Moto::Listener::Base] listener class to be added
|
35
38
|
def add_listener(listener)
|
36
|
-
@listeners << listener.new(@
|
39
|
+
@listeners << listener.new(@run_params)
|
37
40
|
end
|
38
41
|
|
39
42
|
# Reports start of the whole run (set of tests) to attached listeners
|
@@ -57,9 +60,10 @@ module Moto
|
|
57
60
|
|
58
61
|
# Reports star of a test to all attached listeners
|
59
62
|
# @param [Moto::Test::Status] test_status of test which's start is to be reported on
|
60
|
-
|
63
|
+
# @param [Moto::Test::Metadata] test_metadata of test which's start is to be reported on
|
64
|
+
def report_start_test(test_status, test_metadata)
|
61
65
|
@listeners.each do |l|
|
62
|
-
l.start_test(test_status)
|
66
|
+
l.start_test(test_status, test_metadata)
|
63
67
|
end
|
64
68
|
end
|
65
69
|
|
@@ -15,11 +15,11 @@ module Moto
|
|
15
15
|
# Example: A test with no config file will be returned as an array with single Moto::Test::Base in it.
|
16
16
|
# Example: A test with a config file and 2 sets of parameters there will be returned as array with two elements.
|
17
17
|
#
|
18
|
-
# @param [
|
19
|
-
# @return [Array] An array of [Moto::Test::Base]
|
18
|
+
# @param [Moto::Test::Metadata] test_metadata Metadata that describes test to be instantiated
|
19
|
+
# @return [Array] An array of [Moto::Test::Base] descendants
|
20
20
|
# each entry is a Test with set of parameters injected
|
21
|
-
def get_test_with_variants(
|
22
|
-
|
21
|
+
def get_test_with_variants(test_metadata)
|
22
|
+
variantize(test_metadata)
|
23
23
|
end
|
24
24
|
|
25
25
|
# Converts test's path to an array of Moto::Base::Test instances that represent all test variants (params)
|
@@ -28,19 +28,18 @@ module Moto
|
|
28
28
|
# Config files with ruby code will be evaluated thus if you use any classes in them
|
29
29
|
# they must be required prior to that. That might be done in overloaded app's initializer.
|
30
30
|
#
|
31
|
-
# @param [
|
31
|
+
# @param [Moto::Test::Metadata] test_metadata Metadata that describes test to be instantiated, contains path to test
|
32
32
|
# @return [Array] array of already initialized test's variants
|
33
|
-
def variantize(
|
33
|
+
def variantize(test_metadata)
|
34
34
|
variants = []
|
35
35
|
|
36
36
|
# TODO CHANGED TEMPORARY
|
37
37
|
#params_path = test_path_absolute.sub(/\.rb\z/, '')
|
38
|
-
params_directory = File.dirname(
|
38
|
+
params_directory = File.dirname(test_metadata.test_path).to_s + '/params/**'
|
39
39
|
param_files_paths = Dir.glob(params_directory)
|
40
40
|
param_files_paths = [nil] if param_files_paths.empty?
|
41
41
|
|
42
|
-
|
43
|
-
param_files_paths.each_with_index do |params_path, params_index|
|
42
|
+
param_files_paths.each do |params_path|
|
44
43
|
|
45
44
|
#TODO environment support
|
46
45
|
# Filtering out param sets that are specific to certain envs
|
@@ -50,9 +49,9 @@ module Moto
|
|
50
49
|
# end
|
51
50
|
|
52
51
|
#TODO Name/logname/displayname
|
53
|
-
test = generate(
|
54
|
-
test.init(params_path
|
55
|
-
test.log_path = "#{File.dirname(
|
52
|
+
test = generate(test_metadata)
|
53
|
+
test.init(params_path)
|
54
|
+
test.log_path = "#{File.dirname(test_metadata.test_path).to_s}/logs/#{test.name.gsub(/[^0-9A-Za-z.\-]/, '_')}.log"
|
56
55
|
@internal_counter += 1
|
57
56
|
|
58
57
|
variants << test
|
@@ -64,24 +63,27 @@ module Moto
|
|
64
63
|
|
65
64
|
# Generates test instances
|
66
65
|
# @return [Moto::Test::Base]
|
67
|
-
def generate(
|
66
|
+
def generate(test_metadata)
|
67
|
+
|
68
|
+
test_path = test_metadata.test_path
|
68
69
|
|
69
70
|
# Checking if it's possible to create test based on provided path. In case something is wrong with
|
70
71
|
# modules structure in class itself Moto::Test::Base will be instantized with raise injected into its run()
|
71
72
|
# so we can have proper reporting and summary even if the test doesn't execute.
|
72
73
|
begin
|
73
|
-
require
|
74
|
-
class_name =
|
74
|
+
require test_path
|
75
|
+
class_name = test_path.gsub("#{MotoApp::DIR}/", 'moto_app/').camelize.chomp('.rb').constantize
|
75
76
|
test_object = class_name.new
|
76
77
|
rescue NameError => e
|
77
78
|
class_name = Moto::Test::Base
|
78
79
|
test_object = class_name.new
|
79
80
|
|
80
|
-
error_message = "ERROR: Invalid test: #{
|
81
|
+
error_message = "ERROR: Invalid test: #{test_path.gsub("#{MotoApp::DIR}/", 'moto_app/').camelize.chomp('.rb')}.\nMESSAGE: #{e}"
|
81
82
|
inject_error_to_test(test_object, error_message)
|
82
83
|
end
|
83
84
|
|
84
|
-
test_object.static_path =
|
85
|
+
test_object.static_path = test_path
|
86
|
+
test_object.metadata = test_metadata
|
85
87
|
test_object
|
86
88
|
end
|
87
89
|
private :generate
|
data/lib/runner/test_provider.rb
CHANGED
@@ -6,13 +6,13 @@ module Moto
|
|
6
6
|
# Thread safe provider of test instances
|
7
7
|
class TestProvider
|
8
8
|
|
9
|
-
# @param [Array]
|
10
|
-
def initialize(
|
9
|
+
# @param [Array] tests_metadata
|
10
|
+
def initialize(tests_metadata)
|
11
11
|
super()
|
12
12
|
@test_repeats = Moto::Lib::Config.moto[:test_runner][:test_repeats]
|
13
13
|
@current_test_repeat = 1
|
14
14
|
@queue = Queue.new
|
15
|
-
@
|
15
|
+
@tests_metadata = tests_metadata
|
16
16
|
@test_generator = TestGenerator.new
|
17
17
|
end
|
18
18
|
|
@@ -26,9 +26,10 @@ module Moto
|
|
26
26
|
def create_tests
|
27
27
|
if @queue.empty?
|
28
28
|
|
29
|
-
|
29
|
+
test_metadata = get_test_metadata
|
30
30
|
|
31
|
-
if
|
31
|
+
if test_metadata
|
32
|
+
test_variants = @test_generator.get_test_with_variants(test_metadata)
|
32
33
|
test_variants.each do |test|
|
33
34
|
@queue.push(test)
|
34
35
|
end
|
@@ -38,12 +39,12 @@ module Moto
|
|
38
39
|
end
|
39
40
|
private :create_tests
|
40
41
|
|
41
|
-
# Returns
|
42
|
-
# return [
|
43
|
-
def
|
42
|
+
# Returns metadata of the test while supporting the number of repeats specified by the user
|
43
|
+
# return [Moto::Test::Metadata]
|
44
|
+
def get_test_metadata
|
44
45
|
|
45
46
|
if @current_test_repeat == 1
|
46
|
-
@
|
47
|
+
@test_metadata = @tests_metadata.shift
|
47
48
|
end
|
48
49
|
|
49
50
|
if @current_test_repeat == @test_repeats
|
@@ -52,9 +53,9 @@ module Moto
|
|
52
53
|
@current_test_repeat += 1
|
53
54
|
end
|
54
55
|
|
55
|
-
@
|
56
|
+
@test_metadata
|
56
57
|
end
|
57
|
-
private :
|
58
|
+
private :get_test_metadata
|
58
59
|
|
59
60
|
# Number of threads waiting for a job
|
60
61
|
def num_waiting
|
data/lib/runner/test_runner.rb
CHANGED
@@ -7,25 +7,25 @@ module Moto
|
|
7
7
|
|
8
8
|
attr_reader :test_reporter
|
9
9
|
|
10
|
-
# @param [Array]
|
10
|
+
# @param [Array] tests_metadata Collection of [Moto::Test::Metadata] objects describing Tests
|
11
11
|
# @param [Moto::Reporting::TestReporter] test_reporter Reporter of test/run statuses that communicates with external status listeners
|
12
12
|
# @param [Hash] stop_conditions Describe when TestRunner should abnormally stop its execution
|
13
13
|
# :error [Boolean]
|
14
14
|
# :fail [Boolean]
|
15
15
|
# :skip [Boolean]
|
16
|
-
def initialize(
|
17
|
-
@
|
16
|
+
def initialize(tests_metadata, test_reporter, stop_conditions)
|
17
|
+
@tests_metadata = tests_metadata
|
18
18
|
@test_reporter = test_reporter
|
19
19
|
@stop_conditions = stop_conditions
|
20
20
|
end
|
21
21
|
|
22
22
|
def run
|
23
|
-
test_provider = TestProvider.new(@
|
23
|
+
test_provider = TestProvider.new(@tests_metadata)
|
24
24
|
threads_max = Moto::Lib::Config.moto[:test_runner][:thread_count] || 1
|
25
25
|
|
26
26
|
# remove log/screenshot files from previous execution
|
27
|
-
@
|
28
|
-
FileUtils.rm_rf("#{File.dirname(test_path)}/logs")
|
27
|
+
@tests_metadata.each do |metadata|
|
28
|
+
FileUtils.rm_rf("#{File.dirname(metadata.test_path)}/logs")
|
29
29
|
end
|
30
30
|
|
31
31
|
@test_reporter.report_start_run
|
data/lib/test/base.rb
CHANGED
@@ -10,6 +10,10 @@ module Moto
|
|
10
10
|
attr_accessor :static_path
|
11
11
|
attr_accessor :status
|
12
12
|
|
13
|
+
# Contains information specified by user in Test headers, marked with appropriate tags
|
14
|
+
# @return [Moto::Test::Metadata]
|
15
|
+
attr_accessor :metadata
|
16
|
+
|
13
17
|
class << self
|
14
18
|
attr_accessor :_path
|
15
19
|
end
|
@@ -23,17 +27,19 @@ module Moto
|
|
23
27
|
end
|
24
28
|
|
25
29
|
# Initializes test to be executed with specified params and environment
|
26
|
-
def init(params_path
|
30
|
+
def init(params_path)
|
27
31
|
@env = Moto::Lib::Config.environment
|
28
32
|
@params = []
|
29
33
|
@params_path = params_path
|
30
|
-
|
34
|
+
|
31
35
|
@name = self.class.to_s.demodulize
|
32
|
-
@name += "_#{@params_path.split(
|
36
|
+
@name += "_#{@params_path.split('/')[-1].chomp('.param')}" if @params_path
|
37
|
+
|
33
38
|
@status = Moto::Test::Status.new
|
34
39
|
@status.name = @name
|
35
40
|
@status.test_class_name = self.class.name
|
36
41
|
@status.display_name = @status.test_class_name.split('::')[2..-2].join('::')
|
42
|
+
@status.display_name += "_#{@params_path.split('/')[-1].chomp('.param')}" if @params_path
|
37
43
|
@status.env = Moto::Lib::Config.environment
|
38
44
|
end
|
39
45
|
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module Moto
|
2
|
+
module Test
|
3
|
+
# Provides tools for accessing metadata embedded in test files
|
4
|
+
class Metadata
|
5
|
+
|
6
|
+
# Absolute test path
|
7
|
+
attr_reader :test_path
|
8
|
+
|
9
|
+
# @param [String] test_path Absolute path to file with test
|
10
|
+
def initialize(test_path)
|
11
|
+
@test_path = test_path
|
12
|
+
end
|
13
|
+
|
14
|
+
# Text of the file with test
|
15
|
+
def text
|
16
|
+
if @text.nil?
|
17
|
+
@text = ''
|
18
|
+
|
19
|
+
File.foreach(@test_path) do |line|
|
20
|
+
|
21
|
+
# Read lines of file until class specification begins
|
22
|
+
if line.match(/^\s*(class|module)/)
|
23
|
+
break
|
24
|
+
end
|
25
|
+
|
26
|
+
@text += line
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
@text
|
32
|
+
end
|
33
|
+
private :text
|
34
|
+
|
35
|
+
# @return [Array] of [String] which represent contents of #MOTO_TAGS
|
36
|
+
def tags
|
37
|
+
if @tags.nil?
|
38
|
+
matches = text.match(/^#(\s*)MOTO_TAGS:(.*?)$/)
|
39
|
+
|
40
|
+
if matches
|
41
|
+
@tags = matches.to_a[2].gsub(/\s*/, '').split(',')
|
42
|
+
else
|
43
|
+
@tags = []
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
@tags
|
48
|
+
end
|
49
|
+
|
50
|
+
# @return [String] which represents contents of #TICKET_URL
|
51
|
+
def ticket_url
|
52
|
+
if @ticket_url.nil?
|
53
|
+
matches = text.match(/^#(\s*)TICKET_URL:(.*?)$/)
|
54
|
+
|
55
|
+
if matches
|
56
|
+
@ticket_url = matches.to_a[2].gsub(/\s*/, '')
|
57
|
+
else
|
58
|
+
@ticket_url = ''
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
@ticket_url
|
63
|
+
end
|
64
|
+
|
65
|
+
# Overriden eql? so various comparisons, array substractions etc. can be perfromed on
|
66
|
+
# Metadata objects with them being represented by test's location
|
67
|
+
def eql?(other)
|
68
|
+
if self.class == other.class
|
69
|
+
return self.test_path == other.test_path
|
70
|
+
end
|
71
|
+
|
72
|
+
false
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
data/lib/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: moto
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bartek Wilczek
|
@@ -11,7 +11,7 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2016-
|
14
|
+
date: 2016-11-17 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: activesupport
|
@@ -99,6 +99,7 @@ files:
|
|
99
99
|
- lib/reporting/listeners/console_dots.rb
|
100
100
|
- lib/reporting/listeners/junit_xml.rb
|
101
101
|
- lib/reporting/listeners/webui.rb
|
102
|
+
- lib/reporting/listeners/webui_deprecated.rb
|
102
103
|
- lib/reporting/run_status.rb
|
103
104
|
- lib/reporting/test_reporter.rb
|
104
105
|
- lib/runner/test_generator.rb
|
@@ -107,6 +108,7 @@ files:
|
|
107
108
|
- lib/runner/thread_context.rb
|
108
109
|
- lib/runner_logging.rb
|
109
110
|
- lib/test/base.rb
|
111
|
+
- lib/test/metadata.rb
|
110
112
|
- lib/test/result.rb
|
111
113
|
- lib/test/status.rb
|
112
114
|
- lib/test_logging.rb
|
@@ -131,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
131
133
|
version: '0'
|
132
134
|
requirements: []
|
133
135
|
rubyforge_project:
|
134
|
-
rubygems_version: 2.
|
136
|
+
rubygems_version: 2.6.7
|
135
137
|
signing_key:
|
136
138
|
specification_version: 4
|
137
139
|
summary: Moto - yet another web testing framework
|