simmer 2.0.0.pre.alpha.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +7 -1
- data/.ruby-version +1 -1
- data/.travis.yml +3 -3
- data/CHANGELOG.md +44 -1
- data/README.md +40 -0
- data/lib/simmer.rb +17 -99
- data/lib/simmer/bootstrap.rb +133 -0
- data/lib/simmer/configuration.rb +16 -2
- data/lib/simmer/configuration/callback_dsl.rb +79 -0
- data/lib/simmer/database/fixture_set.rb +3 -1
- data/lib/simmer/externals/mysql_database.rb +11 -3
- data/lib/simmer/externals/spoon_client.rb +5 -4
- data/lib/simmer/judge/result.rb +1 -1
- data/lib/simmer/re_runner.rb +46 -0
- data/lib/simmer/runner.rb +52 -52
- data/lib/simmer/runner/result.rb +25 -6
- data/lib/simmer/runner/timeout_error.rb +23 -0
- data/lib/simmer/suite.rb +20 -11
- data/lib/simmer/suite/output_router.rb +73 -0
- data/lib/simmer/suite/pdi_output_writer.rb +56 -0
- data/lib/simmer/suite/result.rb +1 -0
- data/lib/simmer/suite/results_writer.rb +44 -0
- data/lib/simmer/util.rb +1 -0
- data/lib/simmer/util/file_system.rb +25 -0
- data/lib/simmer/version.rb +1 -1
- data/simmer.gemspec +4 -2
- metadata +43 -7
- data/lib/simmer/suite/reporter.rb +0 -83
@@ -0,0 +1,79 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Simmer
|
11
|
+
class Configuration
|
12
|
+
# Defines lifecycle hooks which can be run before and after the entire
|
13
|
+
# suite or just a single test. Very similar to Rspec
|
14
|
+
# (https://relishapp.com/rspec/rspec-core/v/3-9/docs/hooks/before-and-after-hooks).
|
15
|
+
class CallbackDsl
|
16
|
+
def initialize
|
17
|
+
@before_suite = []
|
18
|
+
@after_suite = []
|
19
|
+
@before_each = []
|
20
|
+
@after_each = []
|
21
|
+
|
22
|
+
freeze
|
23
|
+
end
|
24
|
+
|
25
|
+
# Used to create a before callback. This accepts and optional level
|
26
|
+
# parameter which can either be :suite or :each. ":each" is implied if no
|
27
|
+
# level is provided.
|
28
|
+
def before(level = LEVEL_EACH, &block)
|
29
|
+
verify_level!(level)
|
30
|
+
|
31
|
+
level == LEVEL_SUITE ? before_suite.push(block) : before_each.push(block)
|
32
|
+
end
|
33
|
+
|
34
|
+
# Used to create an after callback. This accepts and optional level
|
35
|
+
# parameter which can either be :suite or :each. ":each" is implied if no
|
36
|
+
# level is provided.
|
37
|
+
def after(level = LEVEL_EACH, &block)
|
38
|
+
verify_level!(level)
|
39
|
+
|
40
|
+
level == LEVEL_SUITE ? after_suite.push(block) : after_each.push(block)
|
41
|
+
end
|
42
|
+
|
43
|
+
# :nodoc:
|
44
|
+
def run_single_test_with_callbacks
|
45
|
+
before_each.each(&:call)
|
46
|
+
|
47
|
+
result = yield
|
48
|
+
|
49
|
+
after_each.each { |block| block.call(result) }
|
50
|
+
|
51
|
+
result
|
52
|
+
end
|
53
|
+
|
54
|
+
# :nodoc:
|
55
|
+
def run_suite_with_callbacks
|
56
|
+
before_suite.each(&:call)
|
57
|
+
|
58
|
+
result = yield
|
59
|
+
|
60
|
+
after_suite.each { |block| block.call(result) }
|
61
|
+
|
62
|
+
result
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def verify_level!(level)
|
68
|
+
raise ArgumentError, "unknown test level: #{level}" unless CALLBACK_LEVELS.include?(level)
|
69
|
+
end
|
70
|
+
|
71
|
+
attr_reader :after_each, :before_each, :after_suite, :before_suite
|
72
|
+
|
73
|
+
LEVEL_EACH = :each
|
74
|
+
LEVEL_SUITE = :suite
|
75
|
+
CALLBACK_LEVELS = Set.new([LEVEL_EACH, LEVEL_SUITE])
|
76
|
+
private_constant :LEVEL_EACH, :LEVEL_SUITE, :CALLBACK_LEVELS
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -13,6 +13,8 @@ module Simmer
|
|
13
13
|
module Database
|
14
14
|
# Hydrate a collection of Fixture instances from configuration.
|
15
15
|
class FixtureSet
|
16
|
+
class FixtureMissingError < StandardError; end
|
17
|
+
|
16
18
|
def initialize(config = {})
|
17
19
|
@fixtures_by_name = config_to_fixures_by_name(config)
|
18
20
|
|
@@ -22,7 +24,7 @@ module Simmer
|
|
22
24
|
def get!(name)
|
23
25
|
key = name.to_s
|
24
26
|
|
25
|
-
raise
|
27
|
+
raise FixtureMissingError, "fixture missing: #{name}" unless fixtures_by_name.key?(key)
|
26
28
|
|
27
29
|
fixtures_by_name[key]
|
28
30
|
end
|
@@ -27,7 +27,7 @@ module Simmer
|
|
27
27
|
end
|
28
28
|
|
29
29
|
def records(table, columns = [])
|
30
|
-
query = "SELECT #{sql_select_params(columns)} FROM #{table}"
|
30
|
+
query = "SELECT #{sql_select_params(columns)} FROM #{qualify(table)}"
|
31
31
|
|
32
32
|
client.query(query).to_a
|
33
33
|
end
|
@@ -53,7 +53,11 @@ module Simmer
|
|
53
53
|
attr_reader :client, :fixture_set, :table_names
|
54
54
|
|
55
55
|
def sql_select_params(columns)
|
56
|
-
Array(columns).any?
|
56
|
+
if Array(columns).any?
|
57
|
+
Array(columns).map { |c| qualify(client.escape(c)).to_s }.join(',')
|
58
|
+
else
|
59
|
+
'*'
|
60
|
+
end
|
57
61
|
end
|
58
62
|
|
59
63
|
def seed_sql_statements(fixtures)
|
@@ -62,7 +66,7 @@ module Simmer
|
|
62
66
|
|
63
67
|
def clean_sql_statements
|
64
68
|
table_names.map do |table_name|
|
65
|
-
"TRUNCATE #{table_name}"
|
69
|
+
"TRUNCATE #{qualify(table_name)}"
|
66
70
|
end
|
67
71
|
end
|
68
72
|
|
@@ -110,6 +114,10 @@ module Simmer
|
|
110
114
|
|
111
115
|
raise ArgumentError, "database (#{name}) must end in #{DATABASE_SUFFIX}"
|
112
116
|
end
|
117
|
+
|
118
|
+
def qualify(identifier)
|
119
|
+
"`#{identifier}`"
|
120
|
+
end
|
113
121
|
end
|
114
122
|
end
|
115
123
|
end
|
@@ -24,13 +24,13 @@ module Simmer
|
|
24
24
|
freeze
|
25
25
|
end
|
26
26
|
|
27
|
-
def run(specification, config)
|
27
|
+
def run(specification, config, &output_capturer)
|
28
28
|
execution_result = nil
|
29
29
|
time_in_seconds = nil
|
30
30
|
|
31
31
|
begin
|
32
32
|
time_in_seconds = Benchmark.measure do
|
33
|
-
execution_result = execute!(specification, config)
|
33
|
+
execution_result = execute!(specification, config, &output_capturer)
|
34
34
|
end.real
|
35
35
|
rescue Pdi::Spoon::PanError, Pdi::Spoon::KitchenError => e
|
36
36
|
return Result.new(
|
@@ -50,14 +50,15 @@ module Simmer
|
|
50
50
|
|
51
51
|
attr_reader :files_dir
|
52
52
|
|
53
|
-
def execute!(specification, config)
|
53
|
+
def execute!(specification, config, &output_capturer)
|
54
54
|
act = specification.act
|
55
55
|
|
56
56
|
spoon.run(
|
57
57
|
repository: act.repository,
|
58
58
|
name: act.name,
|
59
59
|
params: act.compiled_params(files_dir, config),
|
60
|
-
type: act.type
|
60
|
+
type: act.type,
|
61
|
+
&output_capturer
|
61
62
|
)
|
62
63
|
end
|
63
64
|
end
|
data/lib/simmer/judge/result.rb
CHANGED
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'judge'
|
11
|
+
require_relative 'runner/result'
|
12
|
+
|
13
|
+
module Simmer
|
14
|
+
# :nodoc:
|
15
|
+
# Wraps a <tt>Simmer::Runner</tt> and knows how to re-run tests based
|
16
|
+
# on certain failure cases.
|
17
|
+
class ReRunner < SimpleDelegator
|
18
|
+
attr_reader :timeout_failure_retry_count
|
19
|
+
|
20
|
+
def initialize(runner, out, timeout_failure_retry_count: 0)
|
21
|
+
@timeout_failure_retry_count = timeout_failure_retry_count.to_i
|
22
|
+
@out = out
|
23
|
+
|
24
|
+
super(runner)
|
25
|
+
end
|
26
|
+
|
27
|
+
def run(*args)
|
28
|
+
rerun_on_timeout(args, timeout_failure_retry_count)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
attr_reader :out
|
34
|
+
|
35
|
+
def rerun_on_timeout(run_args, times)
|
36
|
+
result = __getobj__.run(*run_args)
|
37
|
+
|
38
|
+
if result.timed_out? && times.positive?
|
39
|
+
out.console_puts('Retrying due to a timeout...')
|
40
|
+
rerun_on_timeout(run_args, times - 1)
|
41
|
+
else
|
42
|
+
result
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/simmer/runner.rb
CHANGED
@@ -9,6 +9,7 @@
|
|
9
9
|
|
10
10
|
require_relative 'judge'
|
11
11
|
require_relative 'runner/result'
|
12
|
+
require_relative 'runner/timeout_error'
|
12
13
|
|
13
14
|
module Simmer
|
14
15
|
# Runs a single specification.
|
@@ -26,110 +27,109 @@ module Simmer
|
|
26
27
|
freeze
|
27
28
|
end
|
28
29
|
|
29
|
-
def run(specification, config
|
30
|
-
|
31
|
-
print("Path: #{specification.path}")
|
30
|
+
def run(specification, config:, id: SecureRandom.uuid)
|
31
|
+
out.announce_start(id, specification)
|
32
32
|
|
33
|
-
|
34
|
-
|
35
|
-
clean_file_system
|
36
|
-
seed_file_system(specification)
|
33
|
+
config.run_single_test_with_callbacks do
|
34
|
+
clean_and_seed(specification)
|
37
35
|
|
38
|
-
|
39
|
-
|
36
|
+
spoon_client_result = execute_spoon(specification, config)
|
37
|
+
judge_result = assert(specification, spoon_client_result)
|
40
38
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
39
|
+
Result.new(
|
40
|
+
id: id,
|
41
|
+
judge_result: judge_result,
|
42
|
+
specification: specification,
|
43
|
+
spoon_client_result: spoon_client_result
|
44
|
+
).tap { |result| out.final_verdict(result) }
|
45
|
+
rescue Database::FixtureSet::FixtureMissingError, Simmer::Runner::TimeoutError => e
|
46
|
+
Result.new(id: id, specification: specification, errors: e)
|
47
|
+
.tap { |result| out.final_verdict(result) }
|
45
48
|
end
|
46
49
|
end
|
47
50
|
|
51
|
+
def complete
|
52
|
+
out.close
|
53
|
+
end
|
54
|
+
|
48
55
|
private
|
49
56
|
|
50
57
|
attr_reader :database, :file_system, :fixture_set, :judge, :out
|
51
58
|
|
59
|
+
def clean_and_seed(specification)
|
60
|
+
clean_db
|
61
|
+
seed_db(specification)
|
62
|
+
clean_file_system
|
63
|
+
seed_file_system(specification)
|
64
|
+
end
|
65
|
+
|
52
66
|
def clean_db
|
53
|
-
|
67
|
+
out.waiting('Stage', 'Cleaning database')
|
54
68
|
count = database.clean!
|
55
|
-
|
69
|
+
out.console_puts("#{count} table(s) emptied")
|
56
70
|
|
57
71
|
count
|
58
72
|
end
|
59
73
|
|
60
74
|
def seed_db(specification)
|
61
|
-
|
75
|
+
out.waiting('Stage', 'Seeding database')
|
62
76
|
|
63
77
|
fixtures = specification.stage.fixtures.map { |f| fixture_set.get!(f) }
|
64
78
|
count = database.seed!(fixtures)
|
65
79
|
|
66
|
-
|
80
|
+
out.console_puts("#{count} record(s) inserted")
|
67
81
|
|
68
82
|
count
|
83
|
+
rescue Database::FixtureSet::FixtureMissingError => e
|
84
|
+
out.console_puts('Missing Fixture(s)')
|
85
|
+
raise e
|
69
86
|
end
|
70
87
|
|
71
88
|
def clean_file_system
|
72
|
-
|
89
|
+
out.waiting('Stage', 'Cleaning File System')
|
73
90
|
count = file_system.clean!
|
74
|
-
|
91
|
+
out.console_puts("#{count} file(s) deleted")
|
75
92
|
|
76
93
|
count
|
77
94
|
end
|
78
95
|
|
79
96
|
def seed_file_system(specification)
|
80
|
-
|
97
|
+
out.waiting('Stage', 'Seeding File System')
|
81
98
|
count = file_system.write!(specification.stage.files)
|
82
|
-
|
99
|
+
out.console_puts("#{count} file(s) uploaded")
|
83
100
|
|
84
101
|
count
|
85
102
|
end
|
86
103
|
|
87
104
|
def execute_spoon(specification, config)
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
105
|
+
out.waiting('Act', 'Executing Spoon')
|
106
|
+
|
107
|
+
spoon_client_result = spoon_client.run(specification, config.config) do |output|
|
108
|
+
out.capture_spoon_output(output)
|
109
|
+
end
|
110
|
+
|
111
|
+
out.finish_spec
|
112
|
+
out.spoon_execution_detail_message(spoon_client_result)
|
92
113
|
|
93
114
|
spoon_client_result
|
115
|
+
rescue Timeout::Error => e
|
116
|
+
out.console_puts('Timed out')
|
117
|
+
raise Simmer::Runner::TimeoutError, e
|
94
118
|
end
|
95
119
|
|
96
120
|
def assert(specification, spoon_client_result)
|
97
|
-
|
121
|
+
out.waiting('Assert', 'Checking results')
|
98
122
|
|
99
123
|
if spoon_client_result.fail?
|
100
|
-
|
124
|
+
out.console_puts('Skipped')
|
101
125
|
return nil
|
102
126
|
end
|
103
127
|
|
104
128
|
output = spoon_client_result.execution_result.out
|
105
129
|
judge_result = judge.assert(specification, output)
|
106
|
-
|
107
|
-
|
108
|
-
print(msg)
|
130
|
+
out.result(judge_result)
|
109
131
|
|
110
132
|
judge_result
|
111
133
|
end
|
112
|
-
|
113
|
-
def print(msg)
|
114
|
-
out.puts(msg)
|
115
|
-
end
|
116
|
-
|
117
|
-
def print_waiting(stage, msg)
|
118
|
-
max = 25
|
119
|
-
char = '.'
|
120
|
-
msg = " > #{pad_right(stage, 6)} - #{pad_right(msg, max, char)}"
|
121
|
-
|
122
|
-
out.print(msg)
|
123
|
-
end
|
124
|
-
|
125
|
-
def pad_right(msg, len, char = ' ')
|
126
|
-
missing = len - msg.length
|
127
|
-
|
128
|
-
"#{msg}#{char * missing}"
|
129
|
-
end
|
130
|
-
|
131
|
-
def pass_message(obj)
|
132
|
-
obj.pass? ? 'Pass' : 'Fail'
|
133
|
-
end
|
134
134
|
end
|
135
135
|
end
|
data/lib/simmer/runner/result.rb
CHANGED
@@ -13,29 +13,47 @@ module Simmer
|
|
13
13
|
class Result
|
14
14
|
extend Forwardable
|
15
15
|
|
16
|
-
attr_reader :id, :judge_result, :specification, :spoon_client_result
|
17
|
-
|
18
|
-
def_delegators :spoon_client_result, :time_in_seconds
|
16
|
+
attr_reader :errors, :id, :judge_result, :specification, :spoon_client_result
|
19
17
|
|
20
18
|
def_delegators :specification, :name
|
21
19
|
|
22
|
-
def initialize(
|
20
|
+
def initialize(
|
21
|
+
id:,
|
22
|
+
specification:,
|
23
|
+
judge_result: nil,
|
24
|
+
spoon_client_result: nil,
|
25
|
+
errors: []
|
26
|
+
)
|
23
27
|
@id = id.to_s
|
24
28
|
@judge_result = judge_result
|
25
29
|
@specification = specification
|
26
30
|
@spoon_client_result = spoon_client_result
|
31
|
+
@errors = Array(errors)
|
27
32
|
|
28
33
|
freeze
|
29
34
|
end
|
30
35
|
|
36
|
+
def time_in_seconds
|
37
|
+
spoon_client_result&.time_in_seconds || 0
|
38
|
+
end
|
39
|
+
|
31
40
|
def pass?
|
32
|
-
|
41
|
+
[
|
42
|
+
judge_result&.pass?,
|
43
|
+
spoon_client_result&.pass?,
|
44
|
+
errors.empty?,
|
45
|
+
].all?
|
33
46
|
end
|
47
|
+
alias passing? pass?
|
34
48
|
|
35
49
|
def fail?
|
36
50
|
!pass?
|
37
51
|
end
|
38
52
|
|
53
|
+
def timed_out?
|
54
|
+
errors.any? { |e| e.is_a?(Simmer::Runner::TimeoutError) }
|
55
|
+
end
|
56
|
+
|
39
57
|
def to_h
|
40
58
|
{
|
41
59
|
'name' => specification.name,
|
@@ -44,7 +62,8 @@ module Simmer
|
|
44
62
|
'time_in_seconds' => time_in_seconds,
|
45
63
|
'pass' => pass?,
|
46
64
|
'spoon_client_result' => spoon_client_result.to_h,
|
47
|
-
'judge_result' => judge_result.to_h
|
65
|
+
'judge_result' => judge_result.to_h,
|
66
|
+
'errors' => errors.map(&:message),
|
48
67
|
}
|
49
68
|
end
|
50
69
|
end
|