omnitest-skeptic 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +6 -0
- data/.rspec +2 -0
- data/.rubocop.yml +5 -0
- data/.rubocop_todo.yml +36 -0
- data/Gemfile +24 -0
- data/LICENSE.txt +22 -0
- data/README.md +31 -0
- data/Rakefile +12 -0
- data/bin/skeptic +4 -0
- data/lib/omnitest/skeptic.rb +123 -0
- data/lib/omnitest/skeptic/cli.rb +156 -0
- data/lib/omnitest/skeptic/configuration.rb +48 -0
- data/lib/omnitest/skeptic/errors.rb +19 -0
- data/lib/omnitest/skeptic/evidence.rb +81 -0
- data/lib/omnitest/skeptic/property_definition.rb +8 -0
- data/lib/omnitest/skeptic/result.rb +27 -0
- data/lib/omnitest/skeptic/scenario.rb +167 -0
- data/lib/omnitest/skeptic/scenario_definition.rb +41 -0
- data/lib/omnitest/skeptic/spies.rb +43 -0
- data/lib/omnitest/skeptic/spy.rb +23 -0
- data/lib/omnitest/skeptic/test_manifest.rb +78 -0
- data/lib/omnitest/skeptic/test_statuses.rb +63 -0
- data/lib/omnitest/skeptic/test_transitions.rb +172 -0
- data/lib/omnitest/skeptic/validation.rb +39 -0
- data/lib/omnitest/skeptic/validator.rb +34 -0
- data/lib/omnitest/skeptic/validator_registry.rb +33 -0
- data/lib/omnitest/skeptic/version.rb +5 -0
- data/omnitest-skeptic.gemspec +34 -0
- data/spec/fabricators/psychic_fabricator.rb +12 -0
- data/spec/fabricators/scenario_fabricator.rb +6 -0
- data/spec/fabricators/validator_fabricator.rb +12 -0
- data/spec/fixtures/factorial.py +18 -0
- data/spec/fixtures/skeptic.yaml +16 -0
- data/spec/omnitest/skeptic/evidence_spec.rb +58 -0
- data/spec/omnitest/skeptic/result_spec.rb +51 -0
- data/spec/omnitest/skeptic/scenario_definition_spec.rb +39 -0
- data/spec/omnitest/skeptic/scenario_spec.rb +35 -0
- data/spec/omnitest/skeptic/test_manifest_spec.rb +28 -0
- data/spec/omnitest/skeptic/validator_registry_spec.rb +40 -0
- data/spec/omnitest/skeptic/validator_spec.rb +70 -0
- data/spec/omnitest/skeptic_spec.rb +65 -0
- data/spec/spec_helper.rb +65 -0
- metadata +289 -0
@@ -0,0 +1,63 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
module TestStatuses
|
4
|
+
def failed?
|
5
|
+
evidence.last_attempted_action != evidence.last_completed_action
|
6
|
+
end
|
7
|
+
|
8
|
+
def skipped?
|
9
|
+
result.nil?
|
10
|
+
end
|
11
|
+
|
12
|
+
def sample?
|
13
|
+
!source_file.nil?
|
14
|
+
end
|
15
|
+
|
16
|
+
def status
|
17
|
+
status = last_attempted_action
|
18
|
+
failed? ? "#{status}_failed" : status
|
19
|
+
end
|
20
|
+
|
21
|
+
def status_description
|
22
|
+
case status
|
23
|
+
when 'clone' then 'Cloned'
|
24
|
+
when 'clone_failed' then 'Clone Failed'
|
25
|
+
when 'detect' then 'Sample Found'
|
26
|
+
when 'detect_failed', nil then '<Not Found>'
|
27
|
+
when 'bootstrap' then 'Bootstrapped'
|
28
|
+
when 'bootstrap_failed' then 'Bootstrap Failed'
|
29
|
+
when 'detect' then 'Detected'
|
30
|
+
when 'exec' then 'Executed'
|
31
|
+
when 'exec_failed' then 'Execution Failed'
|
32
|
+
when 'verify', 'verify_failed'
|
33
|
+
validator_count = validators.count
|
34
|
+
validation_count = validations.values.select { |v| v.result == :passed }.count
|
35
|
+
if validator_count == validation_count
|
36
|
+
"Fully Verified (#{validation_count} of #{validator_count})"
|
37
|
+
else
|
38
|
+
"Partially Verified (#{validation_count} of #{validator_count})"
|
39
|
+
end
|
40
|
+
# when 'verify_failed' then 'Verification Failed'
|
41
|
+
else "<Unknown (#{status})>"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def status_color
|
46
|
+
case status_description
|
47
|
+
when '<Not Found>' then :white
|
48
|
+
when 'Cloned' then :magenta
|
49
|
+
when 'Bootstrapped' then :magenta
|
50
|
+
when 'Sample Found' then :cyan
|
51
|
+
when 'Executed' then :blue
|
52
|
+
when /Verified/
|
53
|
+
if status_description =~ /Fully/
|
54
|
+
:green
|
55
|
+
else
|
56
|
+
:yellow
|
57
|
+
end
|
58
|
+
else :red
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,172 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
module TestTransitions
|
4
|
+
def detect
|
5
|
+
transition_to :detect
|
6
|
+
end
|
7
|
+
|
8
|
+
def detect_action
|
9
|
+
perform_action(:detect, 'Detecting code sample') do
|
10
|
+
detect!
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def exec
|
15
|
+
transition_to :exec
|
16
|
+
end
|
17
|
+
|
18
|
+
def exec_action
|
19
|
+
perform_action(:exec, 'Executing') do
|
20
|
+
exec!
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def verify
|
25
|
+
transition_to :verify
|
26
|
+
end
|
27
|
+
|
28
|
+
def verify_action
|
29
|
+
perform_action(:verify, 'Verifying') do
|
30
|
+
verify!
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def clear
|
35
|
+
# Transitioning would try to load the data we're clearing... let's just
|
36
|
+
# jump straight to the action.
|
37
|
+
clear_action
|
38
|
+
end
|
39
|
+
|
40
|
+
def clear_action
|
41
|
+
perform_action(:clear, 'Clearing') do
|
42
|
+
clear!
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def test(_clear_mode = :passing)
|
47
|
+
elapsed = Benchmark.measure do
|
48
|
+
banner "Cleaning up any prior instances of #{slug}"
|
49
|
+
clear
|
50
|
+
banner "Testing #{slug}"
|
51
|
+
verify
|
52
|
+
# clear if clear_mode == :passing
|
53
|
+
end
|
54
|
+
info "Finished testing #{slug} #{Core::Util.duration(elapsed.real)}."
|
55
|
+
evidence.duration = elapsed.real
|
56
|
+
save
|
57
|
+
evidence = nil # it's saved, free up memory...
|
58
|
+
self
|
59
|
+
# ensure
|
60
|
+
# clear if clear_mode == :always
|
61
|
+
end
|
62
|
+
|
63
|
+
def perform_action(verb, output_verb)
|
64
|
+
banner "#{output_verb} #{slug}..."
|
65
|
+
elapsed = action(verb) { yield }
|
66
|
+
# elapsed = action(verb) { |state| driver.public_send(verb, state) }
|
67
|
+
info("Finished #{output_verb.downcase} #{slug}" \
|
68
|
+
" #{Core::Util.duration(elapsed.real)}.")
|
69
|
+
# yield if block_given?
|
70
|
+
self
|
71
|
+
end
|
72
|
+
|
73
|
+
def action(what, &block)
|
74
|
+
evidence.last_attempted_action = what.to_s
|
75
|
+
elapsed = Benchmark.measure do
|
76
|
+
block.call(@state)
|
77
|
+
end
|
78
|
+
evidence.last_completed_action = what.to_s
|
79
|
+
elapsed
|
80
|
+
rescue FeatureNotImplementedError => e
|
81
|
+
raise e
|
82
|
+
rescue ActionFailed => e
|
83
|
+
log_failure(what, e)
|
84
|
+
raise(ScenarioFailure, failure_message(what) +
|
85
|
+
" Please see .omnitest/logs/#{name}.log for more details",
|
86
|
+
e.backtrace)
|
87
|
+
rescue Exception => e # rubocop:disable RescueException
|
88
|
+
log_failure(what, e)
|
89
|
+
raise ActionFailed,
|
90
|
+
"Failed to complete ##{what} action: [#{e.message}]", e.backtrace
|
91
|
+
ensure
|
92
|
+
save unless what == :clear
|
93
|
+
end
|
94
|
+
|
95
|
+
def transition_to(desired)
|
96
|
+
transition_result = nil
|
97
|
+
begin
|
98
|
+
FSM.actions(last_completed_action, desired).each do |transition|
|
99
|
+
transition_result = send("#{transition}_action")
|
100
|
+
end
|
101
|
+
rescue FeatureNotImplementedError
|
102
|
+
warn("#{slug} is not implemented")
|
103
|
+
rescue ActionFailed => e
|
104
|
+
# Need to use with_friendly_errors again somewhere, since errors don't bubble up
|
105
|
+
# without fast-fail?
|
106
|
+
Omnitest.handle_error(e)
|
107
|
+
raise(ScenarioFailure, e.message, e.backtrace)
|
108
|
+
end
|
109
|
+
transition_result
|
110
|
+
end
|
111
|
+
|
112
|
+
def log_failure(what, e)
|
113
|
+
return unless logger.respond_to? :logdev
|
114
|
+
return if logger.logdev.nil?
|
115
|
+
|
116
|
+
logger.logdev.error(failure_message(what))
|
117
|
+
Error.formatted_trace(e).each { |line| logger.logdev.error(line) }
|
118
|
+
end
|
119
|
+
|
120
|
+
# Returns a string explaining what action failed, at a high level. Used
|
121
|
+
# for displaying to end user.
|
122
|
+
#
|
123
|
+
# @param what [String] an action
|
124
|
+
# @return [String] a failure message
|
125
|
+
# @api private
|
126
|
+
def failure_message(what)
|
127
|
+
"#{what.capitalize} failed for test #{slug}."
|
128
|
+
end
|
129
|
+
|
130
|
+
# The simplest finite state machine pseudo-implementation needed to manage
|
131
|
+
# an Instance.
|
132
|
+
#
|
133
|
+
# @api private
|
134
|
+
class FSM
|
135
|
+
# Returns an Array of all transitions to bring an Instance from its last
|
136
|
+
# reported transistioned state into the desired transitioned state.
|
137
|
+
#
|
138
|
+
# @param last [String,Symbol,nil] the last known transitioned state of
|
139
|
+
# the Instance, defaulting to `nil` (for unknown or no history)
|
140
|
+
# @param desired [String,Symbol] the desired transitioned state for the
|
141
|
+
# Instance
|
142
|
+
# @return [Array<Symbol>] an Array of transition actions to perform
|
143
|
+
# @api private
|
144
|
+
def self.actions(last = nil, desired)
|
145
|
+
last_index = index(last)
|
146
|
+
desired_index = index(desired)
|
147
|
+
|
148
|
+
if last_index == desired_index || last_index > desired_index
|
149
|
+
Array(TRANSITIONS[desired_index])
|
150
|
+
else
|
151
|
+
TRANSITIONS.slice(last_index + 1, desired_index - last_index)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
TRANSITIONS = [:clear, :detect, :exec, :verify]
|
156
|
+
|
157
|
+
# Determines the index of a state in the state lifecycle vector. Woah.
|
158
|
+
#
|
159
|
+
# @param transition [Symbol,#to_sym] a state
|
160
|
+
# @param [Integer] the index position
|
161
|
+
# @api private
|
162
|
+
def self.index(transition)
|
163
|
+
if transition.nil?
|
164
|
+
0
|
165
|
+
else
|
166
|
+
TRANSITIONS.find_index { |t| t == transition.to_sym }
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
class Validation < Omnitest::Core::Dash
|
4
|
+
# TODO: Should we have (expectation) 'failed' vs (unexpected) 'error'?
|
5
|
+
ALLOWABLE_STATES = %w(passed pending failed skipped)
|
6
|
+
|
7
|
+
required_field :status, Symbol
|
8
|
+
field :error, Object
|
9
|
+
field :error_source, Object
|
10
|
+
|
11
|
+
def status=(state)
|
12
|
+
state = state.to_s
|
13
|
+
fail invalidate_state_error unless ALLOWABLE_STATES.include? state
|
14
|
+
super
|
15
|
+
end
|
16
|
+
|
17
|
+
ALLOWABLE_STATES.each do |state|
|
18
|
+
define_method "#{state}?" do
|
19
|
+
status == state?
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def error_source?
|
24
|
+
!error_source.nil?
|
25
|
+
end
|
26
|
+
|
27
|
+
def to_hash(*args)
|
28
|
+
self.error_source = error.error_source if error.respond_to? :error_source
|
29
|
+
super
|
30
|
+
end
|
31
|
+
|
32
|
+
protected
|
33
|
+
|
34
|
+
def invalidate_state_error(state)
|
35
|
+
ArgumentError.new "Invalid status: #{state}, should be one of #{ALLOWABLE_STATES.inspect}"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
class Validator
|
4
|
+
include RSpec::Matchers
|
5
|
+
|
6
|
+
UNIVERSAL_MATCHER = //
|
7
|
+
attr_reader :description, :suite, :scenario, :level, :callback
|
8
|
+
|
9
|
+
def initialize(description, scope = {}, &validator)
|
10
|
+
@description = description
|
11
|
+
@suite = scope[:suite] ||= UNIVERSAL_MATCHER
|
12
|
+
@scenario = scope[:scenario] ||= UNIVERSAL_MATCHER
|
13
|
+
@callback = validator
|
14
|
+
end
|
15
|
+
|
16
|
+
def should_validate?(scenario)
|
17
|
+
# TODO: Case-insensitive matching?
|
18
|
+
!!(@suite.match(scenario.suite.to_s) && @scenario.match(scenario.name.to_s)) # rubocop:disable Style/DoubleNegation
|
19
|
+
end
|
20
|
+
|
21
|
+
def validate(scenario)
|
22
|
+
instance_exec(scenario, &@callback) if should_validate?(scenario)
|
23
|
+
scenario.result.validations[description] = Validation.new(status: :passed)
|
24
|
+
rescue StandardError, RSpec::Expectations::ExpectationNotMetError => e
|
25
|
+
validation = Validation.new(status: :failed, error: ValidationFailure.new(e.message, e))
|
26
|
+
scenario.result.validations[description] = validation
|
27
|
+
end
|
28
|
+
|
29
|
+
def to_s
|
30
|
+
@description
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
|
3
|
+
module Omnitest
|
4
|
+
class Skeptic
|
5
|
+
class ValidatorRegistry
|
6
|
+
include Singleton
|
7
|
+
|
8
|
+
def validators
|
9
|
+
@validator ||= []
|
10
|
+
end
|
11
|
+
|
12
|
+
class << self
|
13
|
+
def validators
|
14
|
+
instance.validators
|
15
|
+
end
|
16
|
+
|
17
|
+
def register(validator, scope = {}, &callback)
|
18
|
+
validator = Validator.new(validator, scope, &callback) if block_given?
|
19
|
+
validators << validator
|
20
|
+
end
|
21
|
+
|
22
|
+
def validators_for(scenario)
|
23
|
+
selected_validators = validators.select { |v| v.should_validate? scenario }
|
24
|
+
selected_validators.empty? ? [Skeptic.configuration.default_validator] : selected_validators
|
25
|
+
end
|
26
|
+
|
27
|
+
def clear
|
28
|
+
validators.clear
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'omnitest/skeptic/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'omnitest-skeptic'
|
8
|
+
spec.version = Omnitest::Skeptic::VERSION
|
9
|
+
spec.authors = ['Max Lincoln']
|
10
|
+
spec.email = ['max@devopsy.com']
|
11
|
+
spec.summary = 'Skeptic tests code samples do what they should.'
|
12
|
+
# spec.description = %q{TODO: Write a longer description. Optional.}
|
13
|
+
spec.homepage = ''
|
14
|
+
spec.license = 'MIT'
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ['lib']
|
20
|
+
|
21
|
+
spec.add_dependency 'omnitest-core', '~> 0'
|
22
|
+
spec.add_dependency 'omnitest-psychic', '~> 0'
|
23
|
+
spec.add_dependency 'hashie', '~> 3.0'
|
24
|
+
spec.add_dependency 'middleware', '~> 0.1'
|
25
|
+
spec.add_development_dependency 'bundler', '~> 1.5'
|
26
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
27
|
+
spec.add_development_dependency 'rake-notes'
|
28
|
+
spec.add_development_dependency 'simplecov'
|
29
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
30
|
+
spec.add_development_dependency 'rubocop', '~> 0.18', '<= 0.27'
|
31
|
+
spec.add_development_dependency 'rubocop-rspec', '~> 1.2'
|
32
|
+
spec.add_development_dependency 'aruba'
|
33
|
+
spec.add_development_dependency 'fabrication', '~> 2.11'
|
34
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# Fabricates test manifests (.omnitest.yaml files)
|
2
|
+
|
3
|
+
Fabricator(:psychic, from: Omnitest::Psychic) do
|
4
|
+
initialize_with do
|
5
|
+
transients = @_transient_attributes.to_hash
|
6
|
+
transients[:name] ||= 'my_sample_project'
|
7
|
+
transients[:cwd] ||= "sdks/#{transients[:name]}"
|
8
|
+
@_klass.new transients
|
9
|
+
end # Hash based initialization
|
10
|
+
transient :name
|
11
|
+
transient :cwd
|
12
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
Fabricator(:validator, from: Omnitest::Skeptic::Validator) do
|
2
|
+
initialize_with do
|
3
|
+
callback = @_transient_attributes.delete :callback
|
4
|
+
desc = @_transient_attributes.delete :description
|
5
|
+
scope = @_transient_attributes
|
6
|
+
@_klass.new(desc, scope, &callback)
|
7
|
+
end # Hash based initialization
|
8
|
+
transient description: 'Sample validator'
|
9
|
+
transient suite: LANGUAGES.sample
|
10
|
+
transient scenario: SCENARIO_NAMES.sample
|
11
|
+
transient callback: Proc.new { Proc.new { |_scenario| } } # rubocop:disable Proc
|
12
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# Content above the snippet is ignored
|
3
|
+
|
4
|
+
print 'Hello, world!'
|
5
|
+
|
6
|
+
# {{snippet factorial}}
|
7
|
+
def factorial(n):
|
8
|
+
if n == 0:
|
9
|
+
return 1
|
10
|
+
else:
|
11
|
+
return n * factorial(n-1)
|
12
|
+
# {{endsnippet}}
|
13
|
+
|
14
|
+
# So is content below the snippet
|
15
|
+
print "{{snippet factorial_result}}"
|
16
|
+
print "The result of factorial(7) is:"
|
17
|
+
print " %d" % factorial(7)
|
18
|
+
print "{{endsnippet}}"
|