omnitest-skeptic 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +6 -0
  3. data/.rspec +2 -0
  4. data/.rubocop.yml +5 -0
  5. data/.rubocop_todo.yml +36 -0
  6. data/Gemfile +24 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +31 -0
  9. data/Rakefile +12 -0
  10. data/bin/skeptic +4 -0
  11. data/lib/omnitest/skeptic.rb +123 -0
  12. data/lib/omnitest/skeptic/cli.rb +156 -0
  13. data/lib/omnitest/skeptic/configuration.rb +48 -0
  14. data/lib/omnitest/skeptic/errors.rb +19 -0
  15. data/lib/omnitest/skeptic/evidence.rb +81 -0
  16. data/lib/omnitest/skeptic/property_definition.rb +8 -0
  17. data/lib/omnitest/skeptic/result.rb +27 -0
  18. data/lib/omnitest/skeptic/scenario.rb +167 -0
  19. data/lib/omnitest/skeptic/scenario_definition.rb +41 -0
  20. data/lib/omnitest/skeptic/spies.rb +43 -0
  21. data/lib/omnitest/skeptic/spy.rb +23 -0
  22. data/lib/omnitest/skeptic/test_manifest.rb +78 -0
  23. data/lib/omnitest/skeptic/test_statuses.rb +63 -0
  24. data/lib/omnitest/skeptic/test_transitions.rb +172 -0
  25. data/lib/omnitest/skeptic/validation.rb +39 -0
  26. data/lib/omnitest/skeptic/validator.rb +34 -0
  27. data/lib/omnitest/skeptic/validator_registry.rb +33 -0
  28. data/lib/omnitest/skeptic/version.rb +5 -0
  29. data/omnitest-skeptic.gemspec +34 -0
  30. data/spec/fabricators/psychic_fabricator.rb +12 -0
  31. data/spec/fabricators/scenario_fabricator.rb +6 -0
  32. data/spec/fabricators/validator_fabricator.rb +12 -0
  33. data/spec/fixtures/factorial.py +18 -0
  34. data/spec/fixtures/skeptic.yaml +16 -0
  35. data/spec/omnitest/skeptic/evidence_spec.rb +58 -0
  36. data/spec/omnitest/skeptic/result_spec.rb +51 -0
  37. data/spec/omnitest/skeptic/scenario_definition_spec.rb +39 -0
  38. data/spec/omnitest/skeptic/scenario_spec.rb +35 -0
  39. data/spec/omnitest/skeptic/test_manifest_spec.rb +28 -0
  40. data/spec/omnitest/skeptic/validator_registry_spec.rb +40 -0
  41. data/spec/omnitest/skeptic/validator_spec.rb +70 -0
  42. data/spec/omnitest/skeptic_spec.rb +65 -0
  43. data/spec/spec_helper.rb +65 -0
  44. metadata +289 -0
@@ -0,0 +1,63 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ module TestStatuses
4
+ def failed?
5
+ evidence.last_attempted_action != evidence.last_completed_action
6
+ end
7
+
8
+ def skipped?
9
+ result.nil?
10
+ end
11
+
12
+ def sample?
13
+ !source_file.nil?
14
+ end
15
+
16
+ def status
17
+ status = last_attempted_action
18
+ failed? ? "#{status}_failed" : status
19
+ end
20
+
21
+ def status_description
22
+ case status
23
+ when 'clone' then 'Cloned'
24
+ when 'clone_failed' then 'Clone Failed'
25
+ when 'detect' then 'Sample Found'
26
+ when 'detect_failed', nil then '<Not Found>'
27
+ when 'bootstrap' then 'Bootstrapped'
28
+ when 'bootstrap_failed' then 'Bootstrap Failed'
29
+ when 'detect' then 'Detected'
30
+ when 'exec' then 'Executed'
31
+ when 'exec_failed' then 'Execution Failed'
32
+ when 'verify', 'verify_failed'
33
+ validator_count = validators.count
34
+ validation_count = validations.values.select { |v| v.result == :passed }.count
35
+ if validator_count == validation_count
36
+ "Fully Verified (#{validation_count} of #{validator_count})"
37
+ else
38
+ "Partially Verified (#{validation_count} of #{validator_count})"
39
+ end
40
+ # when 'verify_failed' then 'Verification Failed'
41
+ else "<Unknown (#{status})>"
42
+ end
43
+ end
44
+
45
+ def status_color
46
+ case status_description
47
+ when '<Not Found>' then :white
48
+ when 'Cloned' then :magenta
49
+ when 'Bootstrapped' then :magenta
50
+ when 'Sample Found' then :cyan
51
+ when 'Executed' then :blue
52
+ when /Verified/
53
+ if status_description =~ /Fully/
54
+ :green
55
+ else
56
+ :yellow
57
+ end
58
+ else :red
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,172 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ module TestTransitions
4
+ def detect
5
+ transition_to :detect
6
+ end
7
+
8
+ def detect_action
9
+ perform_action(:detect, 'Detecting code sample') do
10
+ detect!
11
+ end
12
+ end
13
+
14
+ def exec
15
+ transition_to :exec
16
+ end
17
+
18
+ def exec_action
19
+ perform_action(:exec, 'Executing') do
20
+ exec!
21
+ end
22
+ end
23
+
24
+ def verify
25
+ transition_to :verify
26
+ end
27
+
28
+ def verify_action
29
+ perform_action(:verify, 'Verifying') do
30
+ verify!
31
+ end
32
+ end
33
+
34
+ def clear
35
+ # Transitioning would try to load the data we're clearing... let's just
36
+ # jump straight to the action.
37
+ clear_action
38
+ end
39
+
40
+ def clear_action
41
+ perform_action(:clear, 'Clearing') do
42
+ clear!
43
+ end
44
+ end
45
+
46
+ def test(_clear_mode = :passing)
47
+ elapsed = Benchmark.measure do
48
+ banner "Cleaning up any prior instances of #{slug}"
49
+ clear
50
+ banner "Testing #{slug}"
51
+ verify
52
+ # clear if clear_mode == :passing
53
+ end
54
+ info "Finished testing #{slug} #{Core::Util.duration(elapsed.real)}."
55
+ evidence.duration = elapsed.real
56
+ save
57
+ evidence = nil # it's saved, free up memory...
58
+ self
59
+ # ensure
60
+ # clear if clear_mode == :always
61
+ end
62
+
63
+ def perform_action(verb, output_verb)
64
+ banner "#{output_verb} #{slug}..."
65
+ elapsed = action(verb) { yield }
66
+ # elapsed = action(verb) { |state| driver.public_send(verb, state) }
67
+ info("Finished #{output_verb.downcase} #{slug}" \
68
+ " #{Core::Util.duration(elapsed.real)}.")
69
+ # yield if block_given?
70
+ self
71
+ end
72
+
73
+ def action(what, &block)
74
+ evidence.last_attempted_action = what.to_s
75
+ elapsed = Benchmark.measure do
76
+ block.call(@state)
77
+ end
78
+ evidence.last_completed_action = what.to_s
79
+ elapsed
80
+ rescue FeatureNotImplementedError => e
81
+ raise e
82
+ rescue ActionFailed => e
83
+ log_failure(what, e)
84
+ raise(ScenarioFailure, failure_message(what) +
85
+ " Please see .omnitest/logs/#{name}.log for more details",
86
+ e.backtrace)
87
+ rescue Exception => e # rubocop:disable RescueException
88
+ log_failure(what, e)
89
+ raise ActionFailed,
90
+ "Failed to complete ##{what} action: [#{e.message}]", e.backtrace
91
+ ensure
92
+ save unless what == :clear
93
+ end
94
+
95
+ def transition_to(desired)
96
+ transition_result = nil
97
+ begin
98
+ FSM.actions(last_completed_action, desired).each do |transition|
99
+ transition_result = send("#{transition}_action")
100
+ end
101
+ rescue FeatureNotImplementedError
102
+ warn("#{slug} is not implemented")
103
+ rescue ActionFailed => e
104
+ # Need to use with_friendly_errors again somewhere, since errors don't bubble up
105
+ # without fast-fail?
106
+ Omnitest.handle_error(e)
107
+ raise(ScenarioFailure, e.message, e.backtrace)
108
+ end
109
+ transition_result
110
+ end
111
+
112
+ def log_failure(what, e)
113
+ return unless logger.respond_to? :logdev
114
+ return if logger.logdev.nil?
115
+
116
+ logger.logdev.error(failure_message(what))
117
+ Error.formatted_trace(e).each { |line| logger.logdev.error(line) }
118
+ end
119
+
120
+ # Returns a string explaining what action failed, at a high level. Used
121
+ # for displaying to end user.
122
+ #
123
+ # @param what [String] an action
124
+ # @return [String] a failure message
125
+ # @api private
126
+ def failure_message(what)
127
+ "#{what.capitalize} failed for test #{slug}."
128
+ end
129
+
130
+ # The simplest finite state machine pseudo-implementation needed to manage
131
+ # an Instance.
132
+ #
133
+ # @api private
134
+ class FSM
135
+ # Returns an Array of all transitions to bring an Instance from its last
136
+ # reported transistioned state into the desired transitioned state.
137
+ #
138
+ # @param last [String,Symbol,nil] the last known transitioned state of
139
+ # the Instance, defaulting to `nil` (for unknown or no history)
140
+ # @param desired [String,Symbol] the desired transitioned state for the
141
+ # Instance
142
+ # @return [Array<Symbol>] an Array of transition actions to perform
143
+ # @api private
144
+ def self.actions(last = nil, desired)
145
+ last_index = index(last)
146
+ desired_index = index(desired)
147
+
148
+ if last_index == desired_index || last_index > desired_index
149
+ Array(TRANSITIONS[desired_index])
150
+ else
151
+ TRANSITIONS.slice(last_index + 1, desired_index - last_index)
152
+ end
153
+ end
154
+
155
+ TRANSITIONS = [:clear, :detect, :exec, :verify]
156
+
157
+ # Determines the index of a state in the state lifecycle vector. Woah.
158
+ #
159
+ # @param transition [Symbol,#to_sym] a state
160
+ # @param [Integer] the index position
161
+ # @api private
162
+ def self.index(transition)
163
+ if transition.nil?
164
+ 0
165
+ else
166
+ TRANSITIONS.find_index { |t| t == transition.to_sym }
167
+ end
168
+ end
169
+ end
170
+ end
171
+ end
172
+ end
@@ -0,0 +1,39 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ class Validation < Omnitest::Core::Dash
4
+ # TODO: Should we have (expectation) 'failed' vs (unexpected) 'error'?
5
+ ALLOWABLE_STATES = %w(passed pending failed skipped)
6
+
7
+ required_field :status, Symbol
8
+ field :error, Object
9
+ field :error_source, Object
10
+
11
+ def status=(state)
12
+ state = state.to_s
13
+ fail invalidate_state_error unless ALLOWABLE_STATES.include? state
14
+ super
15
+ end
16
+
17
+ ALLOWABLE_STATES.each do |state|
18
+ define_method "#{state}?" do
19
+ status == state?
20
+ end
21
+ end
22
+
23
+ def error_source?
24
+ !error_source.nil?
25
+ end
26
+
27
+ def to_hash(*args)
28
+ self.error_source = error.error_source if error.respond_to? :error_source
29
+ super
30
+ end
31
+
32
+ protected
33
+
34
+ def invalidate_state_error(state)
35
+ ArgumentError.new "Invalid status: #{state}, should be one of #{ALLOWABLE_STATES.inspect}"
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,34 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ class Validator
4
+ include RSpec::Matchers
5
+
6
+ UNIVERSAL_MATCHER = //
7
+ attr_reader :description, :suite, :scenario, :level, :callback
8
+
9
+ def initialize(description, scope = {}, &validator)
10
+ @description = description
11
+ @suite = scope[:suite] ||= UNIVERSAL_MATCHER
12
+ @scenario = scope[:scenario] ||= UNIVERSAL_MATCHER
13
+ @callback = validator
14
+ end
15
+
16
+ def should_validate?(scenario)
17
+ # TODO: Case-insensitive matching?
18
+ !!(@suite.match(scenario.suite.to_s) && @scenario.match(scenario.name.to_s)) # rubocop:disable Style/DoubleNegation
19
+ end
20
+
21
+ def validate(scenario)
22
+ instance_exec(scenario, &@callback) if should_validate?(scenario)
23
+ scenario.result.validations[description] = Validation.new(status: :passed)
24
+ rescue StandardError, RSpec::Expectations::ExpectationNotMetError => e
25
+ validation = Validation.new(status: :failed, error: ValidationFailure.new(e.message, e))
26
+ scenario.result.validations[description] = validation
27
+ end
28
+
29
+ def to_s
30
+ @description
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,33 @@
1
+ require 'singleton'
2
+
3
+ module Omnitest
4
+ class Skeptic
5
+ class ValidatorRegistry
6
+ include Singleton
7
+
8
+ def validators
9
+ @validator ||= []
10
+ end
11
+
12
+ class << self
13
+ def validators
14
+ instance.validators
15
+ end
16
+
17
+ def register(validator, scope = {}, &callback)
18
+ validator = Validator.new(validator, scope, &callback) if block_given?
19
+ validators << validator
20
+ end
21
+
22
+ def validators_for(scenario)
23
+ selected_validators = validators.select { |v| v.should_validate? scenario }
24
+ selected_validators.empty? ? [Skeptic.configuration.default_validator] : selected_validators
25
+ end
26
+
27
+ def clear
28
+ validators.clear
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,5 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ VERSION = '0.0.2'
4
+ end
5
+ end
@@ -0,0 +1,34 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'omnitest/skeptic/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'omnitest-skeptic'
8
+ spec.version = Omnitest::Skeptic::VERSION
9
+ spec.authors = ['Max Lincoln']
10
+ spec.email = ['max@devopsy.com']
11
+ spec.summary = 'Skeptic tests code samples do what they should.'
12
+ # spec.description = %q{TODO: Write a longer description. Optional.}
13
+ spec.homepage = ''
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_dependency 'omnitest-core', '~> 0'
22
+ spec.add_dependency 'omnitest-psychic', '~> 0'
23
+ spec.add_dependency 'hashie', '~> 3.0'
24
+ spec.add_dependency 'middleware', '~> 0.1'
25
+ spec.add_development_dependency 'bundler', '~> 1.5'
26
+ spec.add_development_dependency 'rake', '~> 10.0'
27
+ spec.add_development_dependency 'rake-notes'
28
+ spec.add_development_dependency 'simplecov'
29
+ spec.add_development_dependency 'rspec', '~> 3.0'
30
+ spec.add_development_dependency 'rubocop', '~> 0.18', '<= 0.27'
31
+ spec.add_development_dependency 'rubocop-rspec', '~> 1.2'
32
+ spec.add_development_dependency 'aruba'
33
+ spec.add_development_dependency 'fabrication', '~> 2.11'
34
+ end
@@ -0,0 +1,12 @@
1
+ # Fabricates test manifests (.omnitest.yaml files)
2
+
3
+ Fabricator(:psychic, from: Omnitest::Psychic) do
4
+ initialize_with do
5
+ transients = @_transient_attributes.to_hash
6
+ transients[:name] ||= 'my_sample_project'
7
+ transients[:cwd] ||= "sdks/#{transients[:name]}"
8
+ @_klass.new transients
9
+ end # Hash based initialization
10
+ transient :name
11
+ transient :cwd
12
+ end
@@ -0,0 +1,6 @@
1
+ Fabricator(:scenario_definition, from: Omnitest::Skeptic::ScenarioDefinition) do
2
+ initialize_with { @_klass.new to_hash } # Hash based initialization
3
+ name { SCENARIO_NAMES.sample }
4
+ suite { LANGUAGES.sample }
5
+ properties {}
6
+ end
@@ -0,0 +1,12 @@
1
+ Fabricator(:validator, from: Omnitest::Skeptic::Validator) do
2
+ initialize_with do
3
+ callback = @_transient_attributes.delete :callback
4
+ desc = @_transient_attributes.delete :description
5
+ scope = @_transient_attributes
6
+ @_klass.new(desc, scope, &callback)
7
+ end # Hash based initialization
8
+ transient description: 'Sample validator'
9
+ transient suite: LANGUAGES.sample
10
+ transient scenario: SCENARIO_NAMES.sample
11
+ transient callback: Proc.new { Proc.new { |_scenario| } } # rubocop:disable Proc
12
+ end
@@ -0,0 +1,18 @@
1
+ #!/usr/bin/env python
2
+ # Content above the snippet is ignored
3
+
4
+ print 'Hello, world!'
5
+
6
+ # {{snippet factorial}}
7
+ def factorial(n):
8
+ if n == 0:
9
+ return 1
10
+ else:
11
+ return n * factorial(n-1)
12
+ # {{endsnippet}}
13
+
14
+ # So is content below the snippet
15
+ print "{{snippet factorial_result}}"
16
+ print "The result of factorial(7) is:"
17
+ print " %d" % factorial(7)
18
+ print "{{endsnippet}}"
@@ -0,0 +1,16 @@
1
+ ---
2
+ global_env:
3
+ LOCALE: <%= ENV['LANG'] %>
4
+ FAVORITE_NUMBER: 5
5
+ suites:
6
+ Katas:
7
+ env:
8
+ NAME: 'Max'
9
+ samples:
10
+ - hello world
11
+ - quine
12
+ Tutorials:
13
+ env:
14
+ samples:
15
+ - deploying
16
+ - documenting