omnitest-skeptic 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +6 -0
- data/.rspec +2 -0
- data/.rubocop.yml +5 -0
- data/.rubocop_todo.yml +36 -0
- data/Gemfile +24 -0
- data/LICENSE.txt +22 -0
- data/README.md +31 -0
- data/Rakefile +12 -0
- data/bin/skeptic +4 -0
- data/lib/omnitest/skeptic.rb +123 -0
- data/lib/omnitest/skeptic/cli.rb +156 -0
- data/lib/omnitest/skeptic/configuration.rb +48 -0
- data/lib/omnitest/skeptic/errors.rb +19 -0
- data/lib/omnitest/skeptic/evidence.rb +81 -0
- data/lib/omnitest/skeptic/property_definition.rb +8 -0
- data/lib/omnitest/skeptic/result.rb +27 -0
- data/lib/omnitest/skeptic/scenario.rb +167 -0
- data/lib/omnitest/skeptic/scenario_definition.rb +41 -0
- data/lib/omnitest/skeptic/spies.rb +43 -0
- data/lib/omnitest/skeptic/spy.rb +23 -0
- data/lib/omnitest/skeptic/test_manifest.rb +78 -0
- data/lib/omnitest/skeptic/test_statuses.rb +63 -0
- data/lib/omnitest/skeptic/test_transitions.rb +172 -0
- data/lib/omnitest/skeptic/validation.rb +39 -0
- data/lib/omnitest/skeptic/validator.rb +34 -0
- data/lib/omnitest/skeptic/validator_registry.rb +33 -0
- data/lib/omnitest/skeptic/version.rb +5 -0
- data/omnitest-skeptic.gemspec +34 -0
- data/spec/fabricators/psychic_fabricator.rb +12 -0
- data/spec/fabricators/scenario_fabricator.rb +6 -0
- data/spec/fabricators/validator_fabricator.rb +12 -0
- data/spec/fixtures/factorial.py +18 -0
- data/spec/fixtures/skeptic.yaml +16 -0
- data/spec/omnitest/skeptic/evidence_spec.rb +58 -0
- data/spec/omnitest/skeptic/result_spec.rb +51 -0
- data/spec/omnitest/skeptic/scenario_definition_spec.rb +39 -0
- data/spec/omnitest/skeptic/scenario_spec.rb +35 -0
- data/spec/omnitest/skeptic/test_manifest_spec.rb +28 -0
- data/spec/omnitest/skeptic/validator_registry_spec.rb +40 -0
- data/spec/omnitest/skeptic/validator_spec.rb +70 -0
- data/spec/omnitest/skeptic_spec.rb +65 -0
- data/spec/spec_helper.rb +65 -0
- metadata +289 -0
@@ -0,0 +1,19 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
# Exception class capturing what caused an scenario to die.
|
4
|
+
class ScenarioFailure < TransientFailure; end
|
5
|
+
|
6
|
+
# Exception class capturing what caused a validation to fail.
|
7
|
+
class ValidationFailure < TransientFailure
|
8
|
+
include ErrorSource
|
9
|
+
end
|
10
|
+
|
11
|
+
class FeatureNotImplementedError < StandardError
|
12
|
+
def initialize(feature)
|
13
|
+
super "Feature #{feature} is not implemented"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class ScenarioCheckError < StandardError; end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'pstore'
|
2
|
+
|
3
|
+
module Omnitest
|
4
|
+
class Skeptic
|
5
|
+
class Evidence < Omnitest::Core::Dash
|
6
|
+
module Persistable
|
7
|
+
attr_reader :file
|
8
|
+
attr_writer :autosave
|
9
|
+
|
10
|
+
module ClassMethods
|
11
|
+
def load(file, initial_data = {})
|
12
|
+
new(file, initial_data).tap(&:reload)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.included(base)
|
17
|
+
base.extend(ClassMethods)
|
18
|
+
end
|
19
|
+
|
20
|
+
def initialize(file, initial_data = {})
|
21
|
+
@file = Pathname(file)
|
22
|
+
FileUtils.mkdir_p(@file.dirname)
|
23
|
+
super initial_data
|
24
|
+
end
|
25
|
+
|
26
|
+
def []=(key, value)
|
27
|
+
super
|
28
|
+
save if autosave?
|
29
|
+
end
|
30
|
+
|
31
|
+
def autosave?
|
32
|
+
@autosave == true
|
33
|
+
end
|
34
|
+
|
35
|
+
def reload
|
36
|
+
store.transaction do
|
37
|
+
store.roots.each do | key |
|
38
|
+
self[key] = store[key]
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def save
|
44
|
+
store.transaction do
|
45
|
+
keys.each do | key |
|
46
|
+
store[key] = self[key]
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def clear
|
52
|
+
@store = nil
|
53
|
+
file.delete
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def store
|
59
|
+
@store ||= PStore.new(file)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
include Persistable
|
64
|
+
|
65
|
+
field :last_attempted_action, String
|
66
|
+
field :last_completed_action, String
|
67
|
+
field :result, Result
|
68
|
+
field :spy_data, Hash, default: {}
|
69
|
+
field :error, Object
|
70
|
+
field :vars, TestManifest::Environment, default: {}
|
71
|
+
field :duration, Numeric
|
72
|
+
|
73
|
+
# KEYS_TO_PERSIST = [:result, :spy_data, :error, :vars, :duration]
|
74
|
+
|
75
|
+
# @api private
|
76
|
+
def serialize_hash(hash)
|
77
|
+
::YAML.dump(hash)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
class Result < Omnitest::Core::Dash
|
4
|
+
extend Forwardable
|
5
|
+
field :execution_result, Omnitest::Shell::ExecutionResult
|
6
|
+
def_delegators :execution_result, :stdout, :stderr, :exitstatus
|
7
|
+
field :source_file, Pathname
|
8
|
+
field :data, Hash
|
9
|
+
field :validations, Hash[String => Validation], default: {}
|
10
|
+
|
11
|
+
def successful?
|
12
|
+
execution_result.successful?
|
13
|
+
end
|
14
|
+
|
15
|
+
alias_method :success?, :successful?
|
16
|
+
|
17
|
+
def status
|
18
|
+
# A feature can be validated by different suites, or manually vs an automated suite.
|
19
|
+
# That's why there's a precedence rather than boolean algebra here...
|
20
|
+
return 'failed' if validations.values.any? { |v| v.status == :failed }
|
21
|
+
return 'passed' if validations.values.any? { |v| v.status == :passed }
|
22
|
+
return 'pending' if validations.values.any? { |v| v.status == :pending }
|
23
|
+
'skipped'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,167 @@
|
|
1
|
+
require 'benchmark'
|
2
|
+
|
3
|
+
# TODO: This class really needs to be split-up - and probably renamed.
|
4
|
+
#
|
5
|
+
# There's a few things happening here:
|
6
|
+
# There's the "Scenario" - probably better named "Scenario" - this
|
7
|
+
# is *what* we want to test, i.e. "Fog - Upload Directory". It should
|
8
|
+
# only rely on parsing omnitest.yaml.
|
9
|
+
#
|
10
|
+
# Then there's the "Code Sample" - the code to be tested to verify the
|
11
|
+
# scenario. This can probably be moved to Psychic, since Psychic finds
|
12
|
+
# and executes the code samples.
|
13
|
+
#
|
14
|
+
# And the result or "State File" - this stores and persists the test
|
15
|
+
# results and data captured by spies during test.
|
16
|
+
#
|
17
|
+
# Finally, there's the driver, including the FSM class at the bottom of
|
18
|
+
# this file. It's responsible for managing the test lifecycle.
|
19
|
+
|
20
|
+
module Omnitest
|
21
|
+
class Skeptic
|
22
|
+
class Scenario < Omnitest::Core::Dash # rubocop:disable ClassLength
|
23
|
+
extend Forwardable
|
24
|
+
include Skeptic::TestTransitions
|
25
|
+
include Skeptic::TestStatuses
|
26
|
+
include Omnitest::Core::FileSystem
|
27
|
+
include Omnitest::Core::Logging
|
28
|
+
include Omnitest::Core::Util::String
|
29
|
+
# View helpers
|
30
|
+
include Omnitest::Psychic::Code2Doc::CodeHelper
|
31
|
+
|
32
|
+
field :scenario_definition, ScenarioDefinition
|
33
|
+
required_field :psychic, Omnitest::Psychic
|
34
|
+
field :vars, Skeptic::TestManifest::Environment, default: {}
|
35
|
+
field :code_sample, Psychic::Script
|
36
|
+
field :source_file, Pathname
|
37
|
+
|
38
|
+
def_delegators :scenario_definition, :name, :suite, :full_name
|
39
|
+
def_delegators :psychic, :basedir, :logger
|
40
|
+
# def_delegators :code_sample, :source_file, :absolute_source_file, :source
|
41
|
+
def_delegators :evidence, :save
|
42
|
+
KEYS_TO_PERSIST = [:last_attempted_action, :last_completed_action, :result,
|
43
|
+
:spy_data, :error, :duration]
|
44
|
+
KEYS_TO_PERSIST.each do |key|
|
45
|
+
def_delegators :evidence, key.to_sym, "#{key}=".to_sym
|
46
|
+
end
|
47
|
+
|
48
|
+
attr_reader :slug
|
49
|
+
|
50
|
+
def initialize(data)
|
51
|
+
super
|
52
|
+
@slug = slugify(suite, name, psychic.name)
|
53
|
+
@evidence_file = Pathname.new(Omnitest.basedir).join('.omnitest', "#{slug}.pstore").expand_path.freeze
|
54
|
+
end
|
55
|
+
|
56
|
+
def evidence(initial_data = {})
|
57
|
+
@evidence ||= Skeptic::Evidence.load(@evidence_file, initial_data)
|
58
|
+
end
|
59
|
+
|
60
|
+
def validators
|
61
|
+
Omnitest::Skeptic::ValidatorRegistry.validators_for self
|
62
|
+
end
|
63
|
+
|
64
|
+
def code_sample
|
65
|
+
self[:code_sample] ||= psychic.script(name)
|
66
|
+
rescue Errno::ENOENT
|
67
|
+
nil
|
68
|
+
end
|
69
|
+
|
70
|
+
def source_file
|
71
|
+
return nil unless code_sample
|
72
|
+
|
73
|
+
self[:source_file] ||= Pathname(code_sample)
|
74
|
+
end
|
75
|
+
|
76
|
+
def absolute_source_file
|
77
|
+
return nil unless code_sample
|
78
|
+
|
79
|
+
code_sample.absolute_source_file
|
80
|
+
end
|
81
|
+
|
82
|
+
def source
|
83
|
+
return nil unless code_sample
|
84
|
+
|
85
|
+
code_sample.source
|
86
|
+
end
|
87
|
+
|
88
|
+
def code2doc(options = {})
|
89
|
+
return nil unless code_sample
|
90
|
+
|
91
|
+
doc = code_sample.code2doc(options)
|
92
|
+
end
|
93
|
+
|
94
|
+
def detect!
|
95
|
+
# fail FeatureNotImplementedError, "Project #{psychic.name} has not been cloned" unless psychic.cloned?
|
96
|
+
fail FeatureNotImplementedError, name if source_file.nil?
|
97
|
+
self.source_file = Pathname(code_sample)
|
98
|
+
rescue Errno::ENOENT
|
99
|
+
raise FeatureNotImplementedError, name
|
100
|
+
end
|
101
|
+
|
102
|
+
def exec!
|
103
|
+
detect!
|
104
|
+
evidence.result = run!
|
105
|
+
end
|
106
|
+
|
107
|
+
def run!(spies = Omnitest::Skeptic::Spies) # rubocop:disable Metrics/AbcSize
|
108
|
+
spies.observe(self) do
|
109
|
+
if code_sample.params.is_a? String
|
110
|
+
code_sample.params = YAML.load(Psychic::Tokens.replace_tokens(code_sample.params, vars))
|
111
|
+
else
|
112
|
+
code_sample.params = vars
|
113
|
+
end
|
114
|
+
execution_result = code_sample.execute(env: upcased_hash(vars).merge(ENV.to_hash))
|
115
|
+
evidence.result = Skeptic::Result.new(execution_result: execution_result, source_file: source_file.to_s)
|
116
|
+
end
|
117
|
+
result
|
118
|
+
rescue Omnitest::Shell::ExecutionError => e
|
119
|
+
execution_error = ExecutionError.new(e)
|
120
|
+
execution_error.execution_result = e.execution_result
|
121
|
+
evidence.error = Omnitest::Error.formatted_trace(e).join("\n")
|
122
|
+
raise execution_error
|
123
|
+
rescue => e
|
124
|
+
evidence.error = Omnitest::Error.formatted_trace(e).join("\n")
|
125
|
+
raise e
|
126
|
+
ensure
|
127
|
+
save
|
128
|
+
end
|
129
|
+
|
130
|
+
def verify!
|
131
|
+
validators.each do |validator|
|
132
|
+
validation = validator.validate(self)
|
133
|
+
status = case validation.result
|
134
|
+
when :passed
|
135
|
+
Core::Color.colorize("\u2713 Passed", :green)
|
136
|
+
when :failed
|
137
|
+
Core::Color.colorize('x Failed', :red)
|
138
|
+
Omnitest.handle_validation_failure(validation.error)
|
139
|
+
else
|
140
|
+
Core::Color.colorize(validation.result, :yellow)
|
141
|
+
end
|
142
|
+
info format('%-50s %s', validator.description, status)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def clear!
|
147
|
+
@evidence.clear
|
148
|
+
@evidence = nil
|
149
|
+
end
|
150
|
+
|
151
|
+
def validations
|
152
|
+
return nil if result.nil?
|
153
|
+
result.validations
|
154
|
+
end
|
155
|
+
|
156
|
+
private
|
157
|
+
|
158
|
+
def upcased_hash(hash)
|
159
|
+
new_hash = {}
|
160
|
+
hash.each_pair do | key, value |
|
161
|
+
new_hash[key.upcase] = value
|
162
|
+
end
|
163
|
+
new_hash
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
class ScenarioDefinition < Omnitest::Core::Dash # rubocop:disable ClassLength
|
4
|
+
required_field :name, String
|
5
|
+
required_field :suite, String, required: true
|
6
|
+
field :properties, Hash[String => PropertyDefinition]
|
7
|
+
# TODO: Vars will be replaced by properties
|
8
|
+
field :vars, Skeptic::TestManifest::Environment, default: {}
|
9
|
+
attr_reader :full_name
|
10
|
+
|
11
|
+
def initialize(data)
|
12
|
+
super
|
13
|
+
self.vars ||= Skeptic::TestManifest::Environment.new
|
14
|
+
@full_name = [suite, name].join(' :: ').freeze
|
15
|
+
end
|
16
|
+
|
17
|
+
def build(project)
|
18
|
+
source_file = begin
|
19
|
+
file = Core::FileSystem.find_file project.basedir, name
|
20
|
+
Core::FileSystem.relativize(file, project.basedir)
|
21
|
+
rescue Errno::ENOENT
|
22
|
+
nil
|
23
|
+
end
|
24
|
+
psychic = project.respond_to?(:psychic) ? project.psychic : project
|
25
|
+
Scenario.new(psychic: psychic, scenario_definition: self, vars: build_vars, source_file: source_file)
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def build_vars
|
31
|
+
# TODO: Build vars from properties
|
32
|
+
global_vars = begin
|
33
|
+
Omnitest.manifest[:global_env].dup
|
34
|
+
rescue
|
35
|
+
{}
|
36
|
+
end
|
37
|
+
global_vars.merge(vars.dup)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'middleware'
|
2
|
+
|
3
|
+
module Omnitest
|
4
|
+
class Skeptic
|
5
|
+
module Spies
|
6
|
+
class << self
|
7
|
+
attr_reader :spies
|
8
|
+
|
9
|
+
def middleware
|
10
|
+
@middleware ||= Middleware::Builder.new
|
11
|
+
end
|
12
|
+
|
13
|
+
def spies
|
14
|
+
@spies ||= Set.new
|
15
|
+
end
|
16
|
+
|
17
|
+
def register_spy(spy)
|
18
|
+
spies.add(spy)
|
19
|
+
middleware.insert 0, spy, {}
|
20
|
+
end
|
21
|
+
|
22
|
+
def observe(scenario, &blk)
|
23
|
+
middleware = Middleware::Builder.new
|
24
|
+
spies.each do |spy|
|
25
|
+
middleware.use spy, Thread.current[:test_env_number]
|
26
|
+
end
|
27
|
+
middleware.use blk
|
28
|
+
middleware.call(scenario)
|
29
|
+
end
|
30
|
+
|
31
|
+
def reports
|
32
|
+
# Group by type
|
33
|
+
all_reports = spies.flat_map do |spy|
|
34
|
+
spy.reports.to_a if spy.respond_to? :reports
|
35
|
+
end
|
36
|
+
all_reports.each_with_object({}) do |(k, v), h|
|
37
|
+
(h[k] ||= []) << v
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
# # @abstract
|
4
|
+
class Spy
|
5
|
+
def initialize(app, opts = {})
|
6
|
+
@app = app
|
7
|
+
@opts = opts
|
8
|
+
end
|
9
|
+
|
10
|
+
def call(_scenario)
|
11
|
+
fail NotImplementedError, 'Subclass must implement #call'
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.reports
|
15
|
+
@reports ||= {}
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.report(type, report_class)
|
19
|
+
reports[type] = report_class
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Omnitest
|
4
|
+
class Skeptic
|
5
|
+
# Omnitest::TestManifest acts as a test manifest. It defines the test scenarios that should be run,
|
6
|
+
# and may be shared across multiple projects when used for a compliance suite.
|
7
|
+
#
|
8
|
+
# A manifest is generally defined and loaded from YAML. Here's an example manifest:
|
9
|
+
# ---
|
10
|
+
# global_env:
|
11
|
+
# LOCALE: <%= ENV['LANG'] %>
|
12
|
+
# FAVORITE_NUMBER: 5
|
13
|
+
# suites:
|
14
|
+
# Katas:
|
15
|
+
# env:
|
16
|
+
# NAME: 'Max'
|
17
|
+
# samples:
|
18
|
+
# - hello world
|
19
|
+
# - quine
|
20
|
+
# Tutorials:
|
21
|
+
# env:
|
22
|
+
# samples:
|
23
|
+
# - deploying
|
24
|
+
# - documenting
|
25
|
+
#
|
26
|
+
# The *suites* object defines the tests. Each object, under suites, like *Katas* or *Tutorials* in this
|
27
|
+
# example, represents a test suite. A test suite is subdivided into *samples*, that each act as a scenario.
|
28
|
+
# The *global_env* object and the *env* under each suite define (and standardize) the input for each test.
|
29
|
+
# The *global_env* values will be made available to all tests as environment variables, along with the *env*
|
30
|
+
# values for that specific test.
|
31
|
+
#
|
32
|
+
class TestManifest < Omnitest::Core::Dash
|
33
|
+
include Core::DefaultLogger
|
34
|
+
include Omnitest::Core::Logging
|
35
|
+
extend Omnitest::Core::Dash::Loadable
|
36
|
+
|
37
|
+
class Environment < Omnitest::Core::Mash
|
38
|
+
coerce_value Integer, String
|
39
|
+
end
|
40
|
+
|
41
|
+
class Suite < Omnitest::Core::Dash
|
42
|
+
field :env, Environment, default: {}
|
43
|
+
field :samples, Array[String], required: true
|
44
|
+
field :results, Hash
|
45
|
+
end
|
46
|
+
|
47
|
+
field :global_env, Environment
|
48
|
+
field :suites, Hash[String => Suite]
|
49
|
+
|
50
|
+
attr_accessor :scenario_definitions
|
51
|
+
attr_accessor :scenarios
|
52
|
+
|
53
|
+
def scenario_definitions
|
54
|
+
@scenario_definitions ||= build_scenario_definitions
|
55
|
+
end
|
56
|
+
|
57
|
+
def build_scenario_definitions
|
58
|
+
definitions = Set.new
|
59
|
+
suites.each do | suite_name, suite |
|
60
|
+
suite.samples.each do | sample_pattern |
|
61
|
+
expand_pattern(sample_pattern).each do | sample |
|
62
|
+
definitions << ScenarioDefinition.new(name: sample, suite: suite_name, vars: suite.env)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
definitions
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
|
71
|
+
def expand_pattern(pattern)
|
72
|
+
return [pattern] unless pattern.include? '*'
|
73
|
+
|
74
|
+
Dir[pattern].to_a
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|