omnitest-skeptic 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +6 -0
  3. data/.rspec +2 -0
  4. data/.rubocop.yml +5 -0
  5. data/.rubocop_todo.yml +36 -0
  6. data/Gemfile +24 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +31 -0
  9. data/Rakefile +12 -0
  10. data/bin/skeptic +4 -0
  11. data/lib/omnitest/skeptic.rb +123 -0
  12. data/lib/omnitest/skeptic/cli.rb +156 -0
  13. data/lib/omnitest/skeptic/configuration.rb +48 -0
  14. data/lib/omnitest/skeptic/errors.rb +19 -0
  15. data/lib/omnitest/skeptic/evidence.rb +81 -0
  16. data/lib/omnitest/skeptic/property_definition.rb +8 -0
  17. data/lib/omnitest/skeptic/result.rb +27 -0
  18. data/lib/omnitest/skeptic/scenario.rb +167 -0
  19. data/lib/omnitest/skeptic/scenario_definition.rb +41 -0
  20. data/lib/omnitest/skeptic/spies.rb +43 -0
  21. data/lib/omnitest/skeptic/spy.rb +23 -0
  22. data/lib/omnitest/skeptic/test_manifest.rb +78 -0
  23. data/lib/omnitest/skeptic/test_statuses.rb +63 -0
  24. data/lib/omnitest/skeptic/test_transitions.rb +172 -0
  25. data/lib/omnitest/skeptic/validation.rb +39 -0
  26. data/lib/omnitest/skeptic/validator.rb +34 -0
  27. data/lib/omnitest/skeptic/validator_registry.rb +33 -0
  28. data/lib/omnitest/skeptic/version.rb +5 -0
  29. data/omnitest-skeptic.gemspec +34 -0
  30. data/spec/fabricators/psychic_fabricator.rb +12 -0
  31. data/spec/fabricators/scenario_fabricator.rb +6 -0
  32. data/spec/fabricators/validator_fabricator.rb +12 -0
  33. data/spec/fixtures/factorial.py +18 -0
  34. data/spec/fixtures/skeptic.yaml +16 -0
  35. data/spec/omnitest/skeptic/evidence_spec.rb +58 -0
  36. data/spec/omnitest/skeptic/result_spec.rb +51 -0
  37. data/spec/omnitest/skeptic/scenario_definition_spec.rb +39 -0
  38. data/spec/omnitest/skeptic/scenario_spec.rb +35 -0
  39. data/spec/omnitest/skeptic/test_manifest_spec.rb +28 -0
  40. data/spec/omnitest/skeptic/validator_registry_spec.rb +40 -0
  41. data/spec/omnitest/skeptic/validator_spec.rb +70 -0
  42. data/spec/omnitest/skeptic_spec.rb +65 -0
  43. data/spec/spec_helper.rb +65 -0
  44. metadata +289 -0
@@ -0,0 +1,19 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ # Exception class capturing what caused an scenario to die.
4
+ class ScenarioFailure < TransientFailure; end
5
+
6
+ # Exception class capturing what caused a validation to fail.
7
+ class ValidationFailure < TransientFailure
8
+ include ErrorSource
9
+ end
10
+
11
+ class FeatureNotImplementedError < StandardError
12
+ def initialize(feature)
13
+ super "Feature #{feature} is not implemented"
14
+ end
15
+ end
16
+
17
+ class ScenarioCheckError < StandardError; end
18
+ end
19
+ end
@@ -0,0 +1,81 @@
1
+ require 'pstore'
2
+
3
+ module Omnitest
4
+ class Skeptic
5
+ class Evidence < Omnitest::Core::Dash
6
+ module Persistable
7
+ attr_reader :file
8
+ attr_writer :autosave
9
+
10
+ module ClassMethods
11
+ def load(file, initial_data = {})
12
+ new(file, initial_data).tap(&:reload)
13
+ end
14
+ end
15
+
16
+ def self.included(base)
17
+ base.extend(ClassMethods)
18
+ end
19
+
20
+ def initialize(file, initial_data = {})
21
+ @file = Pathname(file)
22
+ FileUtils.mkdir_p(@file.dirname)
23
+ super initial_data
24
+ end
25
+
26
+ def []=(key, value)
27
+ super
28
+ save if autosave?
29
+ end
30
+
31
+ def autosave?
32
+ @autosave == true
33
+ end
34
+
35
+ def reload
36
+ store.transaction do
37
+ store.roots.each do | key |
38
+ self[key] = store[key]
39
+ end
40
+ end
41
+ end
42
+
43
+ def save
44
+ store.transaction do
45
+ keys.each do | key |
46
+ store[key] = self[key]
47
+ end
48
+ end
49
+ end
50
+
51
+ def clear
52
+ @store = nil
53
+ file.delete
54
+ end
55
+
56
+ private
57
+
58
+ def store
59
+ @store ||= PStore.new(file)
60
+ end
61
+ end
62
+
63
+ include Persistable
64
+
65
+ field :last_attempted_action, String
66
+ field :last_completed_action, String
67
+ field :result, Result
68
+ field :spy_data, Hash, default: {}
69
+ field :error, Object
70
+ field :vars, TestManifest::Environment, default: {}
71
+ field :duration, Numeric
72
+
73
+ # KEYS_TO_PERSIST = [:result, :spy_data, :error, :vars, :duration]
74
+
75
+ # @api private
76
+ def serialize_hash(hash)
77
+ ::YAML.dump(hash)
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,8 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ class PropertyDefinition < Omnitest::Core::Dash # rubocop:disable ClassLength
4
+ field :required, Object, default: false
5
+ field :default, String
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,27 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ class Result < Omnitest::Core::Dash
4
+ extend Forwardable
5
+ field :execution_result, Omnitest::Shell::ExecutionResult
6
+ def_delegators :execution_result, :stdout, :stderr, :exitstatus
7
+ field :source_file, Pathname
8
+ field :data, Hash
9
+ field :validations, Hash[String => Validation], default: {}
10
+
11
+ def successful?
12
+ execution_result.successful?
13
+ end
14
+
15
+ alias_method :success?, :successful?
16
+
17
+ def status
18
+ # A feature can be validated by different suites, or manually vs an automated suite.
19
+ # That's why there's a precedence rather than boolean algebra here...
20
+ return 'failed' if validations.values.any? { |v| v.status == :failed }
21
+ return 'passed' if validations.values.any? { |v| v.status == :passed }
22
+ return 'pending' if validations.values.any? { |v| v.status == :pending }
23
+ 'skipped'
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,167 @@
1
+ require 'benchmark'
2
+
3
+ # TODO: This class really needs to be split-up - and probably renamed.
4
+ #
5
+ # There's a few things happening here:
6
+ # There's the "Scenario" - probably better named "Scenario" - this
7
+ # is *what* we want to test, i.e. "Fog - Upload Directory". It should
8
+ # only rely on parsing omnitest.yaml.
9
+ #
10
+ # Then there's the "Code Sample" - the code to be tested to verify the
11
+ # scenario. This can probably be moved to Psychic, since Psychic finds
12
+ # and executes the code samples.
13
+ #
14
+ # And the result or "State File" - this stores and persists the test
15
+ # results and data captured by spies during test.
16
+ #
17
+ # Finally, there's the driver, including the FSM class at the bottom of
18
+ # this file. It's responsible for managing the test lifecycle.
19
+
20
+ module Omnitest
21
+ class Skeptic
22
+ class Scenario < Omnitest::Core::Dash # rubocop:disable ClassLength
23
+ extend Forwardable
24
+ include Skeptic::TestTransitions
25
+ include Skeptic::TestStatuses
26
+ include Omnitest::Core::FileSystem
27
+ include Omnitest::Core::Logging
28
+ include Omnitest::Core::Util::String
29
+ # View helpers
30
+ include Omnitest::Psychic::Code2Doc::CodeHelper
31
+
32
+ field :scenario_definition, ScenarioDefinition
33
+ required_field :psychic, Omnitest::Psychic
34
+ field :vars, Skeptic::TestManifest::Environment, default: {}
35
+ field :code_sample, Psychic::Script
36
+ field :source_file, Pathname
37
+
38
+ def_delegators :scenario_definition, :name, :suite, :full_name
39
+ def_delegators :psychic, :basedir, :logger
40
+ # def_delegators :code_sample, :source_file, :absolute_source_file, :source
41
+ def_delegators :evidence, :save
42
+ KEYS_TO_PERSIST = [:last_attempted_action, :last_completed_action, :result,
43
+ :spy_data, :error, :duration]
44
+ KEYS_TO_PERSIST.each do |key|
45
+ def_delegators :evidence, key.to_sym, "#{key}=".to_sym
46
+ end
47
+
48
+ attr_reader :slug
49
+
50
+ def initialize(data)
51
+ super
52
+ @slug = slugify(suite, name, psychic.name)
53
+ @evidence_file = Pathname.new(Omnitest.basedir).join('.omnitest', "#{slug}.pstore").expand_path.freeze
54
+ end
55
+
56
+ def evidence(initial_data = {})
57
+ @evidence ||= Skeptic::Evidence.load(@evidence_file, initial_data)
58
+ end
59
+
60
+ def validators
61
+ Omnitest::Skeptic::ValidatorRegistry.validators_for self
62
+ end
63
+
64
+ def code_sample
65
+ self[:code_sample] ||= psychic.script(name)
66
+ rescue Errno::ENOENT
67
+ nil
68
+ end
69
+
70
+ def source_file
71
+ return nil unless code_sample
72
+
73
+ self[:source_file] ||= Pathname(code_sample)
74
+ end
75
+
76
+ def absolute_source_file
77
+ return nil unless code_sample
78
+
79
+ code_sample.absolute_source_file
80
+ end
81
+
82
+ def source
83
+ return nil unless code_sample
84
+
85
+ code_sample.source
86
+ end
87
+
88
+ def code2doc(options = {})
89
+ return nil unless code_sample
90
+
91
+ doc = code_sample.code2doc(options)
92
+ end
93
+
94
+ def detect!
95
+ # fail FeatureNotImplementedError, "Project #{psychic.name} has not been cloned" unless psychic.cloned?
96
+ fail FeatureNotImplementedError, name if source_file.nil?
97
+ self.source_file = Pathname(code_sample)
98
+ rescue Errno::ENOENT
99
+ raise FeatureNotImplementedError, name
100
+ end
101
+
102
+ def exec!
103
+ detect!
104
+ evidence.result = run!
105
+ end
106
+
107
+ def run!(spies = Omnitest::Skeptic::Spies) # rubocop:disable Metrics/AbcSize
108
+ spies.observe(self) do
109
+ if code_sample.params.is_a? String
110
+ code_sample.params = YAML.load(Psychic::Tokens.replace_tokens(code_sample.params, vars))
111
+ else
112
+ code_sample.params = vars
113
+ end
114
+ execution_result = code_sample.execute(env: upcased_hash(vars).merge(ENV.to_hash))
115
+ evidence.result = Skeptic::Result.new(execution_result: execution_result, source_file: source_file.to_s)
116
+ end
117
+ result
118
+ rescue Omnitest::Shell::ExecutionError => e
119
+ execution_error = ExecutionError.new(e)
120
+ execution_error.execution_result = e.execution_result
121
+ evidence.error = Omnitest::Error.formatted_trace(e).join("\n")
122
+ raise execution_error
123
+ rescue => e
124
+ evidence.error = Omnitest::Error.formatted_trace(e).join("\n")
125
+ raise e
126
+ ensure
127
+ save
128
+ end
129
+
130
+ def verify!
131
+ validators.each do |validator|
132
+ validation = validator.validate(self)
133
+ status = case validation.result
134
+ when :passed
135
+ Core::Color.colorize("\u2713 Passed", :green)
136
+ when :failed
137
+ Core::Color.colorize('x Failed', :red)
138
+ Omnitest.handle_validation_failure(validation.error)
139
+ else
140
+ Core::Color.colorize(validation.result, :yellow)
141
+ end
142
+ info format('%-50s %s', validator.description, status)
143
+ end
144
+ end
145
+
146
+ def clear!
147
+ @evidence.clear
148
+ @evidence = nil
149
+ end
150
+
151
+ def validations
152
+ return nil if result.nil?
153
+ result.validations
154
+ end
155
+
156
+ private
157
+
158
+ def upcased_hash(hash)
159
+ new_hash = {}
160
+ hash.each_pair do | key, value |
161
+ new_hash[key.upcase] = value
162
+ end
163
+ new_hash
164
+ end
165
+ end
166
+ end
167
+ end
@@ -0,0 +1,41 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ class ScenarioDefinition < Omnitest::Core::Dash # rubocop:disable ClassLength
4
+ required_field :name, String
5
+ required_field :suite, String, required: true
6
+ field :properties, Hash[String => PropertyDefinition]
7
+ # TODO: Vars will be replaced by properties
8
+ field :vars, Skeptic::TestManifest::Environment, default: {}
9
+ attr_reader :full_name
10
+
11
+ def initialize(data)
12
+ super
13
+ self.vars ||= Skeptic::TestManifest::Environment.new
14
+ @full_name = [suite, name].join(' :: ').freeze
15
+ end
16
+
17
+ def build(project)
18
+ source_file = begin
19
+ file = Core::FileSystem.find_file project.basedir, name
20
+ Core::FileSystem.relativize(file, project.basedir)
21
+ rescue Errno::ENOENT
22
+ nil
23
+ end
24
+ psychic = project.respond_to?(:psychic) ? project.psychic : project
25
+ Scenario.new(psychic: psychic, scenario_definition: self, vars: build_vars, source_file: source_file)
26
+ end
27
+
28
+ private
29
+
30
+ def build_vars
31
+ # TODO: Build vars from properties
32
+ global_vars = begin
33
+ Omnitest.manifest[:global_env].dup
34
+ rescue
35
+ {}
36
+ end
37
+ global_vars.merge(vars.dup)
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,43 @@
1
+ require 'middleware'
2
+
3
+ module Omnitest
4
+ class Skeptic
5
+ module Spies
6
+ class << self
7
+ attr_reader :spies
8
+
9
+ def middleware
10
+ @middleware ||= Middleware::Builder.new
11
+ end
12
+
13
+ def spies
14
+ @spies ||= Set.new
15
+ end
16
+
17
+ def register_spy(spy)
18
+ spies.add(spy)
19
+ middleware.insert 0, spy, {}
20
+ end
21
+
22
+ def observe(scenario, &blk)
23
+ middleware = Middleware::Builder.new
24
+ spies.each do |spy|
25
+ middleware.use spy, Thread.current[:test_env_number]
26
+ end
27
+ middleware.use blk
28
+ middleware.call(scenario)
29
+ end
30
+
31
+ def reports
32
+ # Group by type
33
+ all_reports = spies.flat_map do |spy|
34
+ spy.reports.to_a if spy.respond_to? :reports
35
+ end
36
+ all_reports.each_with_object({}) do |(k, v), h|
37
+ (h[k] ||= []) << v
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,23 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ # # @abstract
4
+ class Spy
5
+ def initialize(app, opts = {})
6
+ @app = app
7
+ @opts = opts
8
+ end
9
+
10
+ def call(_scenario)
11
+ fail NotImplementedError, 'Subclass must implement #call'
12
+ end
13
+
14
+ def self.reports
15
+ @reports ||= {}
16
+ end
17
+
18
+ def self.report(type, report_class)
19
+ reports[type] = report_class
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,78 @@
1
+ require 'yaml'
2
+
3
+ module Omnitest
4
+ class Skeptic
5
+ # Omnitest::TestManifest acts as a test manifest. It defines the test scenarios that should be run,
6
+ # and may be shared across multiple projects when used for a compliance suite.
7
+ #
8
+ # A manifest is generally defined and loaded from YAML. Here's an example manifest:
9
+ # ---
10
+ # global_env:
11
+ # LOCALE: <%= ENV['LANG'] %>
12
+ # FAVORITE_NUMBER: 5
13
+ # suites:
14
+ # Katas:
15
+ # env:
16
+ # NAME: 'Max'
17
+ # samples:
18
+ # - hello world
19
+ # - quine
20
+ # Tutorials:
21
+ # env:
22
+ # samples:
23
+ # - deploying
24
+ # - documenting
25
+ #
26
+ # The *suites* object defines the tests. Each object, under suites, like *Katas* or *Tutorials* in this
27
+ # example, represents a test suite. A test suite is subdivided into *samples*, that each act as a scenario.
28
+ # The *global_env* object and the *env* under each suite define (and standardize) the input for each test.
29
+ # The *global_env* values will be made available to all tests as environment variables, along with the *env*
30
+ # values for that specific test.
31
+ #
32
+ class TestManifest < Omnitest::Core::Dash
33
+ include Core::DefaultLogger
34
+ include Omnitest::Core::Logging
35
+ extend Omnitest::Core::Dash::Loadable
36
+
37
+ class Environment < Omnitest::Core::Mash
38
+ coerce_value Integer, String
39
+ end
40
+
41
+ class Suite < Omnitest::Core::Dash
42
+ field :env, Environment, default: {}
43
+ field :samples, Array[String], required: true
44
+ field :results, Hash
45
+ end
46
+
47
+ field :global_env, Environment
48
+ field :suites, Hash[String => Suite]
49
+
50
+ attr_accessor :scenario_definitions
51
+ attr_accessor :scenarios
52
+
53
+ def scenario_definitions
54
+ @scenario_definitions ||= build_scenario_definitions
55
+ end
56
+
57
+ def build_scenario_definitions
58
+ definitions = Set.new
59
+ suites.each do | suite_name, suite |
60
+ suite.samples.each do | sample_pattern |
61
+ expand_pattern(sample_pattern).each do | sample |
62
+ definitions << ScenarioDefinition.new(name: sample, suite: suite_name, vars: suite.env)
63
+ end
64
+ end
65
+ end
66
+ definitions
67
+ end
68
+
69
+ private
70
+
71
+ def expand_pattern(pattern)
72
+ return [pattern] unless pattern.include? '*'
73
+
74
+ Dir[pattern].to_a
75
+ end
76
+ end
77
+ end
78
+ end