omnitest-skeptic 0.0.2

data/lib/omnitest/skeptic/errors.rb

@@ -0,0 +1,19 @@
+module Omnitest
+  class Skeptic
+    # Exception class capturing what caused an scenario to die.
+    class ScenarioFailure < TransientFailure; end
+
+    # Exception class capturing what caused a validation to fail.
+    class ValidationFailure < TransientFailure
+      include ErrorSource
+    end
+
+    class FeatureNotImplementedError < StandardError
+      def initialize(feature)
+        super "Feature #{feature} is not implemented"
+      end
+    end
+
+    class ScenarioCheckError < StandardError; end
+  end
+end

data/lib/omnitest/skeptic/evidence.rb

@@ -0,0 +1,81 @@
+require 'pstore'
+
+module Omnitest
+  class Skeptic
+    class Evidence < Omnitest::Core::Dash
+      module Persistable
+        attr_reader :file
+        attr_writer :autosave
+
+        module ClassMethods
+          def load(file, initial_data = {})
+            new(file, initial_data).tap(&:reload)
+          end
+        end
+
+        def self.included(base)
+          base.extend(ClassMethods)
+        end
+
+        def initialize(file, initial_data = {})
+          @file = Pathname(file)
+          FileUtils.mkdir_p(@file.dirname)
+          super initial_data
+        end
+
+        def []=(key, value)
+          super
+          save if autosave?
+        end
+
+        def autosave?
+          @autosave == true
+        end
+
+        def reload
+          store.transaction do
+            store.roots.each do | key |
+              self[key] = store[key]
+            end
+          end
+        end
+
+        def save
+          store.transaction do
+            keys.each do | key |
+              store[key] = self[key]
+            end
+          end
+        end
+
+        def clear
+          @store = nil
+          file.delete
+        end
+
+        private
+
+        def store
+          @store ||= PStore.new(file)
+        end
+      end
+
+      include Persistable
+
+      field :last_attempted_action, String
+      field :last_completed_action, String
+      field :result, Result
+      field :spy_data, Hash, default: {}
+      field :error, Object
+      field :vars, TestManifest::Environment, default: {}
+      field :duration, Numeric
+
+      # KEYS_TO_PERSIST = [:result, :spy_data, :error, :vars, :duration]
+
+      # @api private
+      def serialize_hash(hash)
+        ::YAML.dump(hash)
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/property_definition.rb

@@ -0,0 +1,8 @@
+module Omnitest
+  class Skeptic
+    class PropertyDefinition < Omnitest::Core::Dash # rubocop:disable ClassLength
+      field :required, Object, default: false
+      field :default, String
+    end
+  end
+end

data/lib/omnitest/skeptic/result.rb

@@ -0,0 +1,27 @@
+module Omnitest
+  class Skeptic
+    class Result < Omnitest::Core::Dash
+      extend Forwardable
+      field :execution_result, Omnitest::Shell::ExecutionResult
+      def_delegators :execution_result, :stdout, :stderr, :exitstatus
+      field :source_file, Pathname
+      field :data, Hash
+      field :validations, Hash[String => Validation], default: {}
+
+      def successful?
+        execution_result.successful?
+      end
+
+      alias_method :success?, :successful?
+
+      def status
+        # A feature can be validated by different suites, or manually vs an automated suite.
+        # That's why there's a precedence rather than boolean algebra here...
+        return 'failed' if validations.values.any? { |v| v.status == :failed }
+        return 'passed' if validations.values.any? { |v| v.status == :passed }
+        return 'pending' if validations.values.any? { |v| v.status == :pending }
+        'skipped'
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/scenario.rb

@@ -0,0 +1,167 @@
+require 'benchmark'
+
+# TODO: This class really needs to be split-up - and probably renamed.
+#
+# There's a few things happening here:
+#   There's the "Scenario" - probably better named "Scenario" - this
+#   is *what* we want to test, i.e. "Fog - Upload Directory". It should
+#   only rely on parsing omnitest.yaml.
+#
+#   Then there's the "Code Sample" - the code to be tested to verify the
+#   scenario. This can probably be moved to Psychic, since Psychic finds
+#   and executes the code samples.
+#
+#   And the result or "State File" - this stores and persists the test
+#   results and data captured by spies during test.
+#
+#   Finally, there's the driver, including the FSM class at the bottom of
+#   this file. It's responsible for managing the test lifecycle.
+
+module Omnitest
+  class Skeptic
+    class Scenario < Omnitest::Core::Dash # rubocop:disable ClassLength
+      extend Forwardable
+      include Skeptic::TestTransitions
+      include Skeptic::TestStatuses
+      include Omnitest::Core::FileSystem
+      include Omnitest::Core::Logging
+      include Omnitest::Core::Util::String
+      # View helpers
+      include Omnitest::Psychic::Code2Doc::CodeHelper
+
+      field :scenario_definition, ScenarioDefinition
+      required_field :psychic, Omnitest::Psychic
+      field :vars, Skeptic::TestManifest::Environment, default: {}
+      field :code_sample, Psychic::Script
+      field :source_file, Pathname
+
+      def_delegators :scenario_definition, :name, :suite, :full_name
+      def_delegators :psychic, :basedir, :logger
+      # def_delegators :code_sample, :source_file, :absolute_source_file, :source
+      def_delegators :evidence, :save
+      KEYS_TO_PERSIST = [:last_attempted_action, :last_completed_action, :result,
+                         :spy_data, :error, :duration]
+      KEYS_TO_PERSIST.each do |key|
+        def_delegators :evidence, key.to_sym, "#{key}=".to_sym
+      end
+
+      attr_reader :slug
+
+      def initialize(data)
+        super
+        @slug = slugify(suite, name, psychic.name)
+        @evidence_file = Pathname.new(Omnitest.basedir).join('.omnitest', "#{slug}.pstore").expand_path.freeze
+      end
+
+      def evidence(initial_data = {})
+        @evidence ||= Skeptic::Evidence.load(@evidence_file, initial_data)
+      end
+
+      def validators
+        Omnitest::Skeptic::ValidatorRegistry.validators_for self
+      end
+
+      def code_sample
+        self[:code_sample] ||= psychic.script(name)
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def source_file
+        return nil unless code_sample
+
+        self[:source_file] ||= Pathname(code_sample)
+      end
+
+      def absolute_source_file
+        return nil unless code_sample
+
+        code_sample.absolute_source_file
+      end
+
+      def source
+        return nil unless code_sample
+
+        code_sample.source
+      end
+
+      def code2doc(options = {})
+        return nil unless code_sample
+
+        doc = code_sample.code2doc(options)
+      end
+
+      def detect!
+        # fail FeatureNotImplementedError, "Project #{psychic.name} has not been cloned" unless psychic.cloned?
+        fail FeatureNotImplementedError, name if source_file.nil?
+        self.source_file = Pathname(code_sample)
+      rescue Errno::ENOENT
+        raise FeatureNotImplementedError, name
+      end
+
+      def exec!
+        detect!
+        evidence.result = run!
+      end
+
+      def run!(spies = Omnitest::Skeptic::Spies) # rubocop:disable Metrics/AbcSize
+        spies.observe(self) do
+          if code_sample.params.is_a? String
+            code_sample.params = YAML.load(Psychic::Tokens.replace_tokens(code_sample.params, vars))
+          else
+            code_sample.params = vars
+          end
+          execution_result = code_sample.execute(env: upcased_hash(vars).merge(ENV.to_hash))
+          evidence.result = Skeptic::Result.new(execution_result: execution_result, source_file: source_file.to_s)
+        end
+        result
+      rescue Omnitest::Shell::ExecutionError => e
+        execution_error = ExecutionError.new(e)
+        execution_error.execution_result = e.execution_result
+        evidence.error = Omnitest::Error.formatted_trace(e).join("\n")
+        raise execution_error
+      rescue => e
+        evidence.error = Omnitest::Error.formatted_trace(e).join("\n")
+        raise e
+      ensure
+        save
+      end
+
+      def verify!
+        validators.each do |validator|
+          validation = validator.validate(self)
+          status = case validation.result
+                   when :passed
+                     Core::Color.colorize("\u2713 Passed", :green)
+                   when :failed
+                     Core::Color.colorize('x Failed', :red)
+                     Omnitest.handle_validation_failure(validation.error)
+                   else
+                     Core::Color.colorize(validation.result, :yellow)
+                   end
+          info format('%-50s %s', validator.description, status)
+        end
+      end
+
+      def clear!
+        @evidence.clear
+        @evidence = nil
+      end
+
+      def validations
+        return nil if result.nil?
+        result.validations
+      end
+
+      private
+
+      def upcased_hash(hash)
+        new_hash = {}
+        hash.each_pair do | key, value |
+          new_hash[key.upcase] = value
+        end
+        new_hash
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/scenario_definition.rb

@@ -0,0 +1,41 @@
+module Omnitest
+  class Skeptic
+    class ScenarioDefinition < Omnitest::Core::Dash # rubocop:disable ClassLength
+      required_field :name, String
+      required_field :suite, String, required: true
+      field :properties, Hash[String => PropertyDefinition]
+      # TODO: Vars will be replaced by properties
+      field :vars, Skeptic::TestManifest::Environment, default: {}
+      attr_reader :full_name
+
+      def initialize(data)
+        super
+        self.vars ||= Skeptic::TestManifest::Environment.new
+        @full_name = [suite, name].join(' :: ').freeze
+      end
+
+      def build(project)
+        source_file = begin
+          file = Core::FileSystem.find_file project.basedir, name
+          Core::FileSystem.relativize(file, project.basedir)
+        rescue Errno::ENOENT
+          nil
+        end
+        psychic = project.respond_to?(:psychic) ? project.psychic : project
+        Scenario.new(psychic: psychic, scenario_definition: self, vars: build_vars, source_file: source_file)
+      end
+
+      private
+
+      def build_vars
+        # TODO: Build vars from properties
+        global_vars = begin
+          Omnitest.manifest[:global_env].dup
+        rescue
+          {}
+        end
+        global_vars.merge(vars.dup)
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/spies.rb

@@ -0,0 +1,43 @@
+require 'middleware'
+
+module Omnitest
+  class Skeptic
+    module Spies
+      class << self
+        attr_reader :spies
+
+        def middleware
+          @middleware ||= Middleware::Builder.new
+        end
+
+        def spies
+          @spies ||= Set.new
+        end
+
+        def register_spy(spy)
+          spies.add(spy)
+          middleware.insert 0, spy, {}
+        end
+
+        def observe(scenario, &blk)
+          middleware = Middleware::Builder.new
+          spies.each do |spy|
+            middleware.use spy, Thread.current[:test_env_number]
+          end
+          middleware.use blk
+          middleware.call(scenario)
+        end
+
+        def reports
+          # Group by type
+          all_reports = spies.flat_map do |spy|
+            spy.reports.to_a if spy.respond_to? :reports
+          end
+          all_reports.each_with_object({}) do |(k, v), h|
+            (h[k] ||= []) << v
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/spy.rb

@@ -0,0 +1,23 @@
+module Omnitest
+  class Skeptic
+    # # @abstract
+    class Spy
+      def initialize(app, opts = {})
+        @app = app
+        @opts = opts
+      end
+
+      def call(_scenario)
+        fail NotImplementedError, 'Subclass must implement #call'
+      end
+
+      def self.reports
+        @reports ||= {}
+      end
+
+      def self.report(type, report_class)
+        reports[type] = report_class
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/test_manifest.rb

@@ -0,0 +1,78 @@
+require 'yaml'
+
+module Omnitest
+  class Skeptic
+    # Omnitest::TestManifest acts as a test manifest. It defines the test scenarios that should be run,
+    # and may be shared across multiple projects when used for a compliance suite.
+    #
+    # A manifest is generally defined and loaded from YAML. Here's an example manifest:
+    #   ---
+    #   global_env:
+    #     LOCALE: <%= ENV['LANG'] %>
+    #     FAVORITE_NUMBER: 5
+    #   suites:
+    #     Katas:
+    #       env:
+    #         NAME: 'Max'
+    #       samples:
+    #         - hello world
+    #         - quine
+    #     Tutorials:
+    #           env:
+    #           samples:
+    #             - deploying
+    #             - documenting
+    #
+    # The *suites* object defines the tests. Each object, under suites, like *Katas* or *Tutorials* in this
+    # example, represents a test suite. A test suite is subdivided into *samples*, that each act as a scenario.
+    # The *global_env* object and the *env* under each suite define (and standardize) the input for each test.
+    # The *global_env* values will be made available to all tests as environment variables, along with the *env*
+    # values for that specific test.
+    #
+    class TestManifest < Omnitest::Core::Dash
+      include Core::DefaultLogger
+      include Omnitest::Core::Logging
+      extend Omnitest::Core::Dash::Loadable
+
+      class Environment < Omnitest::Core::Mash
+        coerce_value Integer, String
+      end
+
+      class Suite < Omnitest::Core::Dash
+        field :env, Environment, default: {}
+        field :samples, Array[String], required: true
+        field :results, Hash
+      end
+
+      field :global_env, Environment
+      field :suites, Hash[String => Suite]
+
+      attr_accessor :scenario_definitions
+      attr_accessor :scenarios
+
+      def scenario_definitions
+        @scenario_definitions ||= build_scenario_definitions
+      end
+
+      def build_scenario_definitions
+        definitions = Set.new
+        suites.each do | suite_name, suite |
+          suite.samples.each do | sample_pattern |
+            expand_pattern(sample_pattern).each do | sample |
+              definitions << ScenarioDefinition.new(name: sample, suite: suite_name, vars: suite.env)
+            end
+          end
+        end
+        definitions
+      end
+
+      private
+
+      def expand_pattern(pattern)
+        return [pattern] unless pattern.include? '*'
+
+        Dir[pattern].to_a
+      end
+    end
+  end
+end