omnitest-skeptic 0.0.2

data/lib/omnitest/skeptic/test_statuses.rb

@@ -0,0 +1,63 @@
+module Omnitest
+  class Skeptic
+    module TestStatuses
+      def failed?
+        evidence.last_attempted_action != evidence.last_completed_action
+      end
+
+      def skipped?
+        result.nil?
+      end
+
+      def sample?
+        !source_file.nil?
+      end
+
+      def status
+        status = last_attempted_action
+        failed? ? "#{status}_failed" : status
+      end
+
+      def status_description
+        case status
+        when 'clone' then 'Cloned'
+        when 'clone_failed' then 'Clone Failed'
+        when 'detect' then 'Sample Found'
+        when 'detect_failed', nil then '<Not Found>'
+        when 'bootstrap' then 'Bootstrapped'
+        when 'bootstrap_failed' then 'Bootstrap Failed'
+        when 'detect' then 'Detected'
+        when 'exec' then 'Executed'
+        when 'exec_failed' then 'Execution Failed'
+        when 'verify', 'verify_failed'
+          validator_count = validators.count
+          validation_count = validations.values.select { |v| v.result == :passed }.count
+          if validator_count == validation_count
+            "Fully Verified (#{validation_count} of #{validator_count})"
+          else
+            "Partially Verified (#{validation_count} of #{validator_count})"
+          end
+        # when 'verify_failed' then 'Verification Failed'
+        else "<Unknown (#{status})>"
+        end
+      end
+
+      def status_color
+        case status_description
+        when '<Not Found>' then :white
+        when 'Cloned' then :magenta
+        when 'Bootstrapped' then :magenta
+        when 'Sample Found' then :cyan
+        when 'Executed' then :blue
+        when /Verified/
+          if status_description =~ /Fully/
+            :green
+          else
+            :yellow
+          end
+        else :red
+        end
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/test_transitions.rb

@@ -0,0 +1,172 @@
+module Omnitest
+  class Skeptic
+    module TestTransitions
+      def detect
+        transition_to :detect
+      end
+
+      def detect_action
+        perform_action(:detect, 'Detecting code sample') do
+          detect!
+        end
+      end
+
+      def exec
+        transition_to :exec
+      end
+
+      def exec_action
+        perform_action(:exec, 'Executing') do
+          exec!
+        end
+      end
+
+      def verify
+        transition_to :verify
+      end
+
+      def verify_action
+        perform_action(:verify, 'Verifying') do
+          verify!
+        end
+      end
+
+      def clear
+        # Transitioning would try to load the data we're clearing... let's just
+        # jump straight to the action.
+        clear_action
+      end
+
+      def clear_action
+        perform_action(:clear, 'Clearing') do
+          clear!
+        end
+      end
+
+      def test(_clear_mode = :passing)
+        elapsed = Benchmark.measure do
+          banner "Cleaning up any prior instances of #{slug}"
+          clear
+          banner "Testing #{slug}"
+          verify
+          # clear if clear_mode == :passing
+        end
+        info "Finished testing #{slug} #{Core::Util.duration(elapsed.real)}."
+        evidence.duration = elapsed.real
+        save
+        evidence = nil # it's saved, free up memory...
+        self
+        # ensure
+        # clear if clear_mode == :always
+      end
+
+      def perform_action(verb, output_verb)
+        banner "#{output_verb} #{slug}..."
+        elapsed = action(verb) { yield }
+        # elapsed = action(verb) { |state| driver.public_send(verb, state) }
+        info("Finished #{output_verb.downcase} #{slug}" \
+          " #{Core::Util.duration(elapsed.real)}.")
+        # yield if block_given?
+        self
+      end
+
+      def action(what, &block)
+        evidence.last_attempted_action = what.to_s
+        elapsed = Benchmark.measure do
+          block.call(@state)
+        end
+        evidence.last_completed_action = what.to_s
+        elapsed
+      rescue FeatureNotImplementedError => e
+        raise e
+      rescue ActionFailed => e
+        log_failure(what, e)
+        raise(ScenarioFailure, failure_message(what) +
+          "  Please see .omnitest/logs/#{name}.log for more details",
+              e.backtrace)
+      rescue Exception => e # rubocop:disable RescueException
+        log_failure(what, e)
+        raise ActionFailed,
+              "Failed to complete ##{what} action: [#{e.message}]", e.backtrace
+      ensure
+        save unless what == :clear
+      end
+
+      def transition_to(desired)
+        transition_result = nil
+        begin
+          FSM.actions(last_completed_action, desired).each do |transition|
+            transition_result = send("#{transition}_action")
+          end
+        rescue FeatureNotImplementedError
+          warn("#{slug} is not implemented")
+        rescue ActionFailed => e
+          # Need to use with_friendly_errors again somewhere, since errors don't bubble up
+          # without fast-fail?
+          Omnitest.handle_error(e)
+          raise(ScenarioFailure, e.message, e.backtrace)
+        end
+        transition_result
+      end
+
+      def log_failure(what, e)
+        return unless logger.respond_to? :logdev
+        return if logger.logdev.nil?
+
+        logger.logdev.error(failure_message(what))
+        Error.formatted_trace(e).each { |line| logger.logdev.error(line) }
+      end
+
+      # Returns a string explaining what action failed, at a high level. Used
+      # for displaying to end user.
+      #
+      # @param what [String] an action
+      # @return [String] a failure message
+      # @api private
+      def failure_message(what)
+        "#{what.capitalize} failed for test #{slug}."
+      end
+
+      # The simplest finite state machine pseudo-implementation needed to manage
+      # an Instance.
+      #
+      # @api private
+      class FSM
+        # Returns an Array of all transitions to bring an Instance from its last
+        # reported transistioned state into the desired transitioned state.
+        #
+        # @param last [String,Symbol,nil] the last known transitioned state of
+        #   the Instance, defaulting to `nil` (for unknown or no history)
+        # @param desired [String,Symbol] the desired transitioned state for the
+        #   Instance
+        # @return [Array<Symbol>] an Array of transition actions to perform
+        # @api private
+        def self.actions(last = nil, desired)
+          last_index = index(last)
+          desired_index = index(desired)
+
+          if last_index == desired_index || last_index > desired_index
+            Array(TRANSITIONS[desired_index])
+          else
+            TRANSITIONS.slice(last_index + 1, desired_index - last_index)
+          end
+        end
+
+        TRANSITIONS = [:clear, :detect, :exec, :verify]
+
+        # Determines the index of a state in the state lifecycle vector. Woah.
+        #
+        # @param transition [Symbol,#to_sym] a state
+        # @param [Integer] the index position
+        # @api private
+        def self.index(transition)
+          if transition.nil?
+            0
+          else
+            TRANSITIONS.find_index { |t| t == transition.to_sym }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/validation.rb

@@ -0,0 +1,39 @@
+module Omnitest
+  class Skeptic
+    class Validation < Omnitest::Core::Dash
+      # TODO: Should we have (expectation) 'failed' vs (unexpected) 'error'?
+      ALLOWABLE_STATES = %w(passed pending failed skipped)
+
+      required_field :status, Symbol
+      field :error, Object
+      field :error_source, Object
+
+      def status=(state)
+        state = state.to_s
+        fail invalidate_state_error unless ALLOWABLE_STATES.include? state
+        super
+      end
+
+      ALLOWABLE_STATES.each do |state|
+        define_method "#{state}?" do
+          status == state?
+        end
+      end
+
+      def error_source?
+        !error_source.nil?
+      end
+
+      def to_hash(*args)
+        self.error_source = error.error_source if error.respond_to? :error_source
+        super
+      end
+
+      protected
+
+      def invalidate_state_error(state)
+        ArgumentError.new "Invalid status: #{state}, should be one of #{ALLOWABLE_STATES.inspect}"
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/validator.rb

@@ -0,0 +1,34 @@
+module Omnitest
+  class Skeptic
+    class Validator
+      include RSpec::Matchers
+
+      UNIVERSAL_MATCHER = //
+      attr_reader :description, :suite, :scenario, :level, :callback
+
+      def initialize(description, scope = {}, &validator)
+        @description = description
+        @suite = scope[:suite] ||= UNIVERSAL_MATCHER
+        @scenario = scope[:scenario] ||= UNIVERSAL_MATCHER
+        @callback = validator
+      end
+
+      def should_validate?(scenario)
+        # TODO: Case-insensitive matching?
+        !!(@suite.match(scenario.suite.to_s) && @scenario.match(scenario.name.to_s)) # rubocop:disable Style/DoubleNegation
+      end
+
+      def validate(scenario)
+        instance_exec(scenario, &@callback) if should_validate?(scenario)
+        scenario.result.validations[description] = Validation.new(status: :passed)
+      rescue StandardError, RSpec::Expectations::ExpectationNotMetError => e
+        validation = Validation.new(status: :failed, error: ValidationFailure.new(e.message, e))
+        scenario.result.validations[description] = validation
+      end
+
+      def to_s
+        @description
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/validator_registry.rb

@@ -0,0 +1,33 @@
+require 'singleton'
+
+module Omnitest
+  class Skeptic
+    class ValidatorRegistry
+      include Singleton
+
+      def validators
+        @validator ||= []
+      end
+
+      class << self
+        def validators
+          instance.validators
+        end
+
+        def register(validator, scope = {}, &callback)
+          validator = Validator.new(validator, scope, &callback) if block_given?
+          validators << validator
+        end
+
+        def validators_for(scenario)
+          selected_validators = validators.select { |v| v.should_validate? scenario }
+          selected_validators.empty? ? [Skeptic.configuration.default_validator] : selected_validators
+        end
+
+        def clear
+          validators.clear
+        end
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/version.rb

@@ -0,0 +1,5 @@
+module Omnitest
+  class Skeptic
+    VERSION = '0.0.2'
+  end
+end

data/omnitest-skeptic.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'omnitest/skeptic/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'omnitest-skeptic'
+  spec.version       = Omnitest::Skeptic::VERSION
+  spec.authors       = ['Max Lincoln']
+  spec.email         = ['max@devopsy.com']
+  spec.summary       = 'Skeptic tests code samples do what they should.'
+  # spec.description   = %q{TODO: Write a longer description. Optional.}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'omnitest-core', '~> 0'
+  spec.add_dependency 'omnitest-psychic', '~> 0'
+  spec.add_dependency 'hashie', '~> 3.0'
+  spec.add_dependency 'middleware', '~> 0.1'
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake-notes'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.18', '<= 0.27'
+  spec.add_development_dependency 'rubocop-rspec', '~> 1.2'
+  spec.add_development_dependency 'aruba'
+  spec.add_development_dependency 'fabrication', '~> 2.11'
+end

data/spec/fabricators/psychic_fabricator.rb

@@ -0,0 +1,12 @@
+# Fabricates test manifests (.omnitest.yaml files)
+
+Fabricator(:psychic, from: Omnitest::Psychic) do
+  initialize_with do
+    transients = @_transient_attributes.to_hash
+    transients[:name] ||= 'my_sample_project'
+    transients[:cwd] ||= "sdks/#{transients[:name]}"
+    @_klass.new transients
+  end # Hash based initialization
+  transient :name
+  transient :cwd
+end

data/spec/fabricators/scenario_fabricator.rb

@@ -0,0 +1,6 @@
+Fabricator(:scenario_definition, from: Omnitest::Skeptic::ScenarioDefinition) do
+  initialize_with { @_klass.new to_hash } # Hash based initialization
+  name { SCENARIO_NAMES.sample }
+  suite { LANGUAGES.sample }
+  properties {}
+end

data/spec/fabricators/validator_fabricator.rb

@@ -0,0 +1,12 @@
+Fabricator(:validator, from: Omnitest::Skeptic::Validator) do
+  initialize_with do
+    callback = @_transient_attributes.delete :callback
+    desc = @_transient_attributes.delete :description
+    scope = @_transient_attributes
+    @_klass.new(desc, scope, &callback)
+  end # Hash based initialization
+  transient description: 'Sample validator'
+  transient suite: LANGUAGES.sample
+  transient scenario: SCENARIO_NAMES.sample
+  transient callback: Proc.new { Proc.new { |_scenario| } } # rubocop:disable Proc
+end

data/spec/fixtures/factorial.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# Content above the snippet is ignored
+
+print 'Hello, world!'
+
+# {{snippet factorial}}
+def factorial(n):
+    if n == 0:
+        return 1
+    else:
+        return n * factorial(n-1)
+# {{endsnippet}}
+
+# So is content below the snippet
+print "{{snippet factorial_result}}"
+print "The result of factorial(7) is:"
+print "  %d" % factorial(7)
+print "{{endsnippet}}"

data/spec/fixtures/skeptic.yaml

@@ -0,0 +1,16 @@
+---
+  global_env:
+    LOCALE: <%= ENV['LANG'] %>
+    FAVORITE_NUMBER: 5
+  suites:
+    Katas:
+      env:
+        NAME: 'Max'
+      samples:
+        - hello world
+        - quine
+    Tutorials:
+          env:
+          samples:
+            - deploying
+            - documenting