omnitest-skeptic 0.0.2

data/spec/omnitest/skeptic/evidence_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+module Omnitest
+  class Skeptic
+    RSpec.describe Evidence do
+      let(:file) { Pathname('evidence.pstore').expand_path(current_dir) }
+      subject { described_class.new(file) }
+
+      let(:sample_data) do
+        {
+          last_attempted_action: 'foo',
+          last_completed_action: 'bar',
+          result: {
+            execution_result: {
+              command: 'echo foo',
+              stdout: File.read(__FILE__),
+              stderr: File.read(__FILE__),
+              exitstatus: 0
+            },
+            source_file: 'foo/bar.rb',
+            data: {},
+            validations: nil
+          },
+          spy_data: {
+            complex: {
+              nested: %w(object that can be stored)
+            }
+          },
+          error: ::StandardError.new,
+          vars: {
+            a: 'b',
+            c: 'd',
+            e: 'f'
+          },
+          duration: 123.5
+        }
+      end
+
+      describe '#save' do
+        it 'creates a file' do
+          expect { subject.save }.to change { file.exist? }.from(false).to(true)
+        end
+
+        it 'persists data that can be reloaded' do
+          sample_data.each do |key, value|
+            subject[key] = value
+          end
+          subject.save
+
+          reloaded_evidence = described_class.load(file)
+          original_data = subject.to_hash
+          reloaded_data = reloaded_evidence.to_hash
+          expect(reloaded_data).to eq(original_data)
+        end
+      end
+    end
+  end
+end

data/spec/omnitest/skeptic/result_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+module Omnitest
+  class Skeptic
+    describe Result do
+      describe '#status' do
+        context 'mixed pass/fail' do
+          let(:subject) do
+            Omnitest::Skeptic::Result.new(
+              validations: {
+                'max' => { status: 'passed' },
+                'omnitest' => { status: 'failed', error: 'foo!' }
+              }
+            ).status
+          end
+          it 'reports the failed status' do
+            is_expected.to eq('failed')
+          end
+        end
+        context 'mix passed/pending/skipped' do
+          let(:subject) do
+            Omnitest::Skeptic::Result.new(
+              validations: {
+                'max' => { status: 'passed' },
+                'omnitest' => { status: 'pending' },
+                'john doe' => { status: 'skipped' }
+              }
+            ).status
+          end
+          it 'reports the passed status' do
+            is_expected.to eq('passed')
+          end
+        end
+        context 'mix pending/skipped' do
+          let(:subject) do
+            Omnitest::Skeptic::Result.new(
+              validations: {
+                'max' => { status: 'pending' },
+                'omnitest' => { status: 'pending' },
+                'john doe' => { status: 'skipped' }
+              }
+            ).status
+          end
+          it 'reports the pending status' do
+            is_expected.to eq('pending')
+          end
+        end
+      end
+    end
+  end
+end

data/spec/omnitest/skeptic/scenario_definition_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+module Omnitest
+  class Skeptic
+    RSpec.describe ScenarioDefinition do
+      let(:psychic) { Fabricate(:psychic) }
+      let(:definition) do
+        {
+          name: 'My test scenario',
+          suite: 'My API',
+          properties: {
+            foo: {
+              required: true,
+              default: 'bar'
+            }
+          }
+        }
+      end
+
+      subject { described_class.new(definition) }
+
+      describe '#build' do
+        let(:scenario) { subject.build psychic }
+
+        it 'builds a Scenario for the Project' do
+          expect(scenario).to be_an_instance_of Scenario
+          # It actually creates a Psychic for the psychic
+          expect(scenario.psychic.name).to eq(psychic.name)
+          expect(scenario.psychic.basedir.to_s).to eq(psychic.basedir.to_s)
+        end
+
+        xit 'finds the source' do
+          expected_file = Pathname.new 'spec/fixtures/factorial.py'
+          expect(scenario.source_file).to eq(expected_file)
+        end
+      end
+    end
+  end
+end

data/spec/omnitest/skeptic/scenario_spec.rb

@@ -0,0 +1,35 @@
+module Omnitest
+  class Skeptic
+    describe Scenario do
+      subject(:scenario) do
+        project = Omnitest::Psychic.new name: 'some_sdk', cwd: 'spec/fixtures'
+        Fabricate(:scenario_definition, name: 'factorial', vars: {}).build(project)
+      end
+
+      describe '#detect' do
+        pending 'finds a script for the scenario'
+      end
+
+      describe '#check' do
+        pending 'checks the script complies with static rules'
+      end
+
+      describe '#prepare' do
+        pending 'meets unmet dependencies'
+      end
+
+      describe '#exec' do
+        it 'executes the scenario and returns itself' do
+          expect(scenario.exec).to be_an_instance_of Scenario
+          expect(scenario.exec).to eq(scenario)
+        end
+
+        it 'stores the result' do
+          evidence = scenario.exec
+          result = evidence.result
+          expect(result).to be_an_instance_of Skeptic::Result
+        end
+      end
+    end
+  end
+end

data/spec/omnitest/skeptic/test_manifest_spec.rb

@@ -0,0 +1,28 @@
+module Omnitest
+  class Skeptic
+    describe TestManifest do
+      describe '#from_yaml' do
+        subject(:manifest) { described_class.from_yaml 'spec/fixtures/skeptic.yaml' }
+
+        it 'initializes a manifest' do
+          expect(manifest).to be_an_instance_of TestManifest
+        end
+
+        it 'processes ERB' do
+          expect(manifest.global_env.LOCALE).to eq(ENV['LANG'])
+        end
+
+        it 'parses global_env' do
+          expect(manifest.global_env).to be_an_instance_of TestManifest::Environment
+        end
+
+        it 'parses suites' do
+          expect(manifest.suites).to be_an_instance_of ::Hash
+          manifest.suites.each_value do | suite |
+            expect(suite).to be_an_instance_of TestManifest::Suite
+          end
+        end
+      end
+    end
+  end
+end

data/spec/omnitest/skeptic/validator_registry_spec.rb

@@ -0,0 +1,40 @@
+module Omnitest
+  class Skeptic
+    describe ValidatorRegistry do
+      subject(:registry) { Omnitest::Skeptic::ValidatorRegistry }
+
+      describe '#register' do
+        it 'registers a validator' do
+          callback = proc do |scenario|
+            expect(scenario.result).to_not be_nil
+            expect(scenario.result.execution_result.exitstatus).to eq(0)
+          end
+
+          expect(registry.validators).to be_empty
+          registry.register(Validator.new('dummy', suite: 'java', scenario: 'hello world', &callback))
+          validator = registry.validators.first
+          expect(validator.suite).to eql('java')
+          expect(validator.scenario).to eql('hello world')
+          expect(validator.instance_variable_get('@callback')).to eql(callback)
+        end
+      end
+
+      describe '#validators_for' do
+        let(:java_hello_world_validator) { Fabricate(:validator, suite: 'java', scenario: 'hello world') }
+        let(:java_validator) { Fabricate(:validator, suite: 'java', scenario: //) }
+        let(:ruby_validator) { Fabricate(:validator, suite: 'ruby') }
+
+        it 'returns registered validators that match the scope of the scenario' do
+          registry.register(java_hello_world_validator)
+          registry.register(java_validator)
+          registry.register(ruby_validator)
+
+          scenario = Fabricate(:scenario_definition, suite: 'java', name: 'hello world')
+          validators = registry.validators_for scenario
+          expect(validators).to include(java_hello_world_validator, java_validator)
+          expect(validators).to_not include(ruby_validator)
+        end
+      end
+    end
+  end
+end

data/spec/omnitest/skeptic/validator_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+module Omnitest
+  class Skeptic
+    describe 'Validator' do
+      describe '#initialize' do
+        let(:global_matcher) { Validator::UNIVERSAL_MATCHER }
+
+        it 'accepts scope options and callback' do
+          validator = Validator.new 'dummy', suite: 'java', scenario: 'hello world' do |_scenario|
+            # Validate the scenario
+          end
+          expect(validator.suite).to eq('java')
+        end
+
+        it 'defaults suite and scenario to the universal matcher' do
+          validator = Validator.new 'dummy' do |_scenario|
+            # Validate
+          end
+          expect(validator.suite).to eq(Validator::UNIVERSAL_MATCHER)
+          expect(validator.scenario).to eq(Validator::UNIVERSAL_MATCHER)
+        end
+      end
+
+      describe '#should_validate?' do
+        let(:scenario) do
+          Fabricate(:scenario_definition, suite: 'java', name: 'hello world')
+        end
+
+        it 'returns true if the scope matches the scope of the scenario' do
+          expect(validator('java', 'hello world').should_validate? scenario).to be true
+          expect(validator('java').should_validate? scenario).to be true
+          expect(validator(/j/, /hello/).should_validate? scenario).to be true
+        end
+
+        it 'returns false if the scope does not match' do
+          expect(validator('ruby', 'hello world').should_validate? scenario).to be false
+          expect(validator('ruby').should_validate? scenario).to be false
+          expect(validator(/r/, /hello/).should_validate? scenario).to be false
+        end
+      end
+
+      describe '#validate' do
+        let(:psychic) { Fabricate(:psychic) }
+        let(:scenario) do
+          Fabricate(:scenario_definition).build(psychic).tap do | scenario |
+            scenario.result = Result.new
+          end
+        end
+
+        it 'calls the validation callback' do
+          called = false
+          validator = Validator.new 'dummy' do |_scenario|
+            called = true
+          end
+          expect { validator.validate scenario }.to change { called }.from(false).to(true)
+        end
+      end
+
+      def validator(*args)
+        scope = {}
+        scope[:suite] = args[0]
+        scope[:scenario] = args[1] if args[1]
+        Validator.new 'dummy', scope do |_scenario|
+          # Dummy validator
+        end
+      end
+    end
+  end
+end

data/spec/omnitest/skeptic_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+
+module Omnitest
+  RSpec.describe Skeptic do
+    before(:each) do | _example |
+      write_file 'skeptic.yaml', <<-eos
+---
+suites:
+  Execution:
+    samples:
+      - success
+      - failure
+eos
+      write_file 'success.rb', 'puts "foo"; exit 0'
+      write_file 'failure.rb', 'puts "bar"; exit 1'
+
+      Skeptic.configure do | config |
+        config.manifest_file = File.expand_path('skeptic.yaml', current_dir)
+      end
+
+      subject.clear
+    end
+
+    let(:psychic) { Psychic.new(cwd: current_dir) }
+    subject { described_class.new(psychic) }
+
+    describe '#scenarios' do
+      it 'returns the list of all scenarios' do
+        scenarios = subject.scenarios
+        expect(scenarios).to_not be_empty
+        scenarios.each do | scenario |
+          expect(scenario).to be_an_instance_of Skeptic::Scenario
+        end
+      end
+    end
+
+    describe '#prepare' do
+      let(:satisifed_dependency) { double('satisifed_dependency', met?: true) }
+      let(:unsatisifed_dependency) { double('unsatisifed_dependency', met?: false) }
+
+      pending 'meets unmet dependencies' do
+        scenarios = subject.scenarios
+        expect(satisifed_dependency).to_not receive(:meet)
+        expect(satisifed_dependency).to receive(:meet)
+        subject.prepare
+      end
+    end
+
+    describe '#exec' do
+      it 'calls exec on each scenario and stores an execution result' do
+        scenarios = subject.scenarios
+        results = scenarios.map(&:result)
+        expect(results).to all(be nil)
+        subject.exec
+        results = scenarios.map(&:result)
+        expect(results).to all(be_an_instance_of Skeptic::Result)
+        expect(scenarios.map(&:status_description)).to all(eq 'Executed')
+
+        expect(subject.scenario('success').result).to be_successful
+        expect(subject.scenario('failure').result).to_not be_successful
+        puts subject.summary
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,65 @@
+require 'simplecov'
+SimpleCov.start
+
+require 'omnitest/skeptic'
+require 'fabrication'
+
+# For Fabricators
+LANGUAGES = %w(java ruby python nodejs c# golang php)
+SCENARIO_NAMES = [
+  'hello world',
+  'quine',
+  'my_kata'
+]
+
+require 'rspec'
+require 'omnitest/skeptic'
+require 'aruba'
+require 'aruba/api'
+
+# Config required for project
+RSpec.configure do | config |
+  config.include Aruba::Api
+  config.before(:example) do
+    @aruba_timeout_seconds = 30
+    clean_current_dir
+  end
+end
+
+RSpec.configure do |c|
+  c.before(:each) do
+    Omnitest::Skeptic.reset
+  end
+  c.expose_current_running_example_as :example
+end
+
+# Configs recommended by RSpec
+RSpec.configure do |config|
+  # config.warnings = true # Unfortunately this produces too many warnings in third-party code
+  # config.disable_monkey_patching!
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4.
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # config.profile_examples = 10
+  config.order = :random
+  Kernel.srand config.seed
+end