omnitest-skeptic 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b9b6bff210ea41042bae9066f417f98d590d3cea
+  data.tar.gz: 58655d93652db3a0005df71549e61e5df3b10546
+SHA512:
+  metadata.gz: 234205a3ed225713b2b5673167762647c1b515e48238768188004512acd1a18b81cec031e7faf51d7b2ad7abfdc547baf6a710189f63d9aa06ae212ed4780557
+  data.tar.gz: a904457a02e2ba152e4ae663230e1a2b831165d62148092b58199f21b79720a2ca89d50d858e7a8af63e995bdea0ff64e01a6e222172fb8812d69f1ff430ab07

data/.gitignore

@@ -0,0 +1,6 @@
+Gemfile.lock
+tmp/
+coverage/
+site/
+.omnitest/
+reports/

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,5 @@
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  Exclude:
+    - 'tmp/**/*'

data/.rubocop_todo.yml

@@ -0,0 +1,36 @@
+# This configuration was generated by `rubocop --auto-gen-config`
+# on 2015-01-29 15:35:53 -0500 using RuboCop version 0.27.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Lint/UselessAssignment:
+  Enabled: false
+
+# Offense count: 7
+Metrics/AbcSize:
+  Max: 28
+
+# Offense count: 2
+Metrics/CyclomaticComplexity:
+  Max: 12
+
+# Offense count: 88
+# Configuration parameters: AllowURI, URISchemes.
+Metrics/LineLength:
+  Max: 132
+
+# Offense count: 9
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 20
+
+# Offense count: 30
+Style/Documentation:
+  Enabled: false
+
+# Offense count: 2
+Style/RegexpLiteral:
+  MaxSlashes: 0

data/Gemfile

@@ -0,0 +1,24 @@
+source 'https://rubygems.org'
+
+source 'https://rubygems.org'
+
+gemspec
+
+# This hack inspired by rspec...
+branch = begin
+           File.read(File.expand_path('../maintenance-branch', __FILE__)).chomp
+         rescue
+           'working'
+         end
+
+%w(omnitest-core psychic).each do |lib|
+  library_path = File.expand_path("../../#{lib}", __FILE__)
+  gem_name = lib.start_with?('omnitest') ? lib : "omnitest-#{lib}"
+  if File.exist?(library_path) && !ENV['USE_GIT_REPOS']
+    gem gem_name, path: library_path
+  else
+    gem gem_name, git: "git://github.com/omnitest/#{lib}.git", branch: branch
+  end
+end
+
+gem 'pry'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Max Lincoln
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# Psychic::Skeptic
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'psychic-skeptic'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install psychic-skeptic
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/psychic-skeptic/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler/gem_tasks'
+require 'rubocop/rake_task'
+require 'rake/notes/rake_task'
+require 'rspec/core/rake_task'
+
+task default: [:spec, :rubocop, :notes]
+
+RSpec::Core::RakeTask.new('spec')
+RuboCop::RakeTask.new(:rubocop) do |task|
+  # abort rake on failure
+  task.fail_on_error = true
+end

data/bin/skeptic

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'omnitest/skeptic/cli'
+
+Omnitest::Skeptic::CLI.start

data/lib/omnitest/skeptic.rb

@@ -0,0 +1,123 @@
+require 'rspec/expectations' # exceptions are being stored as classes, so this is needed to load
+require 'omnitest/core'
+require 'omnitest/psychic'
+require 'omnitest/skeptic/version'
+require 'omnitest/skeptic/errors'
+
+module Omnitest
+  class Skeptic
+    autoload :Configuration, 'omnitest/skeptic/configuration'
+    autoload :ScenarioDefinition, 'omnitest/skeptic/scenario_definition'
+    autoload :PropertyDefinition, 'omnitest/skeptic/property_definition'
+    autoload :Scenario, 'omnitest/skeptic/scenario'
+    autoload :TestStatuses, 'omnitest/skeptic/test_statuses'
+    autoload :TestTransitions, 'omnitest/skeptic/test_transitions'
+    autoload :TestManifest, 'omnitest/skeptic/test_manifest'
+    autoload :Evidence, 'omnitest/skeptic/evidence'
+    autoload :Result, 'omnitest/skeptic/result'
+    autoload :Spy, 'omnitest/skeptic/spy'
+    autoload :Spies, 'omnitest/skeptic/spies'
+    autoload :Validation, 'omnitest/skeptic/validation'
+    autoload :Validator, 'omnitest/skeptic/validator'
+    autoload :ValidatorRegistry, 'omnitest/skeptic/validator_registry'
+
+    class << self
+      include Core::Configurable
+
+      def acts_on_scenario(action)
+        define_method action do | regex = 'all', options = {} |
+          scenarios(regex, options).each do | scenario |
+            scenario.public_send(action)
+          end
+        end
+      end
+
+      def acts_on_scenario_with_options(action)
+        define_method action do | regex = 'all', options = {} |
+          scenarios(regex, options).each do | scenario |
+            scenario.public_send(action, options)
+          end
+        end
+      end
+
+      # Registers a {Omnitest::Skeptic::Validator} that will be used during test
+      # execution on matching {Omnitest::Skeptic::Scenario}s.
+      def validate(desc, scope = { suite: //, scenario: // }, &block)
+        fail ArgumentError, 'You must pass block' unless block_given?
+        validator = Omnitest::Skeptic::Validator.new(desc, scope, &block)
+
+        Omnitest::Skeptic::ValidatorRegistry.register validator
+        validator
+      end
+    end
+
+    def initialize(psychic = Psychic.new)
+      psychic = Psychic.new(psychic) if psychic.is_a? Hash
+      @psychic = psychic
+    end
+
+    def manifest
+      Skeptic.configuration.manifest
+    end
+
+    def scenario_definitions
+      manifest.scenario_definitions
+    end
+
+    def build_scenarios
+      scenario_definitions.map do | scenario_definition |
+        scenario_definition.build @psychic
+      end
+    end
+
+    def scenario(name)
+      scenarios.find { |s| s.name == name }
+    end
+
+    def all_scenarios
+      @scenarios ||= build_scenarios
+    end
+
+    def select_scenarios(regexp)
+      regexp ||= 'all'
+      if regexp == 'all'
+        return all_scenarios
+      else
+        selected_scenarios = all_scenarios.find { |c| c.full_name == regexp } ||
+                             all_scenarios.select { |c| c.full_name =~ /#{regexp}/i }
+      end
+
+      if selected_scenarios.is_a? Array
+        selected_scenarios
+      else
+        [selected_scenarios]
+      end
+    end
+
+    def scenarios(regexp = 'all', options = {})
+      selected_scenarios = select_scenarios regexp
+      selected_scenarios.keep_if { |scenario| scenario.failed? == options[:failed] } unless options[:failed].nil?
+      selected_scenarios.keep_if { |scenario| scenario.skipped? == options[:skipped] } unless options[:skipped].nil?
+      selected_scenarios.keep_if { |scenario| scenario.sample? == options[:samples] } unless options[:samples].nil?
+      selected_scenarios
+    end
+
+    def summary
+      summary_data = ["#{scenarios.size} scenarios"]
+      scenarios.group_by(&:status).each do | _status, group |
+        # Note: Removes partially verified's parenthetical description
+        status_description = group.first.status_description.gsub(/\(.*/, '')
+        summary_data << "#{group.size} #{status_description.downcase}"
+      end
+      summary_data.join("\n  ")
+    end
+
+    acts_on_scenario_with_options :code2doc
+    acts_on_scenario :test
+    Scenario::FSM::TRANSITIONS.each do | transition |
+      acts_on_scenario transition
+    end
+  end
+end
+
+Omnitest.mutex = Mutex.new

data/lib/omnitest/skeptic/cli.rb

@@ -0,0 +1,156 @@
+require 'omnitest/skeptic'
+
+module Omnitest
+  class Skeptic
+    class BaseCLI < Omnitest::Core::CLI
+      attr_accessor :psychic, :skeptic
+
+      no_commands do
+        def update_config!
+          Omnitest::Skeptic.configuration.manifest_file = options[:skeptic]
+          autogenerate_manifest if options[:glob]
+          runner_opts = { cwd: Dir.pwd, cli: shell, parameters: options.parameters }
+          runner_opts.merge!(Omnitest::Core::Util.symbolized_hash(options))
+          @psychic = Omnitest::Psychic.new(runner_opts)
+          @skeptic = Omnitest::Skeptic.new(@psychic)
+        end
+
+        def autogenerate_manifest
+          data = { suites: {} }
+          suites = Dir[*options[:glob]].group_by do | file |
+            Pathname(file).dirname.to_s
+          end
+          suites.each do | suite, files |
+            data[:suites][suite] = {}
+            data[:suites][suite][:samples] = files
+          end
+          Omnitest::Skeptic.configuration.manifest = TestManifest.new(data)
+        end
+      end
+    end
+
+    class CLI < BaseCLI # rubocop:disable Metrics/ClassLength
+      # The maximum number of concurrent instances that can run--which is a bit
+      # high
+      MAX_CONCURRENCY = 9999
+
+      desc 'code2doc [SCENARIO|REGEXP|all]',
+           'Convert scripts for code samples to lightweight documentation formats'
+      method_option :skeptic,
+                    aliases: '-s',
+                    desc: 'The Skeptic test manifest file',
+                    default: 'skeptic.yaml'
+      method_option :test_dir,
+                    aliases: '-t',
+                    desc: 'The Omnitest test directory',
+                    default: 'tests/omnitest'
+      method_option :glob,
+                    type: :array,
+                    aliases: '-g',
+                    desc: 'Automatically build scenarios for samples matching the glob pattern(s)',
+                    lazy_default: true
+      method_option :format,
+                    enum: %w(md rst),
+                    default: 'md',
+                    desc: 'Target documentation format'
+      method_option :destination,
+                    aliases: '-d',
+                    default: 'docs/',
+                    desc: 'The target directory where documentation for generated documentation.'
+      method_option :glob,
+                    type: :array,
+                    aliases: '-g',
+                    desc: 'Automatically build scenarios for samples matching the glob pattern(s)'
+      def code2doc(regex = 'all')
+        update_config!
+        action_options = options.dup
+        skeptic.public_send(:code2doc, regex, action_options)
+      end
+
+      desc 'test [SCENARIO|REGEXP|all]',
+           'Test (clone, bootstrap, exec, and verify) one or more scenarios'
+      long_desc <<-DESC
+        The scenario states are in order: cloned, bootstrapped, executed, verified.
+        Test changes the state of one or more scenarios executes
+        the actions for each state up to verify.
+      DESC
+      method_option :concurrency,
+                    aliases: '-c',
+                    type: :numeric,
+                    lazy_default: MAX_CONCURRENCY,
+                    desc: <<-DESC.gsub(/^\s+/, '').gsub(/\n/, ' ')
+          Run a test against all matching instances concurrently. Only N
+          instances will run at the same time if a number is given.
+        DESC
+      method_option :log_level,
+                    aliases: '-l',
+                    desc: 'Set the log level (debug, info, warn, error, fatal)'
+      method_option :skeptic,
+                    aliases: '-s',
+                    desc: 'The Skeptic test manifest file',
+                    default: 'skeptic.yaml'
+      method_option :test_dir,
+                    aliases: '-t',
+                    desc: 'The Omnitest test directory',
+                    default: 'tests/omnitest'
+      method_option :glob,
+                    type: :array,
+                    aliases: '-g',
+                    desc: 'Automatically build scenarios for samples matching the glob pattern(s)'
+      def test(regex = 'all')
+        update_config!
+        action_options = options.dup
+        skeptic.public_send(:test, regex, action_options)
+      end
+
+      {
+        detect: 'Find sample code that matches a test scenario. ' \
+                      'Attempts to locate a code sample with a filename that the test scenario name.',
+        exec: 'Change instance state to executed. ' \
+                      'Execute the code sample and capture the results.',
+        verify: 'Change instance state to verified. ' \
+                      'Assert that the captured results match the expectations for the scenario.',
+        clear: 'Clear stored results for the scenario. ' \
+                     'Delete all stored results for one or more scenarios'
+      }.each do |action, short_desc|
+        desc(
+          "#{action} [SCENARIO|REGEXP|all]",
+          short_desc
+        )
+        long_desc <<-DESC
+          The scenario states are in order: cloned, bootstrapped, executed, verified.
+          Change one or more scenarios from the current state to the #{action} state. Actions for all
+          intermediate states will be executed.
+        DESC
+        method_option :concurrency,
+                      aliases: '-c',
+                      type: :numeric,
+                      lazy_default: MAX_CONCURRENCY,
+                      desc: <<-DESC.gsub(/^\s+/, '').gsub(/\n/, ' ')
+            Run a #{action} against all matching instances concurrently. Only N
+            instances will run at the same time if a number is given.
+          DESC
+        method_option :log_level,
+                      aliases: '-l',
+                      desc: 'Set the log level (debug, info, warn, error, fatal)'
+        method_option :file,
+                      aliases: '-f',
+                      desc: 'The Omnitest project set file',
+                      default: 'omnitest.yaml'
+        method_option :skeptic,
+                      aliases: '-s',
+                      desc: 'The Skeptic test manifest file',
+                      default: 'skeptic.yaml'
+        method_option :test_dir,
+                      aliases: '-t',
+                      desc: 'The Omnitest test directory',
+                      default: 'tests/omnitest'
+        define_method(action) do |regex = 'all'|
+          update_config!
+          action_options = options.dup
+          skeptic.public_send(action, regex, action_options)
+        end
+      end
+    end
+  end
+end

data/lib/omnitest/skeptic/configuration.rb

@@ -0,0 +1,48 @@
+module Omnitest
+  class Skeptic
+    class Configuration < Omnitest::Core::Dash
+      field :seed, Integer, default: Process.pid # or Random.new_seed
+      field :manifest, TestManifest
+      field :manifest_file, Pathname, default: 'skeptic.yaml'
+
+      def manifest
+        self[:manifest] ||= load_manifest
+      end
+
+      def manifest_file=(file)
+        self[:manifest] = nil
+        self[:manifest_file] = file
+      end
+
+      def load_manifest
+        ENV['SKEPTIC_SEED'] = seed.to_s
+        Skeptic::TestManifest.from_yaml manifest_file
+      rescue Errno::ENOENT => e
+        raise UserError, "Could not load test manifest: #{e.message}"
+      end
+
+      # The callback used to validate code samples that
+      # don't have a custom validator.  The default
+      # checks that the sample code runs successfully.
+      def default_validator_callback
+        @default_validator_callback ||= proc do |scenario|
+          expect(scenario.result.execution_result.exitstatus).to eq(0)
+        end
+      end
+
+      def default_validator
+        @default_validator ||= Skeptic::Validator.new('default validator', suite: //, scenario: //, &default_validator_callback)
+      end
+
+      attr_writer :default_validator_callback
+
+      def register_spy(spy)
+        Omnitest::Skeptic::Spies.register_spy(spy)
+      end
+
+      def clear
+        ValidatorRegistry.clear
+      end
+    end
+  end
+end