priority_test 0.0.1 → 0.0.2

data/.gitignore

@@ -2,3 +2,4 @@
 .bundle
 Gemfile.lock
 pkg/*
+.priority-test.db

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/Gemfile

@@ -1,4 +1,3 @@
 source "http://rubygems.org"
 
-# Specify your gem's dependencies in priority_test.gemspec
 gemspec

data/README.md

@@ -3,8 +3,71 @@ PriorityTest
 
 # DESCRIPTION
 
-# ASSUMPTION
+PriorityTest is a gem that delivers fast feedback for your tests by
+prioritizing them.
+It prioritizes tests based on two assumptions discovered by [Kent Beck](https://twitter.com/#!/kentbeck) in his tool [JUnit Max](http://junitmax.com/):
+
+> Test run times generally follow a power law distribution - lots of very short tests and a few very long ones. This means that by running the short tests first you can get most of the feedback in a fraction of the runtime of the whole suite.
+
+> Failures are not randomly distributed. A test that failed recently is more likely to fail than one that has run correctly a bazillion times in a row. By putting recently failed (and newly written) tests first in the queue, you maximize the information density of that critical first second of feedback.
+
+PriorityTest inherits from these two assumptions with a simple
+algorithm and prioritizes your tests by looking at the test running history.
+
+# ALGORITHM
+
+PriorityTest captures and stores your test running hisotry.
+Before each test runs, it looks back X number of the previous test results to calculate the test's Degree of Significant (DoS).
+It then prioritizes the running order of all the tests based on each test's DoS.
+Two factors determines a test's DoS: test run time and recent failure times.
 
 # INSTALLATION
 
+## RubyGems
+
+    [sudo] gem install priority_test
+
+## RSpec
+
+In your ```Gemfile```, insert the following line:
+
+```ruby
+gem 'priority_test'
+```
+
+In ```spec_helper.rb```, require the RSpec adapter:
+
+```ruby
+require 'priority_test/rspec'
+```
+
 # USAGE
+
+Getting help:
+
+    $ pt -h
+    Usage: pt <test-framework> [options] [files or directories]
+
+    Test framework:
+      * rspec
+
+    Options:
+            --priority                   Filter and run priority tests
+        -h, --help                       Show help
+        -v, --version                    Show version
+
+Run tests in priority order:
+
+    $ pt rspec spec/a_spec
+
+Filter and run priority tests:
+
+    $ pt rspec spec/a_spec --priority
+
+Directly passing arguments to RSpec:
+
+    $ pt rspec spec/a_spec --priority -fp
+
+Run tests in a Rake task:
+
+    $ rake spec PT_OPTS="--priority"

data/Rakefile

@@ -41,21 +41,18 @@ end
 #
 #############################################################################
 
-task :default => :test
+task :default => :spec
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = "./spec/**/*_spec.rb"
 end
 
 desc "Generate RCov test coverage and open in your browser"
-task :coverage do
-  require 'rcov'
-  sh "rm -fr coverage"
-  sh "rcov test/test_*.rb"
-  sh "open coverage/index.html"
+RSpec::Core::RakeTask.new(:rcov) do |t|
+  t.rcov = true
+  t.pattern = "./spec/**/*_spec.rb"
+  t.rcov_opts = '--exclude /gems/,/Library/,/usr/,lib/tasks,.bundle,config,/lib/rspec/,/lib/rspec-,spec'
 end
 
 require 'rdoc/task'
@@ -68,7 +65,7 @@ end
 
 desc "Open an irb session preloaded with this library"
 task :console do
-  sh "irb -rubygems -r ./lib/#{name}.rb"
+  sh "irb -rubygems -r ./lib/#{name}.rb -I ./lib"
 end
 
 #############################################################################

data/bin/pt

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'priority_test/autorun'
+rescue LoadError
+  $stderr.puts <<-EOS
+#{'*'*50}
+  Could not find 'priority/autorun'
+
+  This may happen if you're using rubygems as your package manager, but it is not
+  being required through some mechanism before executing the priority_test command.
+
+  You may need to do one of the following in your shell:
+
+    # for bash/zsh
+    export RUBYOPT=rubygems
+
+    # for csh, etc.
+    set RUBYOPT=rubygems
+
+  For background, please see http://gist.github.com/54177.
+#{'*'*50}
+  EOS
+  exit(1)
+end

data/lib/priority_test/autorun.rb

@@ -0,0 +1,2 @@
+require 'priority_test'
+PriorityTest::Core::Runner.autorun

data/lib/priority_test/core/all_tests.rb

@@ -0,0 +1,30 @@
+module PriorityTest
+  module Core
+    class AllTests
+      def initialize(tests=[])
+        @test_hash = Hash[tests.collect { |t| [t.identifier, t] }]
+      end
+
+      def add_test(test_params)
+        test = Test.create(test_params)
+        @test_hash[test.identifier] = test
+      end
+
+      def get_test(identifier)
+        @test_hash[identifier]
+      end
+
+      def add_test_result(identifier, test_result_params)
+        test = get_test(identifier)
+        if test
+          test.add_result(test_result_params)
+          test.update_statistics
+        end
+      end
+
+      def size
+        @test_hash.size
+      end
+    end
+  end
+end

data/lib/priority_test/core/configuration.rb

@@ -0,0 +1,23 @@
+module PriorityTest
+  module Core
+    class Configuration
+      def self.add_setting(name, opts={})
+        define_method("#{name}=") { |val| settings[name] = val}
+        define_method(name)       { settings.has_key?(name) ? settings[name] : opts[:default] }
+        define_method("#{name}?") { send name }
+      end
+
+      add_setting :test_framework
+      add_setting :database, :default => "sqlite://#{File.join(File.expand_path('.'), '.priority-test.db')}"
+      add_setting :priority, :default => false
+
+      def settings
+        @settings ||= {}
+      end
+
+      def add_setting(name, opts={})
+        self.class.add_setting(name, opts)
+      end
+    end
+  end
+end

data/lib/priority_test/core/configuration_options.rb

@@ -0,0 +1,19 @@
+module PriorityTest
+  module Core
+    class ConfigurationOptions
+      def initialize(args=[])
+        @args = args
+      end
+
+      def configure(config)
+        options.each do |key, value|
+          config.send("#{key}=", options[key]) if config.respond_to?("#{key}=")
+        end
+      end
+
+      def options
+        @options ||= OptionParser.parse!(@args)
+      end
+    end
+  end
+end

data/lib/priority_test/core/option_parser.rb

@@ -0,0 +1,84 @@
+require 'optparse'
+
+module PriorityTest
+  module Core
+    class OptionParser
+      AVAILABLE_TEST_FRAMEWORKS = ['rspec']
+      OPTIONS = ['--priority']
+
+      def self.parse!(args)
+        new.parse!(args)
+      end
+
+      def self.parse_options(args)
+        new.parse_options(args)
+      end
+
+      def parse_options(args)
+        return {} if args.empty?
+
+        options = {}
+        opts_parser = parser(options)
+        begin
+          opts_parser.parse!(args.clone)
+        rescue ::OptionParser::InvalidOption
+        end
+
+        remove_parsed_options(args)
+
+        options
+      end
+
+      def parse!(args)
+        return {} if args.empty?
+
+        options = parse_options(args)
+        options[:test_framework] = parse_test_framework(args)
+
+        options
+      end
+
+      def parser(options)
+        ::OptionParser.new do |parser|
+          parser.banner = <<-EOS
+Usage: pt <test-framework> [options] [files or directories]
+
+Test framework:
+  * rspec
+
+Options:
+          EOS
+
+          parser.on('--priority', 'Filter and run priority tests') do |o|
+            options[:priority] = true
+          end
+
+          parser.on_tail("-h", "--help", "Show help") do
+            puts parser
+            exit
+          end
+
+          parser.on_tail('-v', '--version', 'Show version') do
+            puts PriorityTest::VERSION
+            exit
+          end
+        end
+      end
+
+      def parse_test_framework(args)
+        test_framework = args.shift
+        if AVAILABLE_TEST_FRAMEWORKS.include?(test_framework)
+          test_framework
+        else
+          puts "Invalid test framework: #{test_framework}"
+          puts "Run `pt -h` for more info"
+          raise ::OptionParser::InvalidArgument
+        end
+      end
+
+      def remove_parsed_options(args)
+        args.reject! { |arg| OPTIONS.include?(arg) }
+      end
+    end
+  end
+end

data/lib/priority_test/core/priority.rb

@@ -0,0 +1,62 @@
+module PriorityTest
+  module Core
+    module Priority
+      PRIORITY_SET_RANKINGS = {
+        "FFFFF" => 1,
+        "FFFFP" => 2,
+        "FFFPF" => 3,
+        "FFFPP" => 4,
+
+        "FFPFF" => 5,
+        "FFPFP" => 6,
+        "FFPPF" => 7,
+        "FFPPP" => 8,
+
+        "FPFFF" => 9,
+        "FPFFP" => 10,
+        "FPFPF" => 11,
+        "FPFPP" => 12,
+
+        "FPPFF" => 13,
+        "FPPFP" => 14,
+        "FPPPF" => 15,
+        "FPPPP" => 16,
+
+        "PFFFF" => 17,
+        "PFFFP" => 18,
+        "PFFPF" => 19,
+        "PFFPP" => 20,
+
+        "PFPFF" => 21,
+        "PFPFP" => 22,
+        "PFPPF" => 23,
+        "PFPPP" => 24,
+      }
+
+      NON_PRIORITY_SET_RANKINGS = {
+        "PPFFF" => 25,
+        "PPFFP" => 26,
+        "PPFPF" => 27,
+        "PPFPP" => 28,
+
+        "PPPFF" => 29,
+        "PPPFP" => 30,
+        "PPPPF" => 31,
+        "PPPPP" => 32
+      }
+
+      # ranking for the last 5 test results
+      PRIORITY_RANKINGS = PRIORITY_SET_RANKINGS.merge(NON_PRIORITY_SET_RANKINGS)
+
+      PRIORITY_THRESHOLD = 24
+
+      def self.[](key)
+        PRIORITY_RANKINGS[key]
+      end
+
+      def self.in_priority_set?(priority)
+        priority <= PRIORITY_THRESHOLD
+      end
+    end
+  end
+end

data/lib/priority_test/core/priority_sort_element.rb

@@ -0,0 +1,23 @@
+module PriorityTest
+  module Core
+    class PrioritySortElement
+      include Comparable
+
+      attr_reader :test
+
+      def initialize(identifier)
+        @test = PriorityTest.all_tests.get_test(identifier)
+      end
+
+      def <=>(other)
+        if test.nil? && !other.test.nil?
+          1
+        elsif !test.nil? && other.test.nil?
+          -1
+        else
+          test <=> other.test
+        end
+      end
+    end
+  end
+end

data/lib/priority_test/core/runner.rb

@@ -0,0 +1,26 @@
+module PriorityTest
+  module Core
+    module Runner
+      def self.autorun
+        return if installed_at_exit?
+        run(ARGV)
+        @installed_at_exit = true
+      end
+      AT_EXIT_HOOK_BACKTRACE_LINE = "#{__FILE__}:#{__LINE__ - 2}:in `autorun'"
+
+      def self.installed_at_exit?
+        @installed_at_exit ||= false
+      end
+
+      def self.run(args)
+        config_options = ConfigurationOptions.new(args)
+        config = PriorityTest.configuration
+        config_options.configure(config)
+
+        if config.test_framework == 'rspec'
+          require 'rspec/autorun'
+        end
+      end
+    end
+  end
+end

data/lib/priority_test/core/service.rb

@@ -0,0 +1,14 @@
+module PriorityTest
+  module Core
+    class Service
+      def initialize(all_tests)
+        @all_tests = all_tests
+      end
+
+      def priority_test?(identifier)
+        test = @all_tests.get_test(identifier)
+        test.nil? ? true : test.priority?
+      end
+    end
+  end
+end

data/lib/priority_test/core/test.rb

@@ -0,0 +1,53 @@
+module PriorityTest
+  module Core
+    class Test < ::Sequel::Model
+      include PriorityTest::Core::ValidationsHelper
+      include Comparable
+
+      NUMBER_OF_RESULTS = 5
+
+      one_to_many :results, :class => PriorityTest::Core::TestResult.name, :class => PriorityTest::Core::TestResult.name do |ds|
+        ds.order(:started_at.desc).limit(NUMBER_OF_RESULTS) # make it configurable
+      end
+
+      def self.all_in_priority_order
+        eager(:results).order(:priority, :avg_run_time).all
+      end
+
+      def <=>(other)
+        result = (priority <=> other.priority)
+        result = (avg_run_time <=> other.avg_run_time) if result == 0 || !result
+        result
+      end
+
+      def validate
+        validates_presence [ :identifier, :file_path ]
+      end
+
+      def results_key
+        results_key = results.collect { |r| r.passed? ? 'P' : 'F' }.join
+        results_key << 'P' * (NUMBER_OF_RESULTS - results_key.size) if results_key.size < NUMBER_OF_RESULTS
+        results_key[0..(NUMBER_OF_RESULTS-1)]
+      end
+
+      def update_statistics
+        stats_to_update = {}
+        prio = Priority[results_key]
+        stats_to_update.merge!(:priority => prio) if prio
+        stats_to_update.merge!(:avg_run_time => calculate_avg_run_time) if results.size > 0
+
+        self.update(stats_to_update)
+      end
+
+      def priority?
+        Priority.in_priority_set?(priority)
+      end
+
+      private
+
+      def calculate_avg_run_time
+        results.collect(&:run_time).reduce(:+) / results.size
+      end
+    end
+  end
+end

data/lib/priority_test/core/test_result.rb

@@ -0,0 +1,25 @@
+module PriorityTest
+  module Core
+    class TestResult < ::Sequel::Model
+      include PriorityTest::Core::ValidationsHelper
+
+      PASSED_STATUS = 'passed'
+      FAILEDED_STATUS = 'failed'
+
+      many_to_one :context, :class => PriorityTest::Core::Test.name, :key => :test_id
+
+      def passed?
+        status == PASSED_STATUS
+      end
+
+      def failed?
+        not passed?
+      end
+
+      def validate
+        validates_presence [ :status, :started_at, :run_time ]
+        validates_includes [ PASSED_STATUS, FAILEDED_STATUS ], :status
+      end
+    end
+  end
+end

data/lib/priority_test/core/test_result_collector.rb

@@ -0,0 +1,40 @@
+module PriorityTest
+  module Core
+    class TestResultCollector
+      attr_reader :all_tests
+
+      def initialize(all_tests)
+        @all_tests = all_tests
+      end
+
+      def add_result(test_result_hash)
+        identifier = test_result_hash[:identifier]
+        return unless identifier
+
+        @all_tests.get_test(identifier) || @all_tests.add_test(test_params(test_result_hash))
+        @all_tests.add_test_result(identifier, test_result_params(test_result_hash))
+      end
+
+      def finish
+        # nothing
+      end
+
+      private
+
+      def test_params(test_result_hash)
+        {
+          :identifier => test_result_hash[:identifier],
+          :file_path => test_result_hash[:file_path]
+        }
+      end
+
+      def test_result_params(test_result_hash)
+        {
+          :status => test_result_hash[:status],
+          :started_at => test_result_hash[:started_at],
+          :run_time => test_result_hash[:run_time]
+        }
+      end
+    end
+  end
+end

data/lib/priority_test/core/validations_helper.rb

@@ -0,0 +1,17 @@
+module PriorityTest
+  module Core
+    module ValidationsHelper
+      def validates_presence(args)
+        args.each do |arg|
+          value = send(arg)
+          errors.add(arg, 'cannot be empty') if !value || value.to_s.empty?
+        end
+      end
+
+      def validates_includes(includes, arg)
+        value = send(arg)
+        errors.add(arg, "should be in list #{includes}") unless includes.include?(value)
+      end
+    end
+  end
+end

data/lib/priority_test/core.rb

@@ -0,0 +1,41 @@
+require_path 'core/configuration'
+require_path 'core/configuration_options'
+require_path 'core/option_parser'
+require_path 'core/runner'
+require_path 'core/priority'
+require_path 'core/validations_helper'
+require_path 'core/all_tests'
+require_path 'core/service'
+require_path 'core/test_result_collector'
+require_path 'core/priority_sort_element'
+
+module PriorityTest
+  module Core
+    # Remove dependency of Sequel so that it can move out of autoload
+    autoload :Test, 'priority_test/core/test'
+    autoload :TestResult, 'priority_test/core/test_result'
+  end
+
+  def self.configure
+    yield configuration if block_given?
+  end
+
+  def self.configuration
+    @configuration ||= Core::Configuration.new
+  end
+
+  def self.all_tests
+    @all_tests ||= begin
+                     tests = Core::Test.all_in_priority_order
+                     Core::AllTests.new(tests)
+                   end
+  end
+
+  def self.service
+    @service ||= Core::Service.new(all_tests)
+  end
+
+  def self.test_result_collector
+    @collector ||= Core::TestResultCollector.new(all_tests)
+  end
+end

data/lib/priority_test/gateway/migrations/001_create_tests.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  change do
+    create_table(:tests) do
+      primary_key :id
+      String :identifier, :text => true, :null => false
+      String :file_path, :text => true, :null => false
+      Integer :priority, :default => 0, :null => false
+      Numeric :avg_run_time, :size => [10, 6], :default => 0, :null => false
+    end
+  end
+end

data/lib/priority_test/gateway/migrations/002_create_test_results.rb

@@ -0,0 +1,12 @@
+Sequel.migration do
+  change do
+    create_table(:test_results) do
+      primary_key :id
+      String :status, :null => false
+      DateTime :started_at, :null => false
+      Numeric :run_time, :size => [10, 6], :null => false
+      foreign_key :test_id, :tests
+      index :test_id
+    end
+  end
+end

data/lib/priority_test/gateway/sequel.rb

@@ -0,0 +1,33 @@
+require 'sequel'
+require 'sequel/extensions/migration'
+
+module PriorityTest
+  module Gateway
+    class Sequel
+      class << self
+        attr_reader :database
+      end
+
+      def self.setup
+        ::Sequel.database_timezone = :utc
+        @database ||= ::Sequel.connect(PriorityTest.configuration.database)
+        run_migration(database)
+      end
+
+      def self.teardown
+        @database.disconnect if @database
+        @database = nil
+      end
+
+      def self.run_migration(database)
+        ::Sequel::Migrator.apply(database, migrations_dir)
+      end
+
+      private
+
+      def self.migrations_dir
+        File.join(File.expand_path(File.dirname(__FILE__)), 'migrations')
+      end
+    end
+  end
+end