rspec-search-and-destroy 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format doc

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rspec-search-and-destroy.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Jake Goulding
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+RSpec Search-and-Destroy (SAD) runs your RSpec suites to automatically
+find the root causes of test ordering bugs.
+
+# Background
+
+Ideally, tests are independent of each other, but sometimes global
+state can leak out of one test and cause another test to fail. When
+you have many tests, these types of failures can be insidious to
+debug.
+
+# Usage
+
+First, add the appropriate hooks in your `spec_helper.rb`:
+
+```ruby
+require 'rspec-sad'
+
+RSpec.configure do |config|
+  RSpecSearchAndDestroy.configure(config)
+end
+```
+
+Then run the driver program:
+
+```
+# Search and destroy mode
+rspec-sad
+
+# If you have a particular ordering that creates issues
+SPEC_OPTIONS="--seed 12345" rspec-sad
+```
+
+# Search and destroy mode
+
+In the search phase, SAD will run your test suite until a test failure
+occurs, then switch to the destroy phase. In the destroy phase, the
+contributing tests are narrowed down until a single test is found that
+causes the failure.
+
+# Caveats
+
+RSpec SAD requires that your test suite not have any flaky tests. Any
+intermittently failing tests will cause false positives or false
+negatives. In this case, the results will not provide any useful
+information.
+
+# How it works
+
+During the search phase, your test suite will be run once to get the
+set of tests that could contribute to the test failure. The order of
+tests will be saved and passed off to the destroy phase.
+
+During each step of the destroy phase, half of the remaining tests
+will be disabled. If the test continues to fail, these disabled tests
+can be ignored. If the test stops failing, then the currently enabled
+tests can be enabled.
+
+## Implementation notes
+
+The driver process runs RSpec as a subprocess. The two processes
+communicate using environment variables and files. The driver program
+specifies a set of tests to run, and RSpec reports the results back to
+the driver.
+
+The test ordering and filtering is done via a custom
+`config.order_examples` block, and the results are saved using a
+custom formatter.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/rspec-sad

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << "$PWD/../lib/"
+require 'rspec-sad'
+
+include RSpecSearchAndDestroy
+
+class TTYOutput
+  def found(causal_example, failed_example)
+    puts "Culprit found"
+    puts "Run    #{causal_example[:location]}"
+    puts "Before #{failed_example[:location]}"
+  end
+end
+
+output = TTYOutput.new
+selector = BinaryChopExampleSelector.new
+driver = RSpecDriver.new
+
+driver.initial_run
+results = driver.load_run_results
+raise "no failures found" unless results.failed?
+
+bisector = Bisector.new(output, selector, driver)
+bisector.bisect(results.causal_examples, results.failed_example)
+
+driver.cleanup

data/examples/spec/bad_ordering_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe "Tests that fail when run in a specific order" do
+  it "leaves bad global state" do
+    $fail_next = true
+    expect($fail_next).to be_true
+  end
+
+  it "just takes up space" do
+    expect(true).to be_true
+  end
+
+  it "fails when run last" do
+    expect($fail_next).to be_false
+  end
+end

data/examples/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require 'rspec-sad'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  config.color = true
+  config.formatter = 'documentation'
+
+  # For this example, run in an order that is guaranteed to be bad
+  config.order = 'default'
+
+  RSpecSearchAndDestroy.configure(config)
+end

data/lib/rspec-sad.rb

@@ -0,0 +1,157 @@
+require "rspec-search-and-destroy/version"
+require "rspec-search-and-destroy/bisector"
+require "rspec-search-and-destroy/binary_chop_example_selector"
+
+require 'rspec'
+require 'rspec/core/formatters/base_formatter'
+
+module RSpecSearchAndDestroy
+  def self.configure(config)
+    config.add_formatter(OrderFormatter)
+
+    source = LocationSource.new
+    if source.enabled?
+      ordering = ReorderAndFilter.new(source)
+      config.order_examples(&ordering.block)
+    end
+  end
+
+  class OrderFormatter < RSpec::Core::Formatters::BaseFormatter
+    def stop
+      File.open(filename, 'wb') do |f|
+        Marshal.dump(results, f)
+      end
+
+      super
+    end
+
+    private
+
+    def filename
+      ENV['RSPEC_SAD_RESULTS'] or raise "No result filename provided"
+    end
+
+    def results
+      examples.map do |e|
+        {
+          location: e.location,
+          failed: !!e.exception
+        }
+      end
+    end
+  end
+
+  class LocationSource
+    def enabled?
+      filename && File.exist?(filename)
+    end
+
+    def example_locations_to_run
+      raise "LocationSource is not currently enabled" unless enabled?
+
+      File.open(filename, 'rb') do |f|
+        Marshal.load(f)
+      end
+    end
+
+    private
+
+    def filename
+      ENV['RSPEC_SAD_EXAMPLES']
+    end
+  end
+
+  class ReorderAndFilter
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+    end
+
+    def block
+      lambda do |examples|
+        locations = source.example_locations_to_run
+
+        enabled_examples = examples.select do |e|
+          locations.include? e.location
+        end
+
+        enabled_examples.sort_by do |e|
+          locations.index(e.location)
+        end
+      end
+    end
+  end
+
+  class RSpecResults
+    attr_reader :results
+
+    def initialize(results)
+      @results = results
+    end
+
+    def causal_examples
+      results.slice(0, failure_index)
+    end
+
+    def failed_example
+      results[failure_index]
+    end
+
+    def failed?
+      results.find {|result| result[:failed] }
+    end
+
+    private
+
+    def failure_index
+      @failure_index ||= results.find_index { |r| r[:failed] }
+    end
+  end
+
+  class RSpecDriver
+    RESULT_FILE = '/tmp/example-results'
+    EXAMPLE_FILE = '/tmp/examples-to-run'
+
+    def load_run_results
+      File.open(RESULT_FILE, 'rb') do |f|
+        RSpecResults.new(Marshal.load(f))
+      end
+    end
+
+    def initial_run
+      cleanup
+      run_rspec
+    end
+
+    def run_examples(examples)
+      locations = examples.map {|x| x[:location]}
+
+      File.open(EXAMPLE_FILE, 'wb') do |f|
+        Marshal.dump(locations, f)
+      end
+
+      run_rspec
+    end
+
+    def cleanup
+      [EXAMPLE_FILE, RESULT_FILE].each do |fname|
+        File.delete(fname) if File.exist? fname
+      end
+    end
+
+    private
+
+    def run_rspec
+      env = {
+        "RSPEC_SAD_EXAMPLES" => EXAMPLE_FILE,
+        "RSPEC_SAD_RESULTS" => RESULT_FILE
+      }
+      cmd = "rspec"
+
+      success = system(env, cmd)
+
+      raise "unable to run command: #{cmd}" if success.nil?
+    end
+  end
+end

data/lib/rspec-search-and-destroy/binary_chop_example_selector.rb

@@ -0,0 +1,12 @@
+module RSpecSearchAndDestroy
+  class BinaryChopExampleSelector
+    def enable_set(examples)
+      half_size = examples.size / 2
+
+      enabled = examples.slice(0, half_size)
+      disabled = examples.slice(half_size..-1)
+
+      [enabled, disabled]
+    end
+  end
+end

data/lib/rspec-search-and-destroy/bisector.rb

@@ -0,0 +1,35 @@
+module RSpecSearchAndDestroy
+  class Bisector
+    attr_reader :output, :selector, :executor
+
+    def initialize(output, selector, executor)
+      @output = output
+      @selector = selector
+      @executor = executor
+    end
+
+    def bisect(causal_examples, failed_example)
+      case causal_examples.size
+      when 1
+        output.found(causal_examples.first, failed_example)
+        return
+      when 0
+        output.unable_to_bisect
+        return
+      end
+
+      enabled, disabled = selector.enable_set(causal_examples)
+
+      to_run = enabled + [failed_example]
+      executor.run_examples(to_run)
+
+      results = executor.load_run_results
+
+      if results.failed?
+        bisect(enabled, failed_example)
+      else
+        bisect(disabled, failed_example)
+      end
+    end
+  end
+end

data/lib/rspec-search-and-destroy/version.rb

@@ -0,0 +1,3 @@
+module RSpecSearchAndDestroy
+  VERSION = "0.0.1"
+end

data/rspec-search-and-destroy.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rspec-search-and-destroy/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "rspec-search-and-destroy"
+  gem.version       = RSpecSearchAndDestroy::VERSION
+  gem.authors       = ["Jake Goulding"]
+  gem.email         = ["jake.goulding@gmail.com"]
+  gem.description   = %q{Finds RSpec test ordering bugs }
+  gem.summary       = %q{Finds RSpec test ordering bugs }
+  gem.homepage      = "https://github.com/shepmaster/rspec-search-and-destroy"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency('rspec', '>= 2.14.0')
+end

data/spec/binary_chop_example_selector_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+require 'rspec-search-and-destroy/binary_chop_example_selector'
+
+describe RSpecSearchAndDestroy::BinaryChopExampleSelector do
+  subject(:selector) { RSpecSearchAndDestroy::BinaryChopExampleSelector.new }
+
+  context "when there is an even number of examples" do
+    let(:examples) { 4.times.map {|i| double("example #{i}") } }
+
+    it "includes all the examples" do
+      enabled, disabled = selector.enable_set(examples)
+
+      expect(enabled + disabled).to eql(examples)
+    end
+
+    it "enables exactly half of the examples" do
+      enabled, disabled = selector.enable_set(examples)
+
+      expect(enabled.size).to eql(examples.size / 2)
+    end
+
+    it "disables exactly half of the examples" do
+      enabled, disabled = selector.enable_set(examples)
+
+      expect(disabled.size).to eql(examples.size / 2)
+    end
+  end
+
+  context "when there is an odd number of examples" do
+    let(:examples) { 3.times.map {|i| double("example #{i}") } }
+
+    it "includes all the examples" do
+      enabled, disabled = selector.enable_set(examples)
+
+      expect(enabled + disabled).to eql(examples)
+    end
+
+    it "enables about half of the examples" do
+      enabled, disabled = selector.enable_set(examples)
+
+      expect(enabled.size).to be_within(1).of(examples.size / 2)
+    end
+
+    it "disables about half of the examples" do
+      enabled, disabled = selector.enable_set(examples)
+
+      expect(disabled.size).to be_within(1).of(examples.size / 2)
+    end
+  end
+end

data/spec/bisector_spec.rb

@@ -0,0 +1,106 @@
+require 'spec_helper'
+require 'rspec-search-and-destroy/bisector.rb'
+
+describe RSpecSearchAndDestroy::Bisector do
+  subject(:bisector) do
+    RSpecSearchAndDestroy::Bisector.new(output, selector, executor)
+  end
+
+  let(:output)   { double("output") }
+  let(:selector) { double("selector") }
+  let(:executor) { double("executor") }
+
+  let(:failing_example) { double("failing example") }
+
+  context "when there is only one potential test left" do
+    let(:potential_causes) { [double("bad example")] }
+
+    it "reports that it has finished" do
+      expect(output).to receive(:found)
+        .with(potential_causes.first, failing_example)
+
+      bisector.bisect(potential_causes, failing_example)
+    end
+  end
+
+  context "when there are no potential tests left" do
+    let(:potential_causes) { [] }
+
+    it "reports the error condition" do
+      expect(output).to receive(:unable_to_bisect)
+
+      bisector.bisect(potential_causes, failing_example)
+    end
+  end
+
+  context "when executing examples" do
+    let(:potential_causes) { 2.times.map {|i| double("potential example #{i}")} }
+    let(:enabled_examples) { [potential_causes.first] }
+    let(:disabled_examples) { [potential_causes.last] }
+
+    before do
+      selector.stub(:enable_set)
+        .and_return([enabled_examples, disabled_examples])
+
+      # These stubs do not provide useful feedback for these tests,
+      # they are just needed to prevent failure
+      results = double("results", :failed? => true)
+      executor.stub(:load_run_results).and_return(results)
+      output.stub(:found)
+    end
+
+    it "executes enabled examples" do
+      expect(executor).to receive(:run_examples) do |enabled_examples|
+        expect(enabled_examples).to include(*enabled_examples)
+      end
+
+      bisector.bisect(potential_causes, failing_example)
+    end
+
+    it "does not execute disabled examples" do
+      expect(executor).to receive(:run_examples) do |enabled_examples|
+        expect(enabled_examples).to_not include(*disabled_examples)
+      end
+
+      bisector.bisect(potential_causes, failing_example)
+    end
+
+    it "executes the failing test last" do
+      expect(executor).to receive(:run_examples) do |enabled_examples|
+        expect(enabled_examples.last).to eql(failing_example)
+      end
+
+      bisector.bisect(potential_causes, failing_example)
+    end
+
+    context "when the executed tests fail" do
+      before do
+        executor.stub(:run_examples)
+        results = double("results", :failed? => true)
+        executor.stub(:load_run_results).and_return(results)
+      end
+
+      it "continues exploring the enabled tests" do
+        expect(output).to receive(:found)
+          .with(enabled_examples.first, failing_example)
+
+        bisector.bisect(potential_causes, failing_example)
+      end
+    end
+
+    context "when the executed tests do not fail" do
+      before do
+        executor.stub(:run_examples)
+        results = double("results", :failed? => false)
+        executor.stub(:load_run_results).and_return(results)
+      end
+
+      it "continues exploring the disabled tests" do
+        expect(output).to receive(:found)
+          .with(disabled_examples.first, failing_example)
+
+        bisector.bisect(potential_causes, failing_example)
+      end
+    end
+  end
+end

data/spec/rspec_example_ordering_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+require 'rspec-sad'
+
+describe RSpecSearchAndDestroy::ReorderAndFilter do
+  subject(:ordering_block) do
+    RSpecSearchAndDestroy::ReorderAndFilter.new(location_source).block
+  end
+
+  let(:location_source) { double("location source") }
+
+  context "when filtering examples" do
+    let(:examples) do
+      [double('example 1', location: 'location 1'),
+       double('example 2', location: 'location 2')]
+    end
+
+    before do
+      location_source.stub(:example_locations_to_run)
+        .and_return ["location 1"]
+    end
+
+    it "leaves examples that are enabled" do
+      sorted_examples = ordering_block.call(examples)
+
+      expect(sorted_examples).to include examples.first
+    end
+
+    it "removes examples that are not enabled" do
+      sorted_examples = ordering_block.call(examples)
+
+      expect(sorted_examples).to_not include examples.last
+    end
+  end
+
+  context "when reordering examples" do
+    let(:examples) do
+      [double('example 1', location: 'location 1'),
+       double('example 2', location: 'location 2')]
+    end
+
+    before do
+      location_source.stub(:example_locations_to_run)
+        .and_return ["location 2", "location 1"]
+    end
+
+    it "reorders examples to match" do
+      sorted_examples = ordering_block.call(examples)
+
+      expect(sorted_examples).to eql [examples.last, examples.first]
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: rspec-search-and-destroy
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jake Goulding
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.14.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.14.0
+description: ! 'Finds RSpec test ordering bugs '
+email:
+- jake.goulding@gmail.com
+executables:
+- rspec-sad
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/rspec-sad
+- examples/spec/bad_ordering_spec.rb
+- examples/spec/spec_helper.rb
+- lib/rspec-sad.rb
+- lib/rspec-search-and-destroy/binary_chop_example_selector.rb
+- lib/rspec-search-and-destroy/bisector.rb
+- lib/rspec-search-and-destroy/version.rb
+- rspec-search-and-destroy.gemspec
+- spec/binary_chop_example_selector_spec.rb
+- spec/bisector_spec.rb
+- spec/rspec_example_ordering_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/shepmaster/rspec-search-and-destroy
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Finds RSpec test ordering bugs
+test_files:
+- spec/binary_chop_example_selector_spec.rb
+- spec/bisector_spec.rb
+- spec/rspec_example_ordering_spec.rb
+- spec/spec_helper.rb