rspec-bisect 0.0.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1fdcaa0d3095d1ab62552d1f3eec1aa095c9d65f
-  data.tar.gz: a5a5f471b893b05ec9f62833972abd7b238fb13d
+  metadata.gz: fa219fdf62e2b1443b569705aa35d614fff5f054
+  data.tar.gz: de6a63d2fee72bbd3ac882b52c1d66878b5ca4d3
 SHA512:
-  metadata.gz: 8936e9b385e22c47f060b1777706cf7012cd8ba980633e9c6bcc093faeabf05c3a85feb7ba2202a6cd134276ccd85fb7ff493ad4cde66287b704b56a76bf3593
-  data.tar.gz: d94969a155f4d9922820f2ebbf1b223423706585ceb9019c2f4f53d1dd8e7e7a0b93751bc02c89dc54c75fa25502506d4b7f5c44e290d0937cfa1138ab9c209e
+  metadata.gz: c9ea2dcdf6e665205f547c10f5cea43c50f376e2c031f673a0d27061599db2a7a197d6527fb89ed2dc7e72d17fd1c109216ec5b7ea87d2952ba31a5d2f9db575
+  data.tar.gz: 44261365ac61fae0389adcebd4a93ffc73b143cd5ac42c763b1ac0b4723dfd2a666f5afdf7ac47b0746ec0ea7195b873080b6b73b363090ffd49ee66c6bb4513

data/README.md

@@ -20,7 +20,11 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+So you think you have order dependencies? This tool will tell you exactly which tests are causing that dependency. It won't try to figure out if one exists or tell you why it's happening.
+
+Have your rspec config set so that ```rspec --seed 1234``` runs your test suite with the specified seed.
+
+Run ```rspec-bisect --seed 1234``` and rspec-bisect will tell you which tests are from an order dependency and the minimum tests required to reproduce it.
 
 ## Contributing
 

data/bin/rspec-bisect

@@ -1,15 +1,107 @@
 #!/usr/bin/env ruby
 
 require 'json'
+require 'colorize'
+require 'optparse'
 
-result = `rspec --format json`
+options = {}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: rspec-bisect [options]'
+
+  opts.on('-s', '--seed N', Integer, 'Seed that causes order dependencies') do |s|
+    options[:seed] = s
+  end
+
+  opts.on('--help', 'Show this message') do
+    puts opts
+    exit
+  end
+end.parse!
+
+if options[:seed].nil?
+  puts 'Running tests with no seed'
+else
+  puts "Running tests with seed #{options[:seed]}"
+end
+
+result = `rspec --format json #{options[:seed].nil? ? '' : "--seed #{options[:seed]}"}`
 parsed = JSON.parse result
 
-failure_count = parsed["summary"]["failure_count"]
+examples = parsed['examples']
+failure_count = parsed['summary']['failure_count']
 if failure_count > 0
-  puts "#{failure_count} failing test#{failure_count > 1 ? 's' : ''}."
+  puts "#{failure_count} failing test#{failure_count > 1 ? 's' : ''}.".red
+else
+  puts 'No failing tests.'.green
+end
+
+failing_examples = examples.select { |e| e['status'] == 'failed' }
+
+def examples_as_rspec_params(examples)
+  examples.map { |e| "#{e['file_path']}:#{e['line_number']}" }.join ' '
+end
+
+def run_examples_command(examples)
+  "rspec #{examples_as_rspec_params(examples)}"
+end
+
+def run_examples(examples)
+  `#{run_examples_command(examples)}`
+end
+
+def last_command_passed?
+  $?.exitstatus == 0
+end
+
+def last_command_failed?
+  not last_command_passed?
+end
+
+order_dependent_examples = failing_examples.select do |example|
+  run_examples([example])
+  last_command_passed?
+end
+
+if order_dependent_examples.size > 0
+  puts "Order dependenc#{ order_dependent_examples.size > 1 ? 'ies' : 'y'} detected:"
+  order_dependent_examples.each do |example|
+    puts "\t#{example['full_description']}".red
+  end
 else
-  puts "No failing tests."
+  puts 'No order dependencies.'.green
+end
+
+order_dependent_examples.each do |example|
+  puts
+  puts "Culprits for #{example['full_description']}:"
+
+  culprits = examples.take_while do |e|
+    example['file_path'] != e['file_path'] ||
+        example['line_number'] != e['line_number']
+  end
+
+  culprit_count_theory = 1
+  while culprits.size > culprit_count_theory
+    found_useless_group = false
+    culprits.each_slice(culprits.size / (culprit_count_theory + 1)) do |excluded_culprits|
+      culprit_group = culprits - excluded_culprits
+
+      run_examples culprit_group + [example]
+      if last_command_failed?
+        culprits = culprit_group
+
+        found_useless_group = true
+        break
+      end
+    end
+
+    culprit_count_theory += 1 unless found_useless_group
+  end
+
+  culprits.each { |candidate| puts candidate['full_description'].green }
+
+  puts example['full_description'].red
+
+  puts (run_examples_command culprits + [example]).red
 end
 
-puts "No order dependencies."

data/features/accepts_seeds.feature

@@ -0,0 +1,23 @@
+Feature: accepts seeds for first rspec run
+
+  @wip
+  Scenario: suite requires seeds
+    Given a file named "spec/failing_spec.rb" with:
+    """ruby
+      RSpec.describe 'a test that' do
+        cause_failure = false
+        it 'fails' do
+          expect(cause_failure).to be false
+        end
+
+        it 'passes' do
+          cause_failure = true
+        end
+      end
+      """
+    When I run `rspec-bisect --seed 2`
+    Then the output should contain:
+    """
+    a test that passes
+    a test that fails
+    """

data/features/failing_tests/multiple_tests_order_dependencies.feature

@@ -0,0 +1,27 @@
+Feature: prints order dependencies for multiple test dependencies
+
+  Scenario: two tests causes another test to fail
+    Given a file named "spec/failing_spec.rb" with:
+      """ruby
+      RSpec.describe 'a test that' do
+        cause_failure = 0
+        it 'passes' do
+          cause_failure += 1
+        end
+
+        it 'also passes' do
+          cause_failure += 1
+        end
+
+        it 'fails' do
+          expect(cause_failure).to be < 2
+        end
+      end
+      """
+    When I run `rspec-bisect`
+    Then the output should contain:
+    """
+    a test that passes
+    a test that also passes
+    a test that fails
+    """

data/features/failing_tests/prints_command_to_reproduce.feature

@@ -0,0 +1,21 @@
+Feature: prints order dependencies for multiple test dependencies
+
+  Scenario: one test causes another test to fail
+    Given a file named "spec/failing_spec.rb" with:
+      """ruby
+      RSpec.describe 'a test that' do
+        cause_failure = false
+        it 'passes' do
+          cause_failure = true
+        end
+
+        it 'fails' do
+          expect(cause_failure).to be false
+        end
+      end
+      """
+    When I run `rspec-bisect`
+    Then the output should contain:
+      """
+      rspec ./spec/failing_spec.rb:3 ./spec/failing_spec.rb:7
+      """

data/features/failing_tests/single_test_order_dependencies.feature

@@ -0,0 +1,53 @@
+Feature: prints order dependencies
+
+  Scenario: one test causes another test to fail
+    Given a file named "spec/failing_spec.rb" with:
+      """ruby
+      RSpec.describe 'a test that' do
+        cause_failure = false
+        it 'passes' do
+          cause_failure = true
+        end
+
+        it 'fails' do
+          expect(cause_failure).to be false
+        end
+      end
+      """
+    When I run `rspec-bisect`
+    Then the output should contain:
+    """
+    a test that passes
+    a test that fails
+    """
+
+  Scenario: one test causes two other tests to fail
+    Given a file named "spec/failing_spec.rb" with:
+      """ruby
+      RSpec.describe 'a test that' do
+        cause_failure = false
+        it 'passes' do
+          cause_failure = true
+        end
+
+        it 'fails' do
+          expect(cause_failure).to be false
+        end
+
+        it 'also fails' do
+          expect(cause_failure).to be false
+        end
+      end
+      """
+    When I run `rspec-bisect`
+    Then the output should contain:
+    """
+    a test that passes
+    a test that fails
+    """
+
+    And the output should contain:
+    """
+    a test that passes
+    a test that also fails
+    """

data/features/failing_tests/which_tests_have_order_dependencies.feature

@@ -0,0 +1,46 @@
+Feature: which tests have order dependencies
+
+  Scenario: one test causes another test to fail
+    Given a file named "spec/failing_spec.rb" with:
+      """ruby
+      RSpec.describe 'a test that' do
+        cause_failure = false
+        it 'passes' do
+          cause_failure = true
+        end
+
+        it 'fails' do
+          expect(cause_failure).to be false
+        end
+      end
+      """
+    When I run `rspec-bisect`
+    Then the output should contain "1 failing test."
+    And the output should not contain "No order dependencies."
+    And the output should contain "Order dependency detected:"
+    And the output should contain "a test that fails"
+
+  Scenario: one test causes two other tests to fail
+    Given a file named "spec/failing_spec.rb" with:
+      """ruby
+      RSpec.describe 'a test that' do
+        cause_failure = false
+        it 'passes' do
+          cause_failure = true
+        end
+
+        it 'fails' do
+          expect(cause_failure).to be false
+        end
+
+        it 'also fails' do
+          expect(cause_failure).to be false
+        end
+      end
+      """
+    When I run `rspec-bisect`
+    Then the output should contain "2 failing tests."
+    And the output should not contain "No order dependencies."
+    And the output should contain "Order dependencies detected:"
+    And the output should contain "a test that fails"
+    And the output should contain "a test that also fails"

data/features/support/env.rb

@@ -3,5 +3,5 @@ require 'rspec/bisect'
 require 'aruba/cucumber'
 
 Before do
-  @aruba_timeout_seconds = 5
+  @aruba_timeout_seconds = 60
 end

data/lib/rspec/bisect/version.rb

@@ -1,5 +1,5 @@
 module Rspec
   module Bisect
-    VERSION = "0.0.1"
+    VERSION = "0.1.1"
   end
 end

data/rspec-bisect.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["shelby@shelbyd.com"]
   spec.summary       = %q{Detect order dependencies in rspec test suites.}
   spec.description   = %q{Detect order dependencies in rspec test suites. Also print the minimal set of tests to reproduce the failure(s).}
-  spec.homepage      = ""
+  spec.homepage      = "https://github.com/shelbyd/rspec-bisect"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")
@@ -20,7 +20,8 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "cucumber"
-  spec.add_development_dependency "aruba"
-  spec.add_development_dependency "rspec", "~> 3"
+  spec.add_development_dependency "cucumber", "1.3.17"
+  spec.add_development_dependency "aruba", "0.6.1"
+  spec.add_dependency "rspec", "~> 3"
+  spec.add_dependency "colorize", "0.7.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspec-bisect
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.1
 platform: ruby
 authors:
 - Shelby Doolittle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-30 00:00:00.000000000 Z
+date: 2014-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -42,30 +42,30 @@ dependencies:
   name: cucumber
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.17
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.17
 - !ruby/object:Gem::Dependency
   name: aruba
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -73,13 +73,27 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.3
 description: Detect order dependencies in rspec test suites. Also print the minimal
   set of tests to reproduce the failure(s).
 email:
@@ -95,14 +109,19 @@ files:
 - README.md
 - Rakefile
 - bin/rspec-bisect
+- features/accepts_seeds.feature
+- features/failing_tests/multiple_tests_order_dependencies.feature
 - features/failing_tests/no_order_dependencies.feature
+- features/failing_tests/prints_command_to_reproduce.feature
+- features/failing_tests/single_test_order_dependencies.feature
+- features/failing_tests/which_tests_have_order_dependencies.feature
 - features/no_tests.feature
 - features/step_definitions/rspec-bisect_steps.rb
 - features/support/env.rb
 - lib/rspec/bisect.rb
 - lib/rspec/bisect/version.rb
 - rspec-bisect.gemspec
-homepage: ''
+homepage: https://github.com/shelbyd/rspec-bisect
 licenses:
 - MIT
 metadata: {}
@@ -127,7 +146,12 @@ signing_key:
 specification_version: 4
 summary: Detect order dependencies in rspec test suites.
 test_files:
+- features/accepts_seeds.feature
+- features/failing_tests/multiple_tests_order_dependencies.feature
 - features/failing_tests/no_order_dependencies.feature
+- features/failing_tests/prints_command_to_reproduce.feature
+- features/failing_tests/single_test_order_dependencies.feature
+- features/failing_tests/which_tests_have_order_dependencies.feature
 - features/no_tests.feature
 - features/step_definitions/rspec-bisect_steps.rb
 - features/support/env.rb