minitest-bisect 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e56b026d55d6353d8a291fbeb69e5988ad1a9a39
-  data.tar.gz: 4cac6746cd45f0a1ada3b2de84d20130b86c129f
+  metadata.gz: 8e164f0aced9bbb7f0fb1f240ab1a32aa57bec59
+  data.tar.gz: ad40830eea222d045b4a19bce62905d7be7a1f9b
 SHA512:
-  metadata.gz: 45267d07d6c66419face27df11adb64905008e99b1b52fc18044546da377988e683dbf0f643239c2f80c143bf2d6672de4211806051f09efb17ff1cf2e17d7e5
-  data.tar.gz: 4193394a7503e8bfe62aa217874d9f838e2e5de1ee3da94a2ed0b5395631a4c2a0b1e24c90903c527e025e4c9ba4e42d629d0a5f10f359f5ddf54bf71ed7365c
+  metadata.gz: c427f5b79be6ee92d49ba4dbe7711362a9bc5f530a5845bd20ecb9260918f4e6bb0a540840b2f5d2d97d27f345a05068d9944543acf39d7a8418dd1211c67f0c
+  data.tar.gz: 82c9c80008423278b9318b9199494abbcdd425eb7b18f2ab8eb3b76d10de6d85589b53a85883a09e73542e1469dcacee8758f5580e7550e5e4320d4ef8631dac

data/History.rdoc

@@ -1,3 +1,11 @@
+=== 1.4.0 / 2016-06-15
+
+* 3 minor enhancements:
+
+  * Added better timing output, so you can estimate much of a coffee break you get.
+  * Added path_expander support.
+  * Added verification step to ensure you're not trying to bisect a non-test-order-dependency bug.
+
 === 1.3.1 / 2016-02-18
 
 * 2 bug fixes:

data/README.rdoc

@@ -140,6 +140,43 @@ quicker with time.
 
 * sudo gem install minitest-bisect
 
+== FAQ:
+
+==== Q: What if my tests are in multiple subdirectories?
+
+This is assuming the following doesn't work for you:
+
+  minitest_bisect --seed 3911 test/*_test.rb
+
+because you need it to span multiple sub-directories.
+
+A: Use your shell to find the files for you
+
+  minitest_bisect --seed 3911 $(find test -type f -name \*_test.rb)
+
+or
+
+  minitest_bisect --seed 3911 test/**/*_test.rb
+
+==== Q: How do I resolve an "invalid option: \--server" error?
+  $ MTB_VERBOSE=2 minitest_bisect --seed 3911 example/test*.rb
+  reproducing...
+  ruby -Ilib -e 'require "./example/test_bad1.rb" ; require "./example/test_bad4.rb"' -- --seed 3911 --server 33352
+
+  invalid option: --server
+
+  minitest options:
+      -h, --help                       Display this help.
+      -s, --seed SEED                  Sets random seed. Also via env. Eg: SEED=n rake
+      -v, --verbose                    Verbose. Show progress processing files.
+      -n, --name PATTERN               Filter run on /regexp/ or string.
+
+  Known extensions: guard_minitest, pride
+      -p, --pride                      Pride. Show your testing pride!
+  Reproduction run passed? Aborting. Try running with MTB_VERBOSE=2 to verify.
+
+A: Try adding "<tt>gem minitest-bisect</tt>" to your Gemfile and running "<tt>bundle install</tt>"
+
 == LICENSE:
 
 (The MIT License)

data/Rakefile

@@ -15,6 +15,7 @@ Hoe.spec "minitest-bisect" do
 
   dependency "rake", "> 0", :development
   dependency "minitest-server", "~> 1.0"
+  dependency "path_expander", "~> 1.0"
 end
 
 require "rake/testtask"
@@ -45,7 +46,7 @@ def req glob
   Dir["#{glob}.rb"].map { |s| "require #{s.inspect}" }.join ";"
 end
 
-task :repro do
+task :repro => :isolate do
   unless ENV.key? "SLEEP" then
     warn "NOTE: Defaulting to sleeping 0.01 seconds per test."
     warn "NOTE: Use SLEEP=0 to disable or any other value to simulate your tests."
@@ -60,7 +61,7 @@ task :repro do
   run "#{ruby} bin/minitest_bisect -Ilib --seed 3911 example/test*.rb"
 end
 
-task :many do
+task :many => :isolate do
   unless ENV.key? "SLEEP" then
     warn "NOTE: Defaulting to sleeping 0.01 seconds per test."
     warn "NOTE: Use SLEEP=0 to disable or any other value to simulate your tests."

data/lib/minitest/bisect.rb

@@ -2,9 +2,41 @@ require "minitest/find_minimal_combination"
 require "minitest/server"
 require "shellwords"
 require "rbconfig"
+require "path_expander"
 
 class Minitest::Bisect
-  VERSION = "1.3.1"
+  VERSION = "1.4.0"
+
+  class PathExpander < ::PathExpander
+    TEST_GLOB = "**/{test_*,*_test,spec_*,*_spec}.rb" # :nodoc:
+
+    attr_accessor :rb_flags
+
+    def initialize args = ARGV # :nodoc:
+      super args, TEST_GLOB
+      self.rb_flags = []
+    end
+
+    ##
+    # Overrides PathExpander#process_flags to filter out ruby flags
+    # from minitest flags. Only supports -I<paths>, -d, and -w for
+    # ruby.
+
+    def process_flags flags
+      flags.reject { |flag| # all hits are truthy, so this works out well
+        case flag
+        when /^-I(.*)/ then
+          rb_flags << flag
+        when /^-d/ then
+          rb_flags << flag
+        when /^-w/ then
+          rb_flags << flag
+        else
+          false
+        end
+      }
+    end
+  end
 
   mtbv = ENV["MTB_VERBOSE"].to_i
   SHH = case
@@ -41,19 +73,22 @@ class Minitest::Bisect
     # not clearing culprits on purpose
   end
 
-  def run files
+  def run args
     Minitest::Server.run self
 
     cmd = nil
 
     if :until_I_have_negative_filtering_in_minitest != 0 then
-      files, flags = files.partition { |arg| File.file? arg }
-      rb_flags, mt_flags = flags.partition { |arg| arg =~ /^-I/ }
+      mt_flags = args.dup
+      expander = Minitest::Bisect::PathExpander.new mt_flags
+
+      files = expander.process
+      rb_flags = expander.rb_flags
       mt_flags += ["--server", $$]
 
       cmd = bisect_methods build_files_cmd(files, rb_flags, mt_flags)
     else
-      cmd = bisect_methods bisect_files files
+      cmd = bisect_methods bisect_files args
     end
 
     puts "Final reproduction:"
@@ -95,25 +130,31 @@ class Minitest::Bisect
   def bisect_methods cmd
     self.mode = :methods
 
-    puts "reproducing..."
-    system "#{build_methods_cmd cmd} #{SHH}"
-    abort "Reproduction run passed? Aborting. Try running with MTB_VERBOSE=2 to verify." unless tainted?
-    puts "reproduced"
+    time_it "reproducing...", build_methods_cmd(cmd)
 
-    # from: {"file.rb"=>{"Class"=>["test_method1", "test_method2"]}}
-    #   to: ["Class#test_method1", "Class#test_method2"]
-    bad = failures.values.map { |h|
-      h.map { |k,vs| vs.map { |v| "#{k}##{v}" } }
-    }.flatten
+    unless tainted? then
+      $stderr.puts "Reproduction run passed? Aborting."
+      abort "Try running with MTB_VERBOSE=2 to verify."
+    end
+
+    bad = map_failures
 
     raise "Nothing to verify against because every test failed. Aborting." if
       culprits.empty? && seen_bad
 
+    time_it "verifying...", build_methods_cmd(cmd, [], bad)
+
+    new_bad = map_failures
+
+    if bad == new_bad then
+      warn "Tests fail by themselves. This may not be an ordering issue."
+    end
+
     # culprits populated by initial reproduction via minitest/server
     found, count = culprits.find_minimal_combination_and_count do |test|
-      puts "# of culprit methods: #{test.size}"
+      prompt = "# of culprit methods: #{test.size}"
 
-      system "#{build_methods_cmd cmd, test, bad} #{SHH}"
+      time_it prompt, build_methods_cmd(cmd, test, bad)
 
       self.tainted?
     end
@@ -127,6 +168,21 @@ class Minitest::Bisect
     cmd
   end
 
+  def time_it prompt, cmd
+    print prompt
+    t0 = Time.now
+    system "#{cmd} #{SHH}"
+    puts " in %.2f sec" % (Time.now - t0)
+  end
+
+  def map_failures
+    # from: {"file.rb"=>{"Class"=>["test_method1", "test_method2"]}}
+    #   to: ["Class#test_method1", "Class#test_method2"]
+    failures.values.map { |h|
+      h.map { |k,vs| vs.map { |v| "#{k}##{v}" } }
+    }.flatten.sort
+  end
+
   def build_files_cmd culprits, rb, mt
     reset
 
@@ -135,26 +191,32 @@ class Minitest::Bisect
     %(#{RUBY} #{rb.shelljoin} -e '#{tests}' -- #{mt.map(&:to_s).shelljoin})
   end
 
-  def build_methods_cmd cmd, culprits = [], bad = nil
-    reset
+  def build_re bad
+    re = []
 
-    if bad then
-      re = []
+    # bad by class, you perv
+    bbc = bad.map { |s| s.split(/#/, 2) }.group_by(&:first)
 
-      # bad by class, you perv
-      bbc = (culprits + bad).map { |s| s.split(/#/, 2) }.group_by(&:first)
+    bbc.each do |klass, methods|
+      methods = methods.map(&:last).flatten.uniq.map { |method|
+        method.gsub(/([`'"!?&\[\]\(\)\|\+])/, '\\\\\1')
+      }
 
-      bbc.each do |klass, methods|
-        methods = methods.map(&:last).flatten.uniq.map { |method|
-          method.gsub(/([`'"!?&\[\]\(\)\|\+])/, '\\\\\1')
-        }
+      re << /#{klass}#(?:#{methods.join "|"})/.to_s[7..-2] # (?-mix:...)
+    end
 
-        re << /#{klass}#(?:#{methods.join "|"})/.to_s[7..-2] # (?-mix:...)
-      end
+    re = re.join("|").to_s.gsub(/-mix/, "")
+
+    "/^(?:#{re})$/"
+  end
 
-      re = re.join("|").to_s.gsub(/-mix/, "")
+  def build_methods_cmd cmd, culprits = [], bad = nil
+    reset
+
+    if bad then
+      re = build_re culprits + bad
 
-      cmd += " -n \"/^(?:#{re})$/\"" if bad
+      cmd += " -n \"#{re}\"" if bad
     end
 
     if ENV["MTB_VERBOSE"].to_i >= 1 then

data/test/minitest/test_bisect.rb

@@ -4,7 +4,236 @@ require "minitest/bisect"
 module TestMinitest; end
 
 class TestMinitest::TestBisect < Minitest::Test
+  attr_accessor :bisect
+
+  def setup
+    self.bisect = Minitest::Bisect.new
+  end
+
+  def test_class_run
+    skip "Need to write test_class_run"
+  end
+
+  def test_bisect_files
+    skip "Need to write test_bisect_files"
+  end
+
+  def test_bisect_methods
+    skip "Need to write test_bisect_methods"
+  end
+
+  def test_build_files_cmd
+    files = %w[a.rb b.rb c.rb]
+    rb    = %w[-Ilib:test]
+    mt    = %w[--seed 42]
+
+    ruby = Minitest::Bisect::RUBY
+    body = "require \"./a.rb\" ; require \"./b.rb\" ; require \"./c.rb\""
+
+    exp = "#{ruby} -Ilib:test -e '#{body}' -- --seed 42"
+    act = bisect.build_files_cmd(files, rb, mt)
+
+    assert_equal exp, act
+  end
+
+  def test_build_methods_cmd
+    cmd = "cmd"
+    assert_equal "cmd", bisect.build_methods_cmd(cmd)
+  end
+
+  def test_build_methods_cmd_verify
+    cmd = "cmd"
+    cul = []
+    bad = %w[A#test_1 B#test_2]
+
+    exp = "cmd -n \"/^(?:A#(?:test_1)|B#(?:test_2))$/\""
+
+    assert_equal exp, bisect.build_methods_cmd(cmd, cul, bad)
+  end
+
+  def test_build_methods_cmd_verify_same
+    cmd = "cmd"
+    cul = []
+    bad = %w[C#test_5 C#test_6]
+
+    exp = "cmd -n \"/^(?:C#(?:test_5|test_6))$/\""
+
+    assert_equal exp, bisect.build_methods_cmd(cmd, cul, bad)
+  end
+
+  def test_build_methods_cmd_full
+    cmd = "cmd"
+    cul = %w[A#test_1 A#test_2 B#test_3 B#test_4]
+    bad = %w[C#test_5 C#test_6]
+
+    a = "A#(?:test_1|test_2)"
+    b = "B#(?:test_3|test_4)"
+    c = "C#(?:test_5|test_6)"
+    exp = "cmd -n \"/^(?:#{a}|#{b}|#{c})$/\""
+
+    assert_equal exp, bisect.build_methods_cmd(cmd, cul, bad)
+  end
+
+  def test_build_re
+    bad = %w[A#test_1 B#test_2]
+
+    exp = "/^(?:A#(?:test_1)|B#(?:test_2))$/"
+
+    assert_equal exp, bisect.build_re(bad)
+  end
+
+  def test_build_re_same
+    bad = %w[C#test_5 C#test_6]
+
+    exp = "/^(?:C#(?:test_5|test_6))$/"
+
+    assert_equal exp, bisect.build_re(bad)
+  end
+
+  def test_build_re_escaping
+    bad = ["Some Class#It shouldn't care what the name is"]
+
+    exp = "/^(?:Some Class#(?:It shouldn\\'t care what the name is))$/"
+
+    assert_equal exp, bisect.build_re(bad)
+  end
+
+  def test_map_failures
+    bisect.failures =
+      {
+       "file.rb" => { "Class" => %w[test_method1 test_method2] },
+       "blah.rb" => { "Apple" => %w[test_method3 test_method4] },
+      }
+
+    exp = %w[
+           Apple#test_method3
+           Apple#test_method4
+           Class#test_method1
+           Class#test_method2
+          ]
+
+    assert_equal exp, bisect.map_failures
+  end
+
+  def test_minitest_result
+    bisect.mode = :methods
+    bisect.reset
+
+    bisect.minitest_result "file.rb", "TestClass", "test_method", [], 1, 1
+
+    assert_equal false, bisect.tainted
+    assert_empty bisect.failures
+    assert_equal ["TestClass#test_method"], bisect.culprits
+  end
+
+  def test_minitest_result_skip
+    bisect.mode = :methods
+    bisect.reset
+
+    fail = Minitest::Skip.new("woot")
+
+    bisect.minitest_result "file.rb", "TestClass", "test_method", [fail], 1, 1
+
+    assert_equal false, bisect.tainted
+    assert_empty bisect.failures
+    assert_equal ["TestClass#test_method"], bisect.culprits
+  end
+
+  def test_minitest_result_fail
+    bisect.mode = :methods
+    bisect.reset
+
+    fail = Minitest::Assertion.new "msg"
+
+    bisect.minitest_result "file.rb", "TestClass", "test_method", [fail], 1, 1
+
+    exp = {"file.rb" => {"TestClass" => ["test_method"] }}
+
+    assert_equal true, bisect.tainted
+    assert_equal exp, bisect.failures
+    assert_empty bisect.culprits
+  end
+
+  def test_minitest_result_error
+    bisect.mode = :methods
+    bisect.reset
+
+    fail = Minitest::UnexpectedError.new RuntimeError.new("woot")
+
+    bisect.minitest_result "file.rb", "TestClass", "test_method", [fail], 1, 1
+
+    exp = {"file.rb" => {"TestClass" => ["test_method"] }}
+
+    assert_equal true, bisect.tainted
+    assert_equal exp, bisect.failures
+    assert_empty bisect.culprits
+  end
+
+  def test_minitest_start
+    bisect.failures["file.rb"]["Class"] << "test_bad1"
+
+    bisect.minitest_start
+
+    assert_empty bisect.failures
+  end
+
+  def test_reset
+    bisect.seen_bad = true
+    bisect.tainted  = true
+    bisect.failures["file.rb"]["Class"] << "test_bad1"
+    bisect.culprits << "A#test_1" << "B#test_2"
+
+    bisect.reset
+
+    assert_equal false, bisect.seen_bad
+    assert_equal false, bisect.tainted
+    assert_empty bisect.failures
+    assert_equal %w[A#test_1 B#test_2], bisect.culprits
+  end
+
+  def test_run
+    skip "Need to write test_run"
+  end
+
+  def test_time_it
+    exp = /\Ado stuff: in 0.\d\d sec\n\z/
+
+    assert_output exp, "" do
+      bisect.time_it "do stuff:", "echo you should not see me"
+    end
+  end
+end
+
+class TestMinitest::TestBisect::TestPathExpander < Minitest::Test
   def test_sanity
-    # nothing yet... ugh.
+    args = %w[1 -Iblah 2 -d 3 -w 4 5 6]
+
+    mtbpe = Minitest::Bisect::PathExpander
+    expander = mtbpe.new args
+
+    assert_kind_of PathExpander, expander
+    assert_equal [], expander.rb_flags
+    assert_same mtbpe::TEST_GLOB, expander.glob
+  end
+
+  def test_process_flags
+    args = %w[1 -Iblah 2 -d 3 -w 4 5 6]
+
+    expander = Minitest::Bisect::PathExpander.new args
+
+    exp_files = %w[1 2 3 4 5 6]
+    exp_flags = %w[-Iblah -d -w]
+
+    files = expander.process_flags(args)
+
+    assert_equal files, exp_files
+
+    # process_flags only filters and does not mutate args
+    assert_same args, expander.args
+    refute_equal args, exp_files
+    refute_equal files, args
+
+    # separates rb_flags out for separate handling
+    assert_equal exp_flags, expander.rb_flags
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: minitest-bisect
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Ryan Davis
@@ -29,7 +29,7 @@ cert_chain:
   qx3h45R1CAsObX0SQDIT+rRbQrtKz1GHIZTOFYvEJjUY1XmRTZupD3CJ8Q7sDqSy
   NLq5jm1fq6Y9Uolu3RJbmycf
   -----END CERTIFICATE-----
-date: 2016-02-19 00:00:00.000000000 Z
+date: 2016-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest-server
@@ -46,19 +46,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: path_expander
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '5.8'
-  type: :development
+        version: '1.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '5.8'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -93,14 +93,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.14'
+        version: '3.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.14'
+        version: '3.15'
 description: |-
   Hunting down random test failures can be very very difficult,
   sometimes impossible, but minitest-bisect makes it easy.