minitest-bisect 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6a2f0e5308b623a3bd5ba94ef584fec183e7dd51
+  data.tar.gz: 62eb672d93534aee515afcbf939b01d01fff7ba3
+SHA512:
+  metadata.gz: ded844dfc9c7dffd7a598b6a2a5fc7a3dd2de11d7091856b508e303a3f1f366797df0dcbe9e1aaa1fa8eec0002f6d460a682878463624ca9b80656671bc0fdac
+  data.tar.gz: bffd131f915c60ed1e2185f957ffb6ac523008690b49c2a03786f9ba29c0c20a2720aad6c5090f2a8dd3d99a17ee8acd4b663f79718486dff902360d2f82d22a

data.tar.gz.sig

@@ -0,0 +1,2 @@
-��n��9;`�	n���'=x�e�7<\�K���
���]��6����4y�cP�]�(?>��|Qq����&jo�;7���L��WU��8\i��@3UMuM
+���
+�ۗgEc�m1��x���wN���H��$��*Y���j����	W����N.���S`{J��\��:���X7���r�u���Z�vѵ��'���`~�ESu铱�j9顅6�2�S������=ȸ��[k�j�vd�u+�n����

data/.autotest

@@ -0,0 +1,26 @@
+# -*- ruby -*-
+
+require "autotest/restart"
+
+Autotest.add_hook :initialize do |at|
+  at.testlib = "minitest/autorun"
+  at.add_exception "tmp"
+
+  # at.extra_files << "../some/external/dependency.rb"
+  #
+  # at.libs << ":../some/external"
+  #
+  # at.add_exception "vendor"
+  #
+  # at.add_mapping(/dependency.rb/) do |f, _|
+  #   at.files_matching(/test_.*rb$/)
+  # end
+  #
+  # %w(TestA TestB).each do |klass|
+  #   at.extra_class_map[klass] = "test/test_misc.rb"
+  # end
+end
+
+# Autotest.add_hook :run_command do |at|
+#   system "rake build"
+# end

data/History.rdoc

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2014-07-16
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,30 @@
+.autotest
+History.rdoc
+Manifest.txt
+README.rdoc
+Rakefile
+bin/minitest_bisect
+example-many/helper.rb
+example-many/test_bad1.rb
+example-many/test_bad2.rb
+example-many/test_bad3.rb
+example-many/test_bad4.rb
+example-many/test_bad5.rb
+example-many/test_bad6.rb
+example-many/test_bad7.rb
+example-many/test_bad8.rb
+example/helper.rb
+example/test_bad1.rb
+example/test_bad2.rb
+example/test_bad3.rb
+example/test_bad4.rb
+example/test_bad5.rb
+example/test_bad6.rb
+example/test_bad7.rb
+example/test_bad8.rb
+lib/minitest/bisect.rb
+lib/minitest/find_minimal_combination.rb
+lib/minitest/server.rb
+lib/minitest/server_plugin.rb
+test/minitest/test_bisect.rb
+test/minitest/test_find_minimal_combination.rb

data/README.rdoc

@@ -0,0 +1,166 @@
+= minitest-bisect
+
+home :: https://github.com/seattlerb/minitest-bisect
+rdoc :: http://docs.seattlerb.org/minitest-bisect
+
+== DESCRIPTION:
+
+Hunting down random test failures can be very very difficult,
+sometimes impossible, but minitest-bisect makes it easy.
+
+minitest-bisect helps you isolate and debug random test failures.
+
+If your tests only fail randomly, you can reproduce the error
+consistently by using `--seed <num>`, but what then? How do you figure
+out which combination of tests out of hundreds are responsible for the
+failure? You know which test is failing, but what others are causing
+it to fail or were helping it succeed in a different order? That's
+what minitest-bisect does best.
+
+== FEATURES/PROBLEMS:
+
+* minitest_bisect first runs your tests on a per-file basis to
+  minimize the number of tests you need to sift through.
+* minitest_bisect next runs the minimized files and figures out your
+  exact failure reproduction.
+
+== SYNOPSIS:
+
+Let's say you have a bunch of test files and they fail sometimes, but
+not consistently. You get a run that fails, so you record the
+randomization seed that minitest displays at the top of every run.
+
+=== Original Failure:
+
+Here's an example run that fails randomly:
+
+  $ rake
+  ruby -I.:lib -e 'require "example/test_bad1.rb";require "example/test_bad2.rb";require "example/test_bad3.rb";require "example/test_bad4.rb";require "example/test_bad5.rb";require "example/test_bad6.rb";require "example/test_bad7.rb";require "example/test_bad8.rb"'
+  Run options: --seed 3911
+
+  # Running:
+
+  ..............................................................................
+  ..............................................................................
+  ..............................................................................
+  ..............................................................................
+  .........................................F....................................
+  ..............................................................................
+  ..............................................................................
+  ..............................................................................
+  ..............................................................................
+  ..............................................................................
+  ....................
+
+  Finished in 200.836561s, 3.9833 runs/s, 3.9784 assertions/s.
+
+    1) Failure:
+  TestBad4#test_bad4_4 [example/helper.rb:16]:
+  muahahaha order dependency bug!
+
+  800 runs, 799 assertions, 1 failures, 0 errors, 0 skips
+
+=== Minimization and Isolation:
+
+The problem is about how to efficiently figure out the minimal
+reproduction of the random failure so you can actually focus on the
+problem and not all the noise.
+
+So, you run the tests again, but this time with minitest_bisect.
+Provide the seed given in the failure so the tests always run in the
+same order and reproduce every time.
+
+minitest_bisect with first minimize the number of files, then it will
+turn around and minimize the number of methods.
+
+  $ minitest_bisect --seed 3911 example/test*.rb
+  reproducing...
+  reproduced
+  # of culprit files: 4
+  # of culprit files: 2
+  # of culprit files: 2
+  # of culprit files: 2
+  # of culprit files: 2
+
+  Minimal files found in 5 steps:
+
+  ruby -Ilib -e 'require "./example/test_bad1.rb" ; require "./example/test_bad4.rb"' -- --seed 3911 -s 48222
+
+  reproducing...
+  reproduced
+  # of culprit methods: 64
+  # of culprit methods: 64
+  # of culprit methods: 32
+  # of culprit methods: 16
+  # of culprit methods: 8
+  # of culprit methods: 8
+  # of culprit methods: 4
+  # of culprit methods: 2
+  # of culprit methods: 2
+  # of culprit methods: 1
+
+  Minimal methods found in 10 steps:
+
+  ruby -Ilib -e 'require "./example/test_bad1.rb" ; require "./example/test_bad4.rb"' -- --seed 3911 -s 48222 -n '/^(?:TestBad1\#test_bad1_1|TestBad4\#test_bad4_4)$/'
+
+  Final reproduction:
+
+  Run options: --seed 3911 -s 48222 -n "/^(?:TestBad1\\#test_bad1_1|TestBad4\\#test_bad4_4)$/"
+
+  # Running:
+
+  .F
+
+  Finished in 0.505776s, 3.9543 runs/s, 1.9772 assertions/s.
+
+    1) Failure:
+  TestBad4#test_bad4_4 [/Users/ryan/Work/p4/zss/src/minitest-bisect/dev/example/helper.rb:16]:
+  muahahaha order dependency bug!
+
+  2 runs, 1 assertions, 1 failures, 0 errors, 0 skips
+
+Voila! This reduced it from 800 tests across 8 files down to 2 tests
+across 2 files. Note how we went from a 200 second test run to a 0.5
+second test run. Debugging that will be much easier.
+
+It is now up to you to look at the source of both of those tests to
+determine what side-effects (or lack thereof) are causing your test
+failure when run in this specific order.
+
+This happens in a single run. Depending on how many files / tests you
+have and how long they take, the first phase might take a long time.
+The second phase might also take a while, but each iteration should
+reduce the number of tests so it should get quicker with each step.
+
+== REQUIREMENTS:
+
+* minitest 5
+
+== INSTALL:
+
+* sudo gem install minitest-bisect
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) Ryan Davis, seattle.rb
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,72 @@
+# -*- ruby -*-
+
+require "rubygems"
+require "hoe"
+
+Hoe.plugin :isolate
+Hoe.plugin :seattlerb
+Hoe.plugin :rdoc
+
+Hoe.spec "minitest-bisect" do
+  developer "Ryan Davis", "ryand-ruby@zenspider.com"
+  license "MIT"
+end
+
+require "rake/testtask"
+
+Rake::TestTask.new(:badtest) do |t|
+  t.test_files = Dir["badtest/test*.rb"]
+end
+
+def banner text
+  puts
+  puts "#" * 70
+  puts "# #{text} ::"
+  puts "#" * 70
+  puts
+  unless ENV["SLEEP"] == "0" then
+    print "Press return to continue "
+    $stdin.gets
+    puts
+  end
+end
+
+def run cmd
+  sh cmd do end
+end
+
+def req glob
+  Dir["#{glob}.rb"].map { |s| "require #{s.inspect}" }.join ";"
+end
+
+task :repro do
+  unless ENV.has_key? "SLEEP" then
+    warn "NOTE: Defaulting to sleeping 0.01 seconds per test."
+    warn "NOTE: Use SLEEP=0 to disable or any other value to simulate your tests."
+  end
+
+  ruby = "ruby -I.:lib"
+
+  banner "Original run that causes the test order dependency bug"
+  run "#{ruby} -e '#{req "example/test*"}' -- --seed 3911"
+
+  banner "Reduce the problem down to the minimal reproduction"
+  run "#{ruby} bin/minitest_bisect -Ilib --seed 3911 example/test*.rb"
+end
+
+task :many do
+  unless ENV.has_key? "SLEEP" then
+    warn "NOTE: Defaulting to sleeping 0.01 seconds per test."
+    warn "NOTE: Use SLEEP=0 to disable or any other value to simulate your tests."
+  end
+
+  ruby = "ruby -I.:lib"
+
+  banner "Original run that causes the test order dependency bug"
+  run "#{ruby} -e '#{req "example-many/test*"}' -- --seed 27083"
+
+  banner "Reduce the problem down to the minimal reproduction"
+  run "#{ruby} bin/minitest_bisect -Ilib --seed 27083 example-many/test*.rb"
+end
+
+# vim: syntax=ruby

data/bin/minitest_bisect

@@ -0,0 +1,5 @@
+#!/usr/bin/ruby -w
+
+require "minitest/bisect"
+
+Minitest::Bisect.run ARGV

data/example-many/helper.rb

@@ -0,0 +1,25 @@
+$hosed ||= 0
+
+def create_test suffix, n_methods, bad_methods = {}
+  raise ArgumentError, "Bad args" if Hash === n_methods
+
+  delay = (ENV["SLEEP"] || 0.01).to_f
+
+  Class.new(Minitest::Test) do
+    n_methods.times do |n|
+      n = n + 1
+      define_method "test_bad#{suffix}_#{n}" do
+        sleep delay if delay > 0
+
+        case bad_methods[n]
+        when true then
+          $hosed += 1
+        when Fixnum then
+          flunk "muahahaha order dependency bug!" if $hosed >= bad_methods[n]
+        else
+          assert true
+        end
+      end
+    end
+  end
+end

data/example-many/test_bad1.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad1 = create_test 1, 100, 1 => true

data/example-many/test_bad2.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad2 = create_test 2, 100

data/example-many/test_bad3.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad3 = create_test 3, 100, 72 => true

data/example-many/test_bad4.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad4 = create_test 4, 100

data/example-many/test_bad5.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad5 = create_test 5, 100, 17 => true

data/example-many/test_bad6.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad6 = create_test 6, 100

data/example-many/test_bad7.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad7 = create_test 7, 100

data/example-many/test_bad8.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad8 = create_test 8, 100, 43 => 3

data/example/helper.rb

@@ -0,0 +1,25 @@
+$hosed ||= false
+
+def create_test suffix, n_methods, bad_methods = {}
+  raise ArgumentError, "Bad args" if Hash === n_methods
+
+  delay = (ENV["SLEEP"] || 0.01).to_f
+
+  Class.new(Minitest::Test) do
+    n_methods.times do |n|
+      n = n + 1
+      define_method "test_bad#{suffix}_#{n}" do
+        sleep delay if delay > 0
+
+        case bad_methods[n]
+        when true then
+          flunk "muahahaha order dependency bug!" if $hosed
+        when false then
+          $hosed = true
+        else
+          assert true
+        end
+      end
+    end
+  end
+end

data/example/test_bad1.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad1 = create_test 1, 100, 1 => false

data/example/test_bad2.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad2 = create_test 2, 100

data/example/test_bad3.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad3 = create_test 3, 100

data/example/test_bad4.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad4 = create_test 4, 100, 4 => true

data/example/test_bad5.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad5 = create_test 5, 100

data/example/test_bad6.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad6 = create_test 6, 100

data/example/test_bad7.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad7 = create_test 7, 100

data/example/test_bad8.rb

@@ -0,0 +1,4 @@
+require "minitest/autorun"
+require_relative "helper"
+
+TestBad8 = create_test 8, 100

data/lib/minitest/bisect.rb

@@ -0,0 +1,150 @@
+require "minitest/find_minimal_combination"
+require "minitest/server"
+require "shellwords"
+
+class Minitest::Bisect
+  VERSION = "1.0.0"
+  SHH = " &> /dev/null"
+
+  attr_accessor :tainted, :failures, :culprits, :mode, :seen_bad
+  alias :tainted? :tainted
+
+  def self.run files
+    new.run files
+  end
+
+  def initialize
+    self.culprits = []
+    self.failures = Hash.new { |h,k| h[k] = Hash.new { |h2,k2| h2[k2] = [] } }
+  end
+
+  def reset
+    self.seen_bad = false
+    self.tainted  = false
+    failures.clear
+    # not clearing culprits on purpose
+  end
+
+  def run files
+    Minitest::Server.run self
+
+    cmd = nil
+
+    if :until_I_have_negative_filtering_in_minitest then
+      files, flags = files.partition { |arg| File.file? arg }
+      rb_flags, mt_flags = flags.partition { |arg| arg =~ /^-I/ }
+      mt_flags += ["-s", $$]
+
+      cmd = bisect_methods build_files_cmd(files, rb_flags, mt_flags)
+    else
+      cmd = bisect_methods bisect_files files
+    end
+
+    puts "Final reproduction:"
+    puts
+
+    system cmd
+  ensure
+    Minitest::Server.stop
+  end
+
+  def bisect_files files
+    self.mode = :files
+
+    files, flags = files.partition { |arg| File.file? arg }
+    rb_flags, mt_flags = flags.partition { |arg| arg =~ /^-I/ }
+    mt_flags += ["-s", $$]
+
+    puts "reproducing..."
+    system "#{build_files_cmd files, rb_flags, mt_flags} #{SHH}"
+    abort "Reproduction run passed? Aborting." unless tainted?
+    puts "reproduced"
+
+    found, count = files.find_minimal_combination_and_count do |test|
+      puts "# of culprit files: #{test.size}"
+
+      system "#{build_files_cmd test, rb_flags, mt_flags} #{SHH}"
+
+      self.tainted?
+    end
+
+    puts
+    puts "Minimal files found in #{count} steps:"
+    puts
+    cmd = build_files_cmd found, rb_flags, mt_flags
+    puts cmd
+    cmd
+  end
+
+  def bisect_methods cmd
+    self.mode = :methods
+
+    puts "reproducing..."
+    system "#{build_methods_cmd cmd} #{SHH}"
+    abort "Reproduction run passed? Aborting." unless tainted?
+    puts "reproduced"
+
+    # from: {"file.rb"=>{"Class"=>["test_method"]}} to: "Class#test_method"
+    bad = failures.values.first.to_a.join "#"
+
+    found, count = culprits.find_minimal_combination_and_count do |test|
+      puts "# of culprit methods: #{test.size}"
+
+      system "#{build_methods_cmd cmd, test, bad} #{SHH}"
+
+      self.tainted?
+    end
+
+    puts
+    puts "Minimal methods found in #{count} steps:"
+    puts
+    cmd = build_methods_cmd cmd, found, bad
+    puts cmd
+    puts
+    cmd
+  end
+
+  def build_files_cmd culprits, rb, mt
+    reset
+
+    tests = culprits.flatten.compact.map {|f| %(require "./#{f}")}.join " ; "
+
+    %(ruby #{rb.shelljoin} -e '#{tests}' -- #{mt.shelljoin})
+  end
+
+  def build_methods_cmd cmd, culprits = [], bad = nil
+    reset
+
+    if bad then
+      re = []
+
+      bbc = (culprits + [bad]).map { |s| s.split(/#/) }.group_by(&:first)
+      bbc.each do |klass, methods|
+        methods = methods.map(&:last).flatten
+
+        re << /#{klass}##{Regexp.union(methods)}/
+      end
+
+      re = Regexp.union(re).to_s.gsub(/-mix/, "")
+
+      cmd += " -n '/^#{re}$/'" if bad
+    end
+
+    cmd
+  end
+
+  def result file, klass, method, fails, assertions, time
+    if mode == :methods then
+      if fails.empty? then
+        culprits << "#{klass}##{method}" unless seen_bad # UGH
+      else
+        self.seen_bad = true
+      end
+    end
+
+    unless fails.empty?
+      self.tainted = true
+      self.failures[file][klass] << method
+    end
+  end
+end

data/lib/minitest/find_minimal_combination.rb

@@ -0,0 +1,106 @@
+#!/usr/bin/ruby -w
+
+module Minitest; end
+
+class ComboFinder
+  ##
+  # Find the minimal combination of a collection of items that satisfy +test+.
+  #
+  # If you think of the collection as a binary tree, this algorithm
+  # does a breadth first search of the combinations that satisfy
+  # +test+.
+  #--
+  #  level   collection
+  #
+  #  0          A
+  #  1      B       C
+  #  2    D   E   F   G
+  #  3   1 2 3 4 5 6 7 8
+  #
+  # This assumes that A has already been tested and you're now trying
+  # to reduce the match. Starting at level 1, test B & C separately.
+  # If either test positive, reduce the search space accordingly. If
+  # not, step down to level 2 and search w/ finer granularity (ie, DF,
+  # DG, EF--DE and FG were already tested as B & C). Repeat until a
+  # minimal combination is found.
+
+  def find_minimal_combination ary
+    level, n_combos = 1, 1
+    seen = {}
+
+    loop do
+      size = 2 ** (Math.log(ary.size) / Math.log(2)).round
+      divs = 2 ** level
+      done = divs >= size
+      divs = size if done
+
+      subsections = ary.each_slice(size/divs).to_a.combination(n_combos)
+
+      found = subsections.find { |a|
+        b = a.flatten
+
+        next if seen[b]
+
+        cache_result yield(b), b, seen
+      }
+
+      if found then
+        ary = found.flatten
+        break if done
+
+        seen.delete ary
+
+        level = n_combos = 1
+      else
+        if done then
+          n_combos += 1
+          break if n_combos > size
+        else
+          level += 1
+          n_combos = level
+        end
+      end
+    end
+
+    ary
+  end
+
+  def cache_result result, data, cache
+    cache[data] = true
+
+    return result if result
+
+    unless result or data.size > 128 then
+      max = data.size
+      subdiv = 2
+      until subdiv >= max do
+        data.each_slice(max / subdiv) do |sub_data|
+          cache[sub_data] = true
+        end
+        subdiv *= 2
+      end
+    end
+
+    result
+  end
+end
+
+class Array
+  ##
+  # Find the minimal combination of a collection of items that satisfy +test+.
+
+  def find_minimal_combination &test
+    ComboFinder.new.find_minimal_combination(self, &test)
+  end
+
+  def find_minimal_combination_and_count
+    count = 0
+
+    found = self.find_minimal_combination do |ary|
+      count += 1
+      yield ary
+    end
+
+    return found, count
+  end
+end

data/lib/minitest/server.rb

@@ -0,0 +1,45 @@
+require "drb"
+require "tmpdir"
+require "minitest"
+
+module Minitest
+  class Server
+    TOPDIR = Dir.pwd + "/"
+
+    def self.path pid = $$
+      "drbunix:#{Dir.tmpdir}/minitest.#{pid}"
+    end
+
+    def self.run client
+      DRb.start_service path, new(client)
+    end
+
+    def self.stop
+      DRb.stop_service
+    end
+
+    attr_accessor :client
+
+    def initialize client
+      self.client = client
+    end
+
+    def quit
+      self.class.stop
+    end
+
+    def start
+      client.failures.clear # TODO: push down to subclass
+    end
+
+    def result file, klass, method, fails, assertions, time
+      file = file.sub(/^#{TOPDIR}/, "")
+
+      client.result file, klass, method, fails, assertions, time
+    end
+
+    def report
+      # do nothing
+    end
+  end
+end

data/lib/minitest/server_plugin.rb

@@ -0,0 +1,44 @@
+require "minitest"
+
+module Minitest
+  @server = false
+
+  def self.plugin_server_options opts, options # :nodoc:
+    opts.on "-s", "--server=pid", Integer, "Connect to minitest server w/ pid." do |s|
+      @server = s
+    end
+  end
+
+  def self.plugin_server_init options
+    if @server then
+      require "minitest/server"
+      self.reporter << Minitest::ServerReporter.new(@server)
+    end
+  end
+end
+
+module Minitest
+  class ServerReporter < Minitest::AbstractReporter
+    def initialize pid
+      DRb.start_service
+      uri = Minitest::Server.path(pid)
+      @mt_server = DRbObject.new_with_uri uri
+      super()
+    end
+
+    def start
+      @mt_server.start
+    end
+
+    def record result
+      r = result
+      c = r.class
+      file, = c.instance_method(r.name).source_location
+      @mt_server.result file, c.name, r.name, r.failures, r.assertions, r.time
+    end
+
+    def report
+      @mt_server.report
+    end
+  end
+end

data/test/minitest/test_bisect.rb

@@ -0,0 +1,10 @@
+require "minitest/autorun"
+require "minitest/bisect"
+
+module TestMinitest; end
+
+class TestMinitest::TestBisect < Minitest::Test
+  def test_sanity
+    flunk "write tests or I will kneecap you"
+  end
+end

data/test/minitest/test_find_minimal_combination.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/ruby -w
+
+$: << "." << "lib"
+
+gem "minitest"
+require "minitest/autorun"
+require "minitest/find_minimal_combination"
+
+describe Array, :find_minimal_combination do
+  def check(*bad)
+    lambda { |sample| bad & sample == bad }
+  end
+
+  def assert_steps input, bad, exp
+    tests = []
+
+    found = input.find_minimal_combination do |test|
+      tests << test
+      bad & test == bad
+    end
+
+    assert_equal bad, found, "algorithm is bad"
+
+    assert_equal exp, tests
+  end
+
+  def test_ordering_best_case
+    a   = (0..15).to_a
+    bad = [0, 1]
+    exp = [[0, 1, 2, 3, 4, 5, 6, 7],
+           [0, 1, 2, 3],
+           [0, 1],
+           [0],
+           [1],
+           [0, 1]]
+
+    assert_steps a, bad, exp
+  end
+
+  def test_ordering
+    a     = (0..15).to_a
+    bad   = [0, 15]
+
+    # lvl              collection
+    #
+    #  0 |                 A
+    #  1 |         B               C
+    #  2 |     D       E       F       G
+    #  3 |   H   I   J   K   L   M   N   O
+    #  4 |  0 1 2 3 4 5 6 7 8 9 A B C D E F
+    #
+    #  0    +++++++++++++++++++++++++++++++
+    #  1    ---------------
+    #  1                    ---------------
+    #  2    -------         -------
+    #  2    +++++++                 +++++++
+    #  3    xxxxxxx
+    #  3                            xxxxxxx
+    #  3    ---                     ---
+    #  3    +++                         +++
+    #  4    xxx
+    #  4                                xxx
+    #  4    -                           -
+    #  4    +                             +
+    #
+    #  - = miss
+    #  + = hit
+    #  x = unwanted test
+
+    exp = [
+           # level 1 = B, C
+           [0, 1, 2, 3, 4, 5, 6, 7],
+           [8, 9, 10, 11, 12, 13, 14, 15],
+
+           # level 2 = DF, DG, EF, EG
+           [0, 1, 2, 3, 8, 9, 10, 11],
+           [0, 1, 2, 3, 12, 13, 14, 15],
+
+           # level 3
+           # [0, 1, 2, 3],     # I think this is bad, we've tested B
+           # [12, 13, 14, 15], # again, bad, we've tested C
+           [0, 1, 12, 13],
+           [0, 1, 14, 15],
+
+           # level 4
+           # [0, 1],
+           # [14, 15],
+           [0, 14],
+           [0, 15],
+          ]
+
+    assert_steps a, bad, exp
+  end
+
+  make_my_diffs_pretty!
+
+  def self.test_find_minimal_combination max, *bad
+    define_method "test_find_minimal_combination_#{max}_#{bad.join "_"}" do
+      a = (1..max).to_a
+      a.find_minimal_combination(&check(*bad)).must_equal bad
+    end
+  end
+
+  test_find_minimal_combination    8, 5
+  test_find_minimal_combination    8, 2, 7
+  test_find_minimal_combination    8, 1, 2, 7
+  test_find_minimal_combination    8, 1, 4, 7
+  test_find_minimal_combination    8, 1, 3, 5, 7
+
+  test_find_minimal_combination    9, 5
+  test_find_minimal_combination    9, 9
+  test_find_minimal_combination    9, 2, 7
+  test_find_minimal_combination    9, 1, 2, 7
+  test_find_minimal_combination    9, 1, 4, 7
+  test_find_minimal_combination    9, 1, 3, 5, 7
+
+  test_find_minimal_combination 1023, 5
+  test_find_minimal_combination 1023, 1005
+  test_find_minimal_combination 1023, 802, 907
+  test_find_minimal_combination 1023, 7, 15, 166, 1001
+  test_find_minimal_combination 1023, 1000, 1001, 1002
+  test_find_minimal_combination 1023, 1001, 1003, 1005, 1007
+
+  # test_find_minimal_combination 1024, 1001, 1003, 1005, 1007
+  # test_find_minimal_combination 1023, 1001, 1003, 1005, 1007
+end

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: minitest-bisect
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Ryan Davis
+autorequire: 
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIDPjCCAiagAwIBAgIBATANBgkqhkiG9w0BAQUFADBFMRMwEQYDVQQDDApyeWFu
+  ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
+  GRYDY29tMB4XDTEzMDkxNjIzMDQxMloXDTE0MDkxNjIzMDQxMlowRTETMBEGA1UE
+  AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
+  JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
+  b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
+  taCPaLmfYIaFcHHCSY4hYDJijRQkLxPeB3xbOfzfLoBDbjvx5JxgJxUjmGa7xhcT
+  oOvjtt5P8+GSK9zLzxQP0gVLS/D0FmoE44XuDr3iQkVS2ujU5zZL84mMNqNB1znh
+  GiadM9GHRaDiaxuX0cIUBj19T01mVE2iymf9I6bEsiayK/n6QujtyCbTWsAS9Rqt
+  qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
+  gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
+  HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBBQUAA4IB
+  AQCFZ7JTzoy1gcG4d8A6dmOJy7ygtO5MFpRIz8HuKCF5566nOvpy7aHhDDzFmQuu
+  FX3zDU6ghx5cQIueDhf2SGOncyBmmJRRYawm3wI0o1MeN6LZJ/3cRaOTjSFy6+S6
+  zqDmHBp8fVA2TGJtO0BLNkbGVrBJjh0UPmSoGzWlRhEVnYC33TpDAbNA+u39UrQI
+  ynwhNN7YbnmSR7+JU2cUjBFv2iPBO+TGuWC+9L2zn3NHjuc6tnmSYipA9y8Hv+As
+  Y4evBVezr3SjXz08vPqRO5YRdO3zfeMT8gBjRqZjWJGMZ2lD4XNfrs7eky74CyZw
+  xx3n58i0lQkBE1EpKE0lFu/y
+  -----END CERTIFICATE-----
+date: 2014-08-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.4'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+description: |-
+  Hunting down random test failures can be very very difficult,
+  sometimes impossible, but minitest-bisect makes it easy.
+
+  minitest-bisect helps you isolate and debug random test failures.
+
+  If your tests only fail randomly, you can reproduce the error
+  consistently by using `--seed <num>`, but what then? How do you figure
+  out which combination of tests out of hundreds are responsible for the
+  failure? You know which test is failing, but what others are causing
+  it to fail or were helping it succeed in a different order? That's
+  what minitest-bisect does best.
+email:
+- ryand-ruby@zenspider.com
+executables:
+- minitest_bisect
+extensions: []
+extra_rdoc_files:
+- History.rdoc
+- Manifest.txt
+- README.rdoc
+files:
+- .autotest
+- .gemtest
+- History.rdoc
+- Manifest.txt
+- README.rdoc
+- Rakefile
+- bin/minitest_bisect
+- example-many/helper.rb
+- example-many/test_bad1.rb
+- example-many/test_bad2.rb
+- example-many/test_bad3.rb
+- example-many/test_bad4.rb
+- example-many/test_bad5.rb
+- example-many/test_bad6.rb
+- example-many/test_bad7.rb
+- example-many/test_bad8.rb
+- example/helper.rb
+- example/test_bad1.rb
+- example/test_bad2.rb
+- example/test_bad3.rb
+- example/test_bad4.rb
+- example/test_bad5.rb
+- example/test_bad6.rb
+- example/test_bad7.rb
+- example/test_bad8.rb
+- lib/minitest/bisect.rb
+- lib/minitest/find_minimal_combination.rb
+- lib/minitest/server.rb
+- lib/minitest/server_plugin.rb
+- test/minitest/test_bisect.rb
+- test/minitest/test_find_minimal_combination.rb
+homepage: https://github.com/seattlerb/minitest-bisect
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options:
+- --main
+- README.rdoc
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: Hunting down random test failures can be very very difficult, sometimes impossible,
+  but minitest-bisect makes it easy
+test_files:
+- test/minitest/test_bisect.rb
+- test/minitest/test_find_minimal_combination.rb

metadata.gz.sig

@@ -0,0 +1,2 @@
+�O��
+�����26I����>W3�����a1\�kI@�{4�:�,"M�����{�]ò��W9dgG�pKM��`��J���$�4����C��8`��S���i���!�0�Df�uܨ��.�����ً[������26���4'rXV��qL#ݟ��:�]�ӈ���\=g<�*�7�l�@�Zg�#�\oR�Qs�e+KG�w�~0%��<�i��ˬ0{*C��(��1�I�<��OֶF���b@���u���