opal-rspec 0.8.0.alpha3 → 1.0.0.alpha1

data/rspec-core/upstream/features/step_definitions/additional_cli_steps.rb

@@ -1,20 +1,22 @@
 require 'rspec/core'  # to fix annoying "undefined method `configuration' for RSpec:Module (NoMethodError)"
 
+require './spec/support/formatter_support'
+
 Then /^the output should contain all of these:$/ do |table|
   table.raw.flatten.each do |string|
-    assert_partial_output(string, all_output)
+    expect(all_output).to include(string)
   end
 end
 
 Then /^the output should not contain any of these:$/ do |table|
   table.raw.flatten.each do |string|
-    expect(all_output).not_to match(regexp(string))
+    expect(all_output).not_to include(string)
   end
 end
 
 Then /^the output should contain one of the following:$/ do |table|
   matching_output = table.raw.flatten.select do |string|
-    all_output =~ regexp(string)
+    all_output.include?(string)
   end
 
   expect(matching_output.count).to eq(1)
@@ -66,7 +68,7 @@ Then /^the backtrace\-normalized output should contain:$/ do |partial_output|
     line =~ /(^\s+# [^:]+:\d+)/ ? $1 : line # http://rubular.com/r/zDD7DdWyzF
   end.join("\n")
 
-  expect(normalized_output).to match(regexp(partial_output))
+  expect(normalized_output).to include(partial_output)
 end
 
 Then /^the output should not contain any error backtraces$/ do
@@ -74,10 +76,16 @@ Then /^the output should not contain any error backtraces$/ do
 end
 
 # This step can be generalized if it's ever used to test other colors
-Then /^the failing example is printed in magenta$/ do
+Then /^the failing example is printed (?:wrapped )?in (.*)$/ do |color|
+  code =
+    case color
+    when "magenta" then "\e[35m"
+    when /"(.*)"/ then "\e[#{$1}m"
+    end
+
   # \e[35m = enable magenta
   # \e[0m  = reset colors
-  expect(all_output).to include("\e[35m" + "F" + "\e[0m")
+  expect(all_output).to include(code + "F" + "\e[0m")
 end
 
 Then /^the output from `([^`]+)` should contain "(.*?)"$/  do |cmd, expected_output|
@@ -91,24 +99,23 @@ Then /^the output from `([^`]+)` should not contain "(.*?)"$/  do |cmd, expected
 end
 
 Given /^I have a brand new project with no files$/ do
-  in_current_dir do
+  cd('.') do
     expect(Dir["**/*"]).to eq([])
   end
 end
 
 Given /^I have run `([^`]*)`$/ do |cmd|
-  fail_on_error = true
-  run_simple(unescape(cmd), fail_on_error)
+  run_command_and_stop(sanitize_text(cmd), :fail_on_error => true)
 end
 
 Given(/^a vendored gem named "(.*?)" containing a file named "(.*?)" with:$/) do |gem_name, file_name, file_contents|
   gem_dir = "vendor/#{gem_name}-1.2.3"
   step %Q{a file named "#{gem_dir}/#{file_name}" with:}, file_contents
-  set_env('RUBYOPT', ENV['RUBYOPT'] + " -I#{gem_dir}/lib")
+  set_environment_variable('RUBYOPT', ENV['RUBYOPT'] + " -I#{gem_dir}/lib")
 end
 
-When "I accept the recommended settings by removing `=begin` and `=end` from `spec/spec_helper.rb`" do
-  in_current_dir do
+When('I accept the recommended settings by removing `=begin` and `=end` from `spec_helper.rb`') do
+  cd('.') do
     spec_helper = File.read("spec/spec_helper.rb")
     expect(spec_helper).to include("=begin", "=end")
 
@@ -124,3 +131,126 @@ end
 When /^I create "([^"]*)" with the following content:$/ do |file_name, content|
   write_file(file_name, content)
 end
+
+Given(/^I have run `([^`]*)` once, resulting in "([^"]*)"$/) do |command, output_snippet|
+  step %Q{I run `#{command}`}
+  step %Q{the output from "#{command}" should contain "#{output_snippet}"}
+end
+
+When(/^I fix "(.*?)" by replacing "(.*?)" with "(.*?)"$/) do |file_name, original, replacement|
+  cd('.') do
+    contents = File.read(file_name)
+    expect(contents).to include(original)
+    fixed = contents.sub(original, replacement)
+    File.open(file_name, "w") { |f| f.write(fixed) }
+  end
+end
+
+Given(/^I have not configured `example_status_persistence_file_path`$/) do
+  cd('.') do
+    return unless File.exist?("spec/spec_helper.rb")
+    return unless File.read("spec/spec_helper.rb").include?("example_status_persistence_file_path")
+    File.open("spec/spec_helper.rb", "w") { |f| f.write("") }
+  end
+end
+
+Given(/^files "(.*?)" through "(.*?)" with an unrelated passing spec in each file$/) do |file1, file2|
+  index_1 = Integer(file1[/\d+/])
+  index_2 = Integer(file2[/\d+/])
+  pattern = file1.sub(/\d+/, '%s')
+
+  index_1.upto(index_2) do |index|
+    write_file(pattern % index, <<-EOS)
+      RSpec.describe "Spec file #{index}" do
+        example { }
+      end
+    EOS
+  end
+end
+
+Then(/^bisect should (succeed|fail) with output like:$/) do |succeed, expected_output|
+  last_process = all_commands.last
+  expected_status = succeed == "succeed" ? 0 : 1
+  expect(last_process.exit_status).to eq(expected_status),
+    "Expected exit status of #{expected_status} but got #{last_process.exit_status} \n\n" \
+    "Output:\n\n#{last_process.stdout}"
+
+  expected = normalize_durations(expected_output)
+  actual   = normalize_durations(last_process.stdout).sub(/\n+\Z/, '')
+
+  if !RSpec::Support::RubyFeatures.fork_supported?
+    expected.gsub!('runner: :fork', 'runner: :shell')
+  end
+
+  if expected.include?("# ...")
+    expected_start, expected_end = expected.split("# ...")
+    expect(actual).to start_with(expected_start).and end_with(expected_end)
+  else
+    expect(actual).to eq(expected)
+  end
+end
+
+When(/^I run `([^`]+)` and abort in the middle with ctrl\-c$/) do |cmd|
+  set_environment_variable('RUBYOPT', ENV['RUBYOPT'] + " -r#{File.expand_path("../../support/send_sigint_during_bisect.rb", __FILE__)}")
+  step "I run `#{cmd}`"
+end
+
+Then(/^it should fail and list all the failures:$/) do |string|
+  step %q{the exit status should not be 0}
+  expect(normalize_failure_output(all_output)).to include(normalize_failure_output(string))
+end
+
+Then(/^it should pass and list all the pending examples:$/) do |string|
+  step %q{the exit status should be 0}
+  expect(normalize_failure_output(all_output)).to include(normalize_failure_output(string))
+end
+
+Then(/^the output should report "slow before context hook" as the slowest example group$/) do
+  # These expectations are trying to guard against a regression that introduced
+  # this output:
+  #   Top 1 slowest example groups:
+  #     slow before context hook
+  #       Inf seconds average (0.00221 seconds / 0 examples) RSpec::ExampleGroups::SlowBeforeContextHook::Nested
+  #
+  # Problems:
+  # - "Inf seconds"
+  # - 0 examples
+  # - "Nested" group listed (it should be the outer group)
+  # - The example group class name is listed (it should be the location)
+
+  output = all_output
+
+  expect(output).not_to match(/nested/i)
+  expect(output).not_to match(/inf/i)
+  expect(output).not_to match(/\b0 examples/i)
+
+  seconds = '\d+(?:\.\d+)? seconds'
+
+  expect(output).to match(
+    %r{Top 1 slowest example groups?:\n\s+slow before context hook\n\s+#{seconds} average \(#{seconds} / 1 example\) \./spec/example_spec\.rb:1}
+  )
+end
+
+Given(/^I have changed `([^`]+)` to `([^`]+)` in "(.*?)"$/) do |old_code, new_code, file_name|
+  cd('.') do
+    file_content = File.read(file_name)
+    expect(file_content).to include(old_code)
+    new_file_content = file_content.sub(old_code, new_code)
+    File.open(file_name, "w") { |f| f.write(new_file_content) }
+  end
+end
+
+module Normalization
+  def normalize_failure_output(text)
+    whitespace_normalized = text.lines.map { |line| line.sub(/\s+$/, '').sub(/:in .*$/, '') }.join
+
+    # 1.8.7 and JRuby produce slightly different output for `Hash#fetch` errors, so we
+    # convert it to the same output here to match our expectation.
+    whitespace_normalized.
+      sub("IndexError", "KeyError").
+      sub(/key not found.*$/, "key not found")
+  end
+end
+
+World(Normalization)
+World(FormatterSupport)

data/rspec-core/upstream/features/step_definitions/core_standalone_steps.rb

@@ -2,11 +2,15 @@ Given(/^only rspec-core is installed$/) do
   if RUBY_VERSION.to_f >= 1.9 # --disable-gems is invalid on 1.8.7
     # Ensure the gem versions of rspec-mocks and rspec-expectations
     # won't be loaded if available on the developers machine.
-    set_env('RUBYOPT', ENV['RUBYOPT'] + ' --disable-gems')
+    set_environment_variable('RUBYOPT', ENV['RUBYOPT'] + ' --disable-gems')
   end
 
   # This will make `require_expect_syntax_in_aruba_specs.rb` (loaded
   # automatically when the specs run) remove rspec-mocks and
   # rspec-expectations from the load path.
-  set_env('REMOVE_OTHER_RSPEC_LIBS_FROM_LOAD_PATH', 'true')
+  set_environment_variable('REMOVE_OTHER_RSPEC_LIBS_FROM_LOAD_PATH', 'true')
+end
+
+Given(/^rspec-expectations is not installed$/) do
+  step "only rspec-core is installed"
 end

data/rspec-core/upstream/features/subject/implicit_subject.feature

@@ -1,7 +1,8 @@
 Feature: implicitly defined subject
 
-  If the first argument to the outermost example group is a class, an instance
-  of that class is exposed to each example via the `subject` method.
+  If the first argument to an example group is a class, an instance of that
+  class is exposed to each example in that example group via the `subject`
+  method.
 
   While the examples below demonstrate how `subject` can be used as a
   user-facing concept, we recommend that you reserve it for support of custom

data/rspec-core/upstream/features/subject/one_liner_syntax.feature

@@ -18,7 +18,11 @@ Feature: One-liner syntax
       `:should` syntax is disabled (since that merely removes `Object#should`
       but this is `RSpec::Core::ExampleGroup#should`).
 
-  Note: this feature is only available when using rspec-expectations.
+  Notes:
+
+    * This feature is only available when using rspec-expectations.
+    * Examples defined using this one-liner syntax cannot be directly selected from the command line using the [`--example` option](../command-line/example-option).
+    * The one-liner syntax only works with non-block expectations (e.g. `expect(obj).to eq`, etc) and it cannot be used with block expectations (e.g. `expect { object }`).
 
   Scenario: Implicit subject
     Given a file named "example_spec.rb" with:
@@ -42,8 +46,8 @@ Feature: One-liner syntax
        """
        Array
          when first created
-           should be empty
-           should be empty
+           is expected to be empty
+           is expected to be empty
        """
 
   Scenario: Explicit subject
@@ -64,6 +68,6 @@ Feature: One-liner syntax
        """
        Array
          with 3 items
-           should not be empty
-           should not be empty
+           is expected not to be empty
+           is expected not to be empty
        """

data/rspec-core/upstream/features/support/diff_lcs_versions.rb

@@ -0,0 +1,17 @@
+require 'diff-lcs'
+
+Around "@skip-when-diff-lcs-1.4" do |scenario, block|
+  if Diff::LCS::VERSION >= '1.4'
+    skip_this_scenario "Skipping scenario #{scenario.name} on `diff-lcs` v#{Diff::LCS::VERSION}"
+  else
+    block.call
+  end
+end
+
+Around "@skip-when-diff-lcs-1.3" do |scenario, block|
+  if Diff::LCS::VERSION < '1.4'
+    skip_this_scenario "Skipping scenario #{scenario.name} on `diff-lcs` v#{Diff::LCS::VERSION}"
+  else
+    block.call
+  end
+end

data/rspec-core/upstream/features/support/env.rb

@@ -1,21 +1,34 @@
 require 'aruba/cucumber'
 
 Before do
-  if RUBY_PLATFORM =~ /java/ || defined?(Rubinius)
-    @aruba_timeout_seconds = 60
+  # Force ids to be printed unquoted for consistency
+  set_environment_variable('SHELL', '/usr/bin/bash')
+end
+
+Aruba.configure do |config|
+  config.exit_timeout = if RUBY_PLATFORM =~ /java/ || defined?(Rubinius) || (defined?(RUBY_ENGINE) && RUBY_ENGINE == 'truffleruby')
+    120
   else
-    @aruba_timeout_seconds = 10
+    10
   end
 end
 
 Aruba.configure do |config|
-  config.before_cmd do |cmd|
-    set_env('JRUBY_OPTS', "-X-C #{ENV['JRUBY_OPTS']}") # disable JIT since these processes are so short lived
+  config.before(:command) do |cmd|
+    set_environment_variable('JRUBY_OPTS', "-X-C #{ENV['JRUBY_OPTS']}") # disable JIT since these processes are so short lived
   end
 end if RUBY_PLATFORM == 'java'
 
 Aruba.configure do |config|
-  config.before_cmd do |cmd|
-    set_env('RBXOPT', "-Xint=true #{ENV['RBXOPT']}") # disable JIT since these processes are so short lived
+  config.before(:command) do |cmd|
+    set_environment_variable('RBXOPT', "-Xint=true #{ENV['RBXOPT']}") # disable JIT since these processes are so short lived
   end
 end if defined?(Rubinius)
+
+module ArubaHelpers
+  def all_output
+    all_commands.map { |c| c.output }.join("\n")
+  end
+end
+
+World(ArubaHelpers)

data/rspec-core/upstream/features/support/jruby.rb

@@ -0,0 +1,8 @@
+Around "@broken-on-jruby-9000" do |scenario, block|
+  require 'rspec/support/ruby_features'
+  if RSpec::Support::Ruby.jruby_9000?
+    skip_this_scenario "Skipping scenario #{scenario.name} not supported on JRuby 9000"
+  else
+    block.call
+  end
+end

data/rspec-core/upstream/features/support/require_expect_syntax_in_aruba_specs.rb

@@ -1,11 +1,14 @@
 if defined?(Cucumber)
   require 'shellwords'
-  Before('~@allow-should-syntax') do
-    set_env('SPEC_OPTS', "-r#{Shellwords.escape(__FILE__)}")
+  use_tilde_tags = !defined?(::RUBY_ENGINE_VERSION) || (::RUBY_ENGINE_VERSION < '2.0.0')
+  exclude_allow_should_syntax = use_tilde_tags ? '~@allow-should-syntax' : 'not @allow-should-syntax'
+  exclude_with_clean_spec_ops = use_tilde_tags ? '~@with-clean-spec-opts' : 'not @with-clean-spec-opts'
+  Before(exclude_allow_should_syntax, exclude_with_clean_spec_ops) do
+    set_environment_variable('SPEC_OPTS', "-r#{Shellwords.escape(__FILE__)}")
   end
 
   Before('@oneliner-should') do
-    set_env('ALLOW_ONELINER_SHOULD', 'true')
+    set_environment_variable('ALLOW_ONELINER_SHOULD', 'true')
   end
 else
   if ENV['REMOVE_OTHER_RSPEC_LIBS_FROM_LOAD_PATH']

data/rspec-core/upstream/features/support/rubinius.rb

@@ -2,5 +2,9 @@
 ENV['RBXOPT'] = "#{ENV["RBXOPT"]} -Xcompiler.no_rbc"
 
 Around "@unsupported-on-rbx" do |scenario, block|
-  block.call unless defined?(Rubinius)
+  if defined?(Rubinius)
+    block.call
+  else
+    skip_this_scenario "Skipping scenario #{scenario.name} not supported on Rubinius"
+  end
 end

data/rspec-core/upstream/features/support/ruby_27_support.rb

@@ -0,0 +1,7 @@
+Around "@ruby-2-7" do |scenario, block|
+  if RUBY_VERSION.to_f == 2.7
+    block.call
+  else
+    skip_this_scenario "Skipping scenario #{scenario.name} on Ruby v#{RUBY_VERSION}"
+  end
+end

data/rspec-core/upstream/features/support/send_sigint_during_bisect.rb

@@ -0,0 +1,21 @@
+require 'rspec/core'
+RSpec::Support.require_rspec_core "formatters/bisect_progress_formatter"
+
+module RSpec::Core::Formatters
+  BisectProgressFormatter = Class.new(remove_const :BisectProgressFormatter) do
+    RSpec::Core::Formatters.register self
+
+    def bisect_round_started(notification)
+      return super unless @round_count == 3
+
+      Process.kill("INT", Process.pid)
+      # Process.kill is not a synchronous call, so to ensure the output
+      # below aborts at a deterministic place, we need to block here.
+      # The sleep will be interrupted by the signal once the OS sends it.
+      # For the most part, this is only needed on JRuby, but we saw
+      # the asynchronous behavior on an MRI 2.0 travis build as well.
+      sleep 5
+    end
+  end
+end
+

data/rspec-core/upstream/lib/rspec/autorun.rb

@@ -1,2 +1,3 @@
 require 'rspec/core'
+# Ensure the default config is loaded
 RSpec::Core::Runner.autorun

data/rspec-core/upstream/lib/rspec/core/backtrace_formatter.rb

@@ -8,7 +8,7 @@ module RSpec
       def initialize
         @full_backtrace = false
 
-        patterns = %w[ /lib\d*/ruby/ bin/ exe/rspec ]
+        patterns = %w[ /lib\d*/ruby/ bin/ exe/rspec /lib/bundler/ /exe/bundle: ]
         patterns << "org/jruby/" if RUBY_PLATFORM == 'java'
         patterns.map! { |s| Regexp.new(s.gsub("/", File::SEPARATOR)) }
 
@@ -31,7 +31,8 @@ module RSpec
       end
 
       def format_backtrace(backtrace, options={})
-        return backtrace if options[:full_backtrace]
+        return [] unless backtrace
+        return backtrace if options[:full_backtrace] || backtrace.empty?
 
         backtrace.map { |l| backtrace_line(l) }.compact.
           tap do |filtered|
@@ -47,8 +48,6 @@ module RSpec
 
       def backtrace_line(line)
         Metadata.relative_path(line) unless exclude?(line)
-      rescue SecurityError
-        nil
       end
 
       def exclude?(line)

data/rspec-core/upstream/lib/rspec/core/bisect/coordinator.rb

@@ -0,0 +1,62 @@
+RSpec::Support.require_rspec_core "bisect/shell_command"
+RSpec::Support.require_rspec_core "bisect/example_minimizer"
+RSpec::Support.require_rspec_core "bisect/utilities"
+RSpec::Support.require_rspec_core "formatters/bisect_progress_formatter"
+
+module RSpec
+  module Core
+    module Bisect
+      # The main entry point into the bisect logic. Coordinates among:
+      #   - Bisect::ShellCommand: Generates shell commands to run spec subsets
+      #   - Bisect::ExampleMinimizer: Contains the core bisect logic.
+      #   - A bisect runner: runs a set of examples and returns the results.
+      #   - A bisect formatter: provides progress updates to the user.
+      # @private
+      class Coordinator
+        def self.bisect_with(spec_runner, original_cli_args, formatter)
+          new(spec_runner, original_cli_args, formatter).bisect
+        end
+
+        def initialize(spec_runner, original_cli_args, formatter)
+          @spec_runner   = spec_runner
+          @shell_command = ShellCommand.new(original_cli_args)
+          @notifier      = Bisect::Notifier.new(formatter)
+        end
+
+        def bisect
+          repro = start_bisect_runner do |runner|
+            minimizer = ExampleMinimizer.new(@shell_command, runner, @notifier)
+
+            gracefully_abort_on_sigint(minimizer)
+            minimizer.find_minimal_repro
+            minimizer.repro_command_for_currently_needed_ids
+          end
+
+          @notifier.publish(:bisect_repro_command, :repro => repro)
+
+          true
+        rescue BisectFailedError => e
+          @notifier.publish(:bisect_failed, :failure_explanation => e.message)
+          false
+        ensure
+          @notifier.publish(:close)
+        end
+
+      private
+
+        def start_bisect_runner(&block)
+          klass = @spec_runner.configuration.bisect_runner_class
+          klass.start(@shell_command, @spec_runner, &block)
+        end
+
+        def gracefully_abort_on_sigint(minimizer)
+          trap('INT') do
+            repro = minimizer.repro_command_for_currently_needed_ids
+            @notifier.publish(:bisect_aborted, :repro => repro)
+            exit(1)
+          end
+        end
+      end
+    end
+  end
+end

data/rspec-core/upstream/lib/rspec/core/bisect/example_minimizer.rb

@@ -0,0 +1,173 @@
+RSpec::Support.require_rspec_core "bisect/utilities"
+
+module RSpec
+  module Core
+    module Bisect
+      # @private
+      # Contains the core bisect logic. Searches for examples we can ignore by
+      # repeatedly running different subsets of the suite.
+      class ExampleMinimizer
+        attr_reader :shell_command, :runner, :all_example_ids, :failed_example_ids
+        attr_accessor :remaining_ids
+
+        def initialize(shell_command, runner, notifier)
+          @shell_command = shell_command
+          @runner        = runner
+          @notifier      = notifier
+        end
+
+        def find_minimal_repro
+          prep
+
+          _, duration = track_duration do
+            bisect(non_failing_example_ids)
+          end
+
+          notify(:bisect_complete, :duration => duration,
+                                   :original_non_failing_count => non_failing_example_ids.size,
+                                   :remaining_count => remaining_ids.size)
+
+          remaining_ids + failed_example_ids
+        end
+
+        def bisect(candidate_ids)
+          notify(:bisect_dependency_check_started)
+          if get_expected_failures_for?([])
+            notify(:bisect_dependency_check_failed)
+            self.remaining_ids = []
+            return
+          end
+          notify(:bisect_dependency_check_passed)
+
+          bisect_over(candidate_ids)
+        end
+
+        def bisect_over(candidate_ids)
+          return if candidate_ids.one?
+
+          notify(
+            :bisect_round_started,
+            :candidate_range => example_range(candidate_ids),
+            :candidates_count => candidate_ids.size
+          )
+
+          slice_size = (candidate_ids.length / 2.0).ceil
+          lhs, rhs = candidate_ids.each_slice(slice_size).to_a
+
+          ids_to_ignore, duration = track_duration do
+            [lhs, rhs].find do |ids|
+              get_expected_failures_for?(remaining_ids - ids)
+            end
+          end
+
+          if ids_to_ignore
+            self.remaining_ids -= ids_to_ignore
+            notify(
+              :bisect_round_ignoring_ids,
+              :ids_to_ignore => ids_to_ignore,
+              :ignore_range => example_range(ids_to_ignore),
+              :remaining_ids => remaining_ids,
+              :duration => duration
+            )
+            bisect_over(candidate_ids - ids_to_ignore)
+          else
+            notify(
+              :bisect_round_detected_multiple_culprits,
+              :duration => duration
+            )
+            bisect_over(lhs)
+            bisect_over(rhs)
+          end
+        end
+
+        def currently_needed_ids
+          remaining_ids + failed_example_ids
+        end
+
+        def repro_command_for_currently_needed_ids
+          return shell_command.repro_command_from(currently_needed_ids) if remaining_ids
+          "(Not yet enough information to provide any repro command)"
+        end
+
+        # @private
+        # Convenience class for describing a subset of the candidate examples
+        ExampleRange = Struct.new(:start, :finish) do
+          def description
+            if start == finish
+              "example #{start}"
+            else
+              "examples #{start}-#{finish}"
+            end
+          end
+        end
+
+      private
+
+        def example_range(ids)
+          ExampleRange.new(
+            non_failing_example_ids.find_index(ids.first) + 1,
+            non_failing_example_ids.find_index(ids.last) + 1
+          )
+        end
+
+        def prep
+          notify(:bisect_starting, :original_cli_args => shell_command.original_cli_args,
+                                   :bisect_runner => runner.class.name)
+
+          _, duration = track_duration do
+            original_results    = runner.original_results
+            @all_example_ids    = original_results.all_example_ids
+            @failed_example_ids = original_results.failed_example_ids
+            @remaining_ids      = non_failing_example_ids
+          end
+
+          if @failed_example_ids.empty?
+            raise BisectFailedError, "\n\nNo failures found. Bisect only works " \
+                  "in the presence of one or more failing examples."
+          else
+            notify(:bisect_original_run_complete, :failed_example_ids => failed_example_ids,
+                                                  :non_failing_example_ids => non_failing_example_ids,
+                                                  :duration => duration)
+          end
+        end
+
+        def non_failing_example_ids
+          @non_failing_example_ids ||= all_example_ids - failed_example_ids
+        end
+
+        def get_expected_failures_for?(ids)
+          ids_to_run = ids + failed_example_ids
+          notify(
+            :bisect_individual_run_start,
+            :command => shell_command.repro_command_from(ids_to_run),
+            :ids_to_run => ids_to_run
+          )
+
+          results, duration = track_duration { runner.run(ids_to_run) }
+          notify(:bisect_individual_run_complete, :duration => duration, :results => results)
+
+          abort_if_ordering_inconsistent(results)
+          (failed_example_ids & results.failed_example_ids) == failed_example_ids
+        end
+
+        def track_duration
+          start = ::RSpec::Core::Time.now
+          [yield, ::RSpec::Core::Time.now - start]
+        end
+
+        def abort_if_ordering_inconsistent(results)
+          expected_order = all_example_ids & results.all_example_ids
+          return if expected_order == results.all_example_ids
+
+          raise BisectFailedError, "\n\nThe example ordering is inconsistent. " \
+                "`--bisect` relies upon consistent ordering (e.g. by passing " \
+                "`--seed` if you're using random ordering) to work properly."
+        end
+
+        def notify(*args)
+          @notifier.publish(*args)
+        end
+      end
+    end
+  end
+end