rspec-core 2.11.1 → 3.11.0

data/lib/rspec/core/bisect/example_minimizer.rb

@@ -0,0 +1,173 @@
+RSpec::Support.require_rspec_core "bisect/utilities"
+
+module RSpec
+  module Core
+    module Bisect
+      # @private
+      # Contains the core bisect logic. Searches for examples we can ignore by
+      # repeatedly running different subsets of the suite.
+      class ExampleMinimizer
+        attr_reader :shell_command, :runner, :all_example_ids, :failed_example_ids
+        attr_accessor :remaining_ids
+
+        def initialize(shell_command, runner, notifier)
+          @shell_command = shell_command
+          @runner        = runner
+          @notifier      = notifier
+        end
+
+        def find_minimal_repro
+          prep
+
+          _, duration = track_duration do
+            bisect(non_failing_example_ids)
+          end
+
+          notify(:bisect_complete, :duration => duration,
+                                   :original_non_failing_count => non_failing_example_ids.size,
+                                   :remaining_count => remaining_ids.size)
+
+          remaining_ids + failed_example_ids
+        end
+
+        def bisect(candidate_ids)
+          notify(:bisect_dependency_check_started)
+          if get_expected_failures_for?([])
+            notify(:bisect_dependency_check_failed)
+            self.remaining_ids = []
+            return
+          end
+          notify(:bisect_dependency_check_passed)
+
+          bisect_over(candidate_ids)
+        end
+
+        def bisect_over(candidate_ids)
+          return if candidate_ids.one?
+
+          notify(
+            :bisect_round_started,
+            :candidate_range => example_range(candidate_ids),
+            :candidates_count => candidate_ids.size
+          )
+
+          slice_size = (candidate_ids.length / 2.0).ceil
+          lhs, rhs = candidate_ids.each_slice(slice_size).to_a
+
+          ids_to_ignore, duration = track_duration do
+            [lhs, rhs].find do |ids|
+              get_expected_failures_for?(remaining_ids - ids)
+            end
+          end
+
+          if ids_to_ignore
+            self.remaining_ids -= ids_to_ignore
+            notify(
+              :bisect_round_ignoring_ids,
+              :ids_to_ignore => ids_to_ignore,
+              :ignore_range => example_range(ids_to_ignore),
+              :remaining_ids => remaining_ids,
+              :duration => duration
+            )
+            bisect_over(candidate_ids - ids_to_ignore)
+          else
+            notify(
+              :bisect_round_detected_multiple_culprits,
+              :duration => duration
+            )
+            bisect_over(lhs)
+            bisect_over(rhs)
+          end
+        end
+
+        def currently_needed_ids
+          remaining_ids + failed_example_ids
+        end
+
+        def repro_command_for_currently_needed_ids
+          return shell_command.repro_command_from(currently_needed_ids) if remaining_ids
+          "(Not yet enough information to provide any repro command)"
+        end
+
+        # @private
+        # Convenience class for describing a subset of the candidate examples
+        ExampleRange = Struct.new(:start, :finish) do
+          def description
+            if start == finish
+              "example #{start}"
+            else
+              "examples #{start}-#{finish}"
+            end
+          end
+        end
+
+      private
+
+        def example_range(ids)
+          ExampleRange.new(
+            non_failing_example_ids.find_index(ids.first) + 1,
+            non_failing_example_ids.find_index(ids.last) + 1
+          )
+        end
+
+        def prep
+          notify(:bisect_starting, :original_cli_args => shell_command.original_cli_args,
+                                   :bisect_runner => runner.class.name)
+
+          _, duration = track_duration do
+            original_results    = runner.original_results
+            @all_example_ids    = original_results.all_example_ids
+            @failed_example_ids = original_results.failed_example_ids
+            @remaining_ids      = non_failing_example_ids
+          end
+
+          if @failed_example_ids.empty?
+            raise BisectFailedError, "\n\nNo failures found. Bisect only works " \
+                  "in the presence of one or more failing examples."
+          else
+            notify(:bisect_original_run_complete, :failed_example_ids => failed_example_ids,
+                                                  :non_failing_example_ids => non_failing_example_ids,
+                                                  :duration => duration)
+          end
+        end
+
+        def non_failing_example_ids
+          @non_failing_example_ids ||= all_example_ids - failed_example_ids
+        end
+
+        def get_expected_failures_for?(ids)
+          ids_to_run = ids + failed_example_ids
+          notify(
+            :bisect_individual_run_start,
+            :command => shell_command.repro_command_from(ids_to_run),
+            :ids_to_run => ids_to_run
+          )
+
+          results, duration = track_duration { runner.run(ids_to_run) }
+          notify(:bisect_individual_run_complete, :duration => duration, :results => results)
+
+          abort_if_ordering_inconsistent(results)
+          (failed_example_ids & results.failed_example_ids) == failed_example_ids
+        end
+
+        def track_duration
+          start = ::RSpec::Core::Time.now
+          [yield, ::RSpec::Core::Time.now - start]
+        end
+
+        def abort_if_ordering_inconsistent(results)
+          expected_order = all_example_ids & results.all_example_ids
+          return if expected_order == results.all_example_ids
+
+          raise BisectFailedError, "\n\nThe example ordering is inconsistent. " \
+                "`--bisect` relies upon consistent ordering (e.g. by passing " \
+                "`--seed` if you're using random ordering) to work properly."
+        end
+
+        def notify(*args)
+          @notifier.publish(*args)
+        end
+      end
+    end
+  end
+end

data/lib/rspec/core/bisect/fork_runner.rb

@@ -0,0 +1,138 @@
+require 'stringio'
+RSpec::Support.require_rspec_core "formatters/base_bisect_formatter"
+RSpec::Support.require_rspec_core "bisect/utilities"
+
+module RSpec
+  module Core
+    module Bisect
+      # A Bisect runner that runs requested subsets of the suite by forking
+      # sub-processes. The main process bootstraps RSpec and the application
+      # environment (including preloading files specified via `--require`) so
+      # that the individual spec runs do not have to re-pay that cost.  Each
+      # spec run happens in a forked process, ensuring that the spec files are
+      # not loaded in the main process.
+      #
+      # For most projects, bisections that use `ForkRunner` instead of
+      # `ShellRunner` will finish significantly faster, because the `ShellRunner`
+      # pays the cost of booting RSpec and the app environment on _every_ run of
+      # a subset. In contrast, `ForkRunner` pays that cost only once.
+      #
+      # However, not all projects can use `ForkRunner`. Obviously, on platforms
+      # that do not support forking (e.g. Windows), it cannot be used. In addition,
+      # it can cause problems for some projects that put side-effectful spec
+      # bootstrapping logic that should run on every spec run directly at the top
+      # level in a file loaded by `--require`, rather than in a `before(:suite)`
+      # hook. For example, consider a project that relies on some top-level logic
+      # in `spec_helper` to boot a Redis server for the test suite, intending the
+      # Redis bootstrapping to happen on every spec run. With `ShellRunner`, the
+      # bootstrapping logic will happen for each run of any subset of the suite,
+      # but for `ForkRunner`, such logic will only get run once, when the
+      # `RunDispatcher` boots the application environment. This might cause
+      # problems. The solution is for users to move the bootstrapping logic into
+      # a `before(:suite)` hook, or use the slower `ShellRunner`.
+      #
+      # @private
+      class ForkRunner
+        def self.start(shell_command, spec_runner)
+          instance = new(shell_command, spec_runner)
+          yield instance
+        ensure
+          instance.shutdown
+        end
+
+        def self.name
+          :fork
+        end
+
+        def initialize(shell_command, spec_runner)
+          @shell_command = shell_command
+          @channel = Channel.new
+          @run_dispatcher = RunDispatcher.new(spec_runner, @channel)
+        end
+
+        def run(locations)
+          run_descriptor = ExampleSetDescriptor.new(locations, original_results.failed_example_ids)
+          dispatch_run(run_descriptor)
+        end
+
+        def original_results
+          @original_results ||= dispatch_run(ExampleSetDescriptor.new(
+            @shell_command.original_locations, []))
+        end
+
+        def shutdown
+          @channel.close
+        end
+
+      private
+
+        def dispatch_run(run_descriptor)
+          @run_dispatcher.dispatch_specs(run_descriptor)
+          @channel.receive.tap do |result|
+            if result.is_a?(String)
+              raise BisectFailedError.for_failed_spec_run(result)
+            end
+          end
+        end
+
+        # @private
+        class RunDispatcher
+          def initialize(runner, channel)
+            @runner = runner
+            @channel = channel
+
+            @spec_output = StringIO.new
+
+            runner.configuration.tap do |c|
+              c.reset_reporter
+              c.output_stream = @spec_output
+              c.error_stream = @spec_output
+            end
+          end
+
+          def dispatch_specs(run_descriptor)
+            pid = fork { run_specs(run_descriptor) }
+            # We don't use Process.waitpid here as it was causing bisects to
+            # block due to the file descriptor limit on OSX / Linux. We need
+            # to detach the process to avoid having zombie processes
+            # consuming slots in the kernel process table during bisect runs.
+            Process.detach(pid)
+          end
+
+        private
+
+          def run_specs(run_descriptor)
+            $stdout = $stderr = @spec_output
+            formatter = CaptureFormatter.new(run_descriptor.failed_example_ids)
+
+            @runner.configuration.tap do |c|
+              c.files_or_directories_to_run = run_descriptor.all_example_ids
+              c.formatter = formatter
+              c.load_spec_files
+            end
+
+            # `announce_filters` has the side effect of implementing the logic
+            # that honors `config.run_all_when_everything_filtered` so we need
+            # to call it here. When we remove `run_all_when_everything_filtered`
+            # (slated for RSpec 4), we can remove this call to `announce_filters`.
+            @runner.world.announce_filters
+
+            @runner.run_specs(@runner.world.ordered_example_groups)
+            latest_run_results = formatter.results
+
+            if latest_run_results.nil? || latest_run_results.all_example_ids.empty?
+              @channel.send(@spec_output.string)
+            else
+              @channel.send(latest_run_results)
+            end
+          end
+        end
+
+        class CaptureFormatter < Formatters::BaseBisectFormatter
+          attr_accessor :results
+          alias_method :notify_results, :results=
+        end
+      end
+    end
+  end
+end

data/lib/rspec/core/bisect/server.rb

@@ -0,0 +1,61 @@
+require 'drb/drb'
+require 'drb/acl'
+RSpec::Support.require_rspec_core "bisect/utilities"
+
+module RSpec
+  module Core
+    # @private
+    module Bisect
+      # @private
+      # A DRb server that receives run results from a separate RSpec process
+      # started by the bisect process.
+      class Server
+        def self.run
+          server = new
+          server.start
+          yield server
+        ensure
+          server.stop
+        end
+
+        def capture_run_results(files_or_directories_to_run=[], expected_failures=[])
+          self.expected_failures  = expected_failures
+          self.files_or_directories_to_run = files_or_directories_to_run
+          self.latest_run_results = nil
+          run_output = yield
+
+          if latest_run_results.nil? || latest_run_results.all_example_ids.empty?
+            raise BisectFailedError.for_failed_spec_run(run_output)
+          end
+
+          latest_run_results
+        end
+
+        def start
+          # Only allow remote DRb requests from this machine.
+          DRb.install_acl ACL.new(%w[ deny all allow localhost allow 127.0.0.1 allow ::1 ])
+
+          # We pass `nil` as the first arg to allow it to pick a DRb port.
+          @drb = DRb.start_service(nil, self)
+        end
+
+        def stop
+          @drb.stop_service
+        end
+
+        def drb_port
+          @drb_port ||= Integer(@drb.uri[/\d+$/])
+        end
+
+        # Fetched via DRb by the BisectDRbFormatter to determine when to abort.
+        attr_accessor :expected_failures
+
+        # Set via DRb by the BisectDRbFormatter with the results of the run.
+        attr_accessor :latest_run_results
+
+        # Fetched via DRb to tell clients which files to run
+        attr_accessor :files_or_directories_to_run
+      end
+    end
+  end
+end

data/lib/rspec/core/bisect/shell_command.rb

@@ -0,0 +1,126 @@
+RSpec::Support.require_rspec_core "shell_escape"
+require 'shellwords'
+
+module RSpec
+  module Core
+    module Bisect
+      # Provides an API to generate shell commands to run the suite for a
+      # set of locations, using the given bisect server to capture the results.
+      # @private
+      class ShellCommand
+        attr_reader :original_cli_args
+
+        def initialize(original_cli_args)
+          @original_cli_args = original_cli_args.reject { |arg| arg.start_with?("--bisect") }
+        end
+
+        def command_for(locations, server)
+          parts = []
+
+          parts << RUBY << load_path
+          parts << open3_safe_escape(RSpec::Core.path_to_executable)
+
+          parts << "--format"   << "bisect-drb"
+          parts << "--drb-port" << server.drb_port
+
+          parts.concat(reusable_cli_options)
+          parts.concat(locations.map { |l| open3_safe_escape(l) })
+
+          parts.join(" ")
+        end
+
+        def repro_command_from(locations)
+          parts = []
+
+          parts.concat environment_repro_parts
+          parts << "rspec"
+          parts.concat Formatters::Helpers.organize_ids(locations)
+          parts.concat original_cli_args_without_locations
+
+          parts.join(" ")
+        end
+
+        def original_locations
+          parsed_original_cli_options.fetch(:files_or_directories_to_run)
+        end
+
+        def bisect_environment_hash
+          if ENV.key?('SPEC_OPTS')
+            { 'SPEC_OPTS' => spec_opts_without_bisect }
+          else
+            {}
+          end
+        end
+
+        def spec_opts_without_bisect
+          Shellwords.join(
+            Shellwords.split(ENV.fetch('SPEC_OPTS', '')).reject do |arg|
+              arg =~ /^--bisect/
+            end
+          )
+        end
+
+      private
+
+        include RSpec::Core::ShellEscape
+        # On JRuby, Open3.popen3 does not handle shellescaped args properly:
+        # https://github.com/jruby/jruby/issues/2767
+        if RSpec::Support::Ruby.jruby?
+          # :nocov:
+          alias open3_safe_escape quote
+          # :nocov:
+        else
+          alias open3_safe_escape escape
+        end
+
+        def environment_repro_parts
+          bisect_environment_hash.map do |k, v|
+            %Q(#{k}="#{v}")
+          end
+        end
+
+        def reusable_cli_options
+          @reusable_cli_options ||= begin
+            opts = original_cli_args_without_locations
+
+            if (port = parsed_original_cli_options[:drb_port])
+              opts -= %W[ --drb-port #{port} ]
+            end
+
+            parsed_original_cli_options.fetch(:formatters) { [] }.each do |(name, out)|
+              opts -= %W[ --format #{name} -f -f#{name} ]
+              opts -= %W[ --out #{out} -o -o#{out} ]
+            end
+
+            opts
+          end
+        end
+
+        def original_cli_args_without_locations
+          @original_cli_args_without_locations ||= begin
+            files_or_dirs = parsed_original_cli_options.fetch(:files_or_directories_to_run)
+            @original_cli_args - files_or_dirs
+          end
+        end
+
+        def parsed_original_cli_options
+          @parsed_original_cli_options ||= Parser.parse(@original_cli_args)
+        end
+
+        def load_path
+          @load_path ||= "-I#{$LOAD_PATH.map { |p| open3_safe_escape(p) }.join(':')}"
+        end
+
+        # Path to the currently running Ruby executable, borrowed from Rake:
+        # https://github.com/ruby/rake/blob/v10.4.2/lib/rake/file_utils.rb#L8-L12
+        # Note that we skip `ENV['RUBY']` because we don't have to deal with running
+        # RSpec from within a MRI source repository:
+        # https://github.com/ruby/rake/commit/968682759b3b65e42748cd2befb2ff3e982272d9
+        RUBY = File.join(
+          RbConfig::CONFIG['bindir'],
+          RbConfig::CONFIG['ruby_install_name'] + RbConfig::CONFIG['EXEEXT']).
+          sub(/.*\s.*/m, '"\&"')
+      end
+    end
+  end
+end

data/lib/rspec/core/bisect/shell_runner.rb

@@ -0,0 +1,73 @@
+require 'open3'
+RSpec::Support.require_rspec_core "bisect/server"
+
+module RSpec
+  module Core
+    module Bisect
+      # Provides an API to run the suite for a set of locations, using
+      # the given bisect server to capture the results.
+      #
+      # Sets of specs are run by shelling out.
+      # @private
+      class ShellRunner
+        def self.start(shell_command, _spec_runner)
+          Server.run do |server|
+            yield new(server, shell_command)
+          end
+        end
+
+        def self.name
+          :shell
+        end
+
+        def initialize(server, shell_command)
+          @server        = server
+          @shell_command = shell_command
+        end
+
+        def run(locations)
+          run_locations(locations, original_results.failed_example_ids)
+        end
+
+        def original_results
+          @original_results ||= run_locations(@shell_command.original_locations)
+        end
+
+      private
+
+        def run_locations(*capture_args)
+          @server.capture_run_results(*capture_args) do
+            run_command @shell_command.command_for([], @server)
+          end
+        end
+
+        # `Open3.capture2e` does not work on JRuby:
+        # https://github.com/jruby/jruby/issues/2766
+        if Open3.respond_to?(:capture2e) && !RSpec::Support::Ruby.jruby?
+          def run_command(cmd)
+            Open3.capture2e(@shell_command.bisect_environment_hash, cmd).first
+          end
+        else # for 1.8.7
+          # :nocov:
+          def run_command(cmd)
+            out = err = nil
+
+            original_spec_opts = ENV['SPEC_OPTS']
+            ENV['SPEC_OPTS'] = @shell_command.spec_opts_without_bisect
+
+            Open3.popen3(cmd) do |_, stdout, stderr|
+              # Reading the streams blocks until the process is complete
+              out = stdout.read
+              err = stderr.read
+            end
+
+            "Stdout:\n#{out}\n\nStderr:\n#{err}"
+          ensure
+            ENV['SPEC_OPTS'] = original_spec_opts
+          end
+          # :nocov:
+        end
+      end
+    end
+  end
+end

data/lib/rspec/core/bisect/utilities.rb

@@ -0,0 +1,69 @@
+module RSpec
+  module Core
+    module Bisect
+      # @private
+      ExampleSetDescriptor = Struct.new(:all_example_ids, :failed_example_ids)
+
+      # @private
+      class BisectFailedError < StandardError
+        def self.for_failed_spec_run(spec_output)
+          new("Failed to get results from the spec run. Spec run output:\n\n" +
+              spec_output)
+        end
+      end
+
+      # Wraps a `formatter` providing a simple means to notify it in place
+      # of an `RSpec::Core::Reporter`, without involving configuration in
+      # any way.
+      # @private
+      class Notifier
+        def initialize(formatter)
+          @formatter = formatter
+        end
+
+        def publish(event, *args)
+          return unless @formatter.respond_to?(event)
+          notification = Notifications::CustomNotification.for(*args)
+          @formatter.__send__(event, notification)
+        end
+      end
+
+      # Wraps a pipe to support sending objects between a child and
+      # parent process. Where supported, encoding is explicitly
+      # set to ensure binary data is able to pass from child to
+      # parent.
+      # @private
+      class Channel
+        if String.method_defined?(:encoding)
+          MARSHAL_DUMP_ENCODING = Marshal.dump("").encoding
+        end
+
+        def initialize
+          @read_io, @write_io = IO.pipe
+
+          if defined?(MARSHAL_DUMP_ENCODING) && IO.method_defined?(:set_encoding)
+            # Ensure the pipe can send any content produced by Marshal.dump
+            @write_io.set_encoding MARSHAL_DUMP_ENCODING
+          end
+        end
+
+        def send(message)
+          packet = Marshal.dump(message)
+          @write_io.write("#{packet.bytesize}\n#{packet}")
+        end
+
+        # rubocop:disable Security/MarshalLoad
+        def receive
+          packet_size = Integer(@read_io.gets)
+          Marshal.load(@read_io.read(packet_size))
+        end
+        # rubocop:enable Security/MarshalLoad
+
+        def close
+          @read_io.close
+          @write_io.close
+        end
+      end
+    end
+  end
+end