rspec-core 3.2.3 → 3.3.0

data/lib/rspec/core/bisect/example_minimizer.rb

@@ -0,0 +1,130 @@
+RSpec::Support.require_rspec_core "bisect/subset_enumerator"
+
+module RSpec
+  module Core
+    module Bisect
+      # @private
+      # Contains the core bisect logic. Searches for examples we can ignore by
+      # repeatedly running different subsets of the suite.
+      class ExampleMinimizer
+        attr_reader :runner, :reporter, :all_example_ids, :failed_example_ids
+        attr_accessor :remaining_ids
+
+        def initialize(runner, reporter)
+          @runner   = runner
+          @reporter = reporter
+        end
+
+        def find_minimal_repro
+          prep
+
+          self.remaining_ids = non_failing_example_ids
+
+          each_bisect_round do |subsets|
+            ids_to_ignore = subsets.find do |ids|
+              get_expected_failures_for?(remaining_ids - ids)
+            end
+
+            next :done unless ids_to_ignore
+
+            self.remaining_ids -= ids_to_ignore
+            notify(:bisect_ignoring_ids, :ids_to_ignore => ids_to_ignore, :remaining_ids => remaining_ids)
+          end
+
+          currently_needed_ids
+        end
+
+        def currently_needed_ids
+          remaining_ids + failed_example_ids
+        end
+
+        def repro_command_for_currently_needed_ids
+          return runner.repro_command_from(currently_needed_ids) if remaining_ids
+          "(Not yet enough information to provide any repro command)"
+        end
+
+      private
+
+        def prep
+          notify(:bisect_starting, :original_cli_args => runner.original_cli_args)
+
+          _, duration = track_duration do
+            original_results    = runner.original_results
+            @all_example_ids    = original_results.all_example_ids
+            @failed_example_ids = original_results.failed_example_ids
+          end
+
+          if @failed_example_ids.empty?
+            raise BisectFailedError, "\n\nNo failures found. Bisect only works " \
+                  "in the presence of one or more failing examples."
+          else
+            notify(:bisect_original_run_complete, :failed_example_ids => failed_example_ids,
+                                                  :non_failing_example_ids => non_failing_example_ids,
+                                                  :duration => duration)
+          end
+        end
+
+        def non_failing_example_ids
+          @non_failing_example_ids ||= all_example_ids - failed_example_ids
+        end
+
+        def get_expected_failures_for?(ids)
+          ids_to_run = ids + failed_example_ids
+          notify(:bisect_individual_run_start, :command => runner.repro_command_from(ids_to_run))
+
+          results, duration = track_duration { runner.run(ids_to_run) }
+          notify(:bisect_individual_run_complete, :duration => duration, :results => results)
+
+          abort_if_ordering_inconsistent(results)
+          (failed_example_ids & results.failed_example_ids) == failed_example_ids
+        end
+
+        INFINITY = (1.0 / 0) # 1.8.7 doesn't define Float::INFINITY so we define our own...
+
+        def each_bisect_round(&block)
+          last_round, duration = track_duration do
+            1.upto(INFINITY) do |round|
+              break if :done == bisect_round(round, &block)
+            end
+          end
+
+          notify(:bisect_complete, :round => last_round, :duration => duration,
+                                   :original_non_failing_count => non_failing_example_ids.size,
+                                   :remaining_count => remaining_ids.size)
+        end
+
+        def bisect_round(round)
+          value, duration = track_duration do
+            subsets = SubsetEnumerator.new(remaining_ids)
+            notify(:bisect_round_started, :round => round,
+                                          :subset_size => subsets.subset_size,
+                                          :remaining_count => remaining_ids.size)
+
+            yield subsets
+          end
+
+          notify(:bisect_round_finished, :duration => duration, :round => round)
+          value
+        end
+
+        def track_duration
+          start = ::RSpec::Core::Time.now
+          [yield, ::RSpec::Core::Time.now - start]
+        end
+
+        def abort_if_ordering_inconsistent(results)
+          expected_order = all_example_ids & results.all_example_ids
+          return if expected_order == results.all_example_ids
+
+          raise BisectFailedError, "\n\nThe example ordering is inconsistent. " \
+                "`--bisect` relies upon consistent ordering (e.g. by passing " \
+                "`--seed` if you're using random ordering) to work properly."
+        end
+
+        def notify(*args)
+          reporter.publish(*args)
+        end
+      end
+    end
+  end
+end

data/lib/rspec/core/bisect/runner.rb

@@ -0,0 +1,139 @@
+RSpec::Support.require_rspec_core "shell_escape"
+require 'open3'
+
+module RSpec
+  module Core
+    module Bisect
+      # Provides an API to run the suite for a set of locations, using
+      # the given bisect server to capture the results.
+      # @private
+      class Runner
+        attr_reader :original_cli_args
+
+        def initialize(server, original_cli_args)
+          @server            = server
+          @original_cli_args = original_cli_args.reject { |arg| arg.start_with?("--bisect") }
+        end
+
+        def run(locations)
+          run_locations(locations, original_results.failed_example_ids)
+        end
+
+        def command_for(locations)
+          parts = []
+
+          parts << RUBY << load_path
+          parts << open3_safe_escape(RSpec::Core.path_to_executable)
+
+          parts << "--format"   << "bisect"
+          parts << "--drb-port" << @server.drb_port
+          parts.concat reusable_cli_options
+          parts.concat locations.map { |l| open3_safe_escape(l) }
+
+          parts.join(" ")
+        end
+
+        def repro_command_from(locations)
+          parts = []
+
+          parts << "rspec"
+          parts.concat Formatters::Helpers.organize_ids(locations)
+          parts.concat original_cli_args_without_locations
+
+          parts.join(" ")
+        end
+
+        def original_results
+          @original_results ||= run_locations(original_locations)
+        end
+
+      private
+
+        include RSpec::Core::ShellEscape
+        # On JRuby, Open3.popen3 does not handle shellescaped args properly:
+        # https://github.com/jruby/jruby/issues/2767
+        if RSpec::Support::Ruby.jruby?
+          # :nocov:
+          alias open3_safe_escape quote
+          # :nocov:
+        else
+          alias open3_safe_escape escape
+        end
+
+        def run_locations(locations, *capture_args)
+          @server.capture_run_results(*capture_args) do
+            run_command command_for(locations)
+          end
+        end
+
+        # `Open3.capture2e` does not work on JRuby:
+        # https://github.com/jruby/jruby/issues/2766
+        if Open3.respond_to?(:capture2e) && !RSpec::Support::Ruby.jruby?
+          def run_command(cmd)
+            Open3.capture2e(cmd).first
+          end
+        else # for 1.8.7
+          # :nocov:
+          def run_command(cmd)
+            out = err = nil
+
+            Open3.popen3(cmd) do |_, stdout, stderr|
+              # Reading the streams blocks until the process is complete
+              out = stdout.read
+              err = stderr.read
+            end
+
+            "Stdout:\n#{out}\n\nStderr:\n#{err}"
+          end
+          # :nocov:
+        end
+
+        def reusable_cli_options
+          @reusable_cli_options ||= begin
+            opts = original_cli_args_without_locations
+
+            if (port = parsed_original_cli_options[:drb_port])
+              opts -= %W[ --drb-port #{port} ]
+            end
+
+            parsed_original_cli_options.fetch(:formatters) { [] }.each do |(name, out)|
+              opts -= %W[ --format #{name} -f -f#{name} ]
+              opts -= %W[ --out #{out} -o -o#{out} ]
+            end
+
+            opts
+          end
+        end
+
+        def original_cli_args_without_locations
+          @original_cli_args_without_locations ||= begin
+            files_or_dirs = parsed_original_cli_options.fetch(:files_or_directories_to_run)
+            @original_cli_args - files_or_dirs
+          end
+        end
+
+        def parsed_original_cli_options
+          @parsed_original_cli_options ||= Parser.parse(@original_cli_args)
+        end
+
+        def original_locations
+          parsed_original_cli_options.fetch(:files_or_directories_to_run)
+        end
+
+        def load_path
+          @load_path ||= "-I#{$LOAD_PATH.map { |p| open3_safe_escape(p) }.join(':')}"
+        end
+
+        # Path to the currently running Ruby executable, borrowed from Rake:
+        # https://github.com/ruby/rake/blob/v10.4.2/lib/rake/file_utils.rb#L8-L12
+        # Note that we skip `ENV['RUBY']` because we don't have to deal with running
+        # RSpec from within a MRI source repository:
+        # https://github.com/ruby/rake/commit/968682759b3b65e42748cd2befb2ff3e982272d9
+        RUBY = File.join(
+          RbConfig::CONFIG['bindir'],
+          RbConfig::CONFIG['ruby_install_name'] + RbConfig::CONFIG['EXEEXT']).
+          sub(/.*\s.*/m, '"\&"')
+      end
+    end
+  end
+end

data/lib/rspec/core/bisect/server.rb

@@ -0,0 +1,61 @@
+require 'drb/drb'
+require 'drb/acl'
+
+module RSpec
+  module Core
+    # @private
+    module Bisect
+      # @private
+      BisectFailedError = Class.new(StandardError)
+
+      # @private
+      # A DRb server that receives run results from a separate RSpec process
+      # started by the bisect process.
+      class Server
+        def self.run
+          server = new
+          server.start
+          yield server
+        ensure
+          server.stop
+        end
+
+        def capture_run_results(expected_failures=[])
+          self.expected_failures  = expected_failures
+          self.latest_run_results = nil
+          run_output = yield
+          latest_run_results || raise_bisect_failed(run_output)
+        end
+
+        def start
+          # Only allow remote DRb requests from this machine.
+          DRb.install_acl ACL.new(%w[ deny all allow localhost allow 127.0.0.1 ])
+
+          # We pass `nil` as the first arg to allow it to pick a DRb port.
+          @drb = DRb.start_service(nil, self)
+        end
+
+        def stop
+          @drb.stop_service
+        end
+
+        def drb_port
+          @drb_port ||= Integer(@drb.uri[/\d+$/])
+        end
+
+        # Fetched via DRb by the BisectFormatter to determine when to abort.
+        attr_accessor :expected_failures
+
+        # Set via DRb by the BisectFormatter with the results of the run.
+        attr_accessor :latest_run_results
+
+      private
+
+        def raise_bisect_failed(run_output)
+          raise BisectFailedError, "Failed to get results from the spec " \
+                "run. Spec run output:\n\n#{run_output}"
+        end
+      end
+    end
+  end
+end

data/lib/rspec/core/bisect/subset_enumerator.rb

@@ -0,0 +1,39 @@
+module RSpec
+  module Core
+    module Bisect
+      # Enumerates each subset of the given list of ids that is half the
+      # size of the total list, so that hopefully we can discard half the
+      # list each repeatedly in order to our minimal repro case.
+      # @private
+      class SubsetEnumerator
+        include Enumerable
+
+        def initialize(ids)
+          @ids = ids
+        end
+
+        def subset_size
+          @subset_size ||= (@ids.size / 2.0).ceil
+        end
+
+        def each
+          yielded     = Set.new
+          slice_size  = subset_size
+          combo_count = 1
+
+          while slice_size > 0
+            @ids.each_slice(slice_size).to_a.combination(combo_count) do |combos|
+              subset = combos.flatten
+              next if yielded.include?(subset)
+              yield subset
+              yielded << subset
+            end
+
+            slice_size  /= 2
+            combo_count *= 2
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/core/configuration.rb

@@ -154,6 +154,33 @@ module RSpec
         end
       end
 
+      # @macro define_reader
+      # The file path to use for persisting example statuses. Necessary for the
+      # `--only-failures` and `--next-failures` CLI options.
+      #
+      # @overload example_status_persistence_file_path
+      #   @return [String] the file path
+      # @overload example_status_persistence_file_path=(value)
+      #   @param value [String] the file path
+      define_reader :example_status_persistence_file_path
+
+      # Sets the file path to use for persisting example statuses. Necessary for the
+      # `--only-failures` and `--next-failures` CLI options.
+      def example_status_persistence_file_path=(value)
+        @example_status_persistence_file_path = value
+        clear_values_derived_from_example_status_persistence_file_path
+      end
+
+      # @macro define_reader
+      # Indicates if the `--only-failures` (or `--next-failure`) flag is being used.
+      define_reader :only_failures
+      alias_method :only_failures?, :only_failures
+
+      # @private
+      def only_failures_but_not_configured?
+        only_failures? && !example_status_persistence_file_path
+      end
+
       # @macro add_setting
       # Clean up and exit after the first failure (default: `false`).
       add_setting :fail_fast
@@ -281,6 +308,11 @@ module RSpec
       # Record the start time of the spec suite to measure load time.
       add_setting :start_time
 
+      # @macro add_setting
+      # Use threadsafe options where available.
+      # Currently this will place a mutex around memoized values such as let blocks.
+      add_setting :threadsafe
+
       # @private
       add_setting :tty
       # @private
@@ -334,6 +366,9 @@ module RSpec
         @requires = []
         @libs = []
         @derived_metadata_blocks = FilterableItemRepository::QueryOptimized.new(:any?)
+        @threadsafe = true
+
+        define_built_in_hooks
       end
 
       # @private
@@ -342,6 +377,9 @@ module RSpec
       def force(hash)
         ordering_manager.force(hash)
         @preferred_options.merge!(hash)
+
+        return unless hash.key?(:example_status_persistence_file_path)
+        clear_values_derived_from_example_status_persistence_file_path
       end
 
       # @private
@@ -600,6 +638,13 @@ module RSpec
             framework
           when :rspec
             require 'rspec/expectations'
+
+            # Tag this exception class so our exception formatting logic knows
+            # that it satisfies the `MultipleExceptionError` interface.
+            ::RSpec::Expectations::MultipleExpectationsNotMetError.__send__(
+              :include, MultipleExceptionError::InterfaceTag
+            )
+
             ::RSpec::Matchers
           when :test_unit
             require 'rspec/core/test_unit_assertions_adapter'
@@ -792,7 +837,11 @@ module RSpec
       # @private
       def files_or_directories_to_run=(*files)
         files = files.flatten
-        files << default_path if (command == 'rspec' || Runner.running_in_drb?) && default_path && files.empty?
+
+        if (command == 'rspec' || Runner.running_in_drb?) && default_path && files.empty?
+          files << default_path
+        end
+
         @files_or_directories_to_run = files
         @files_to_run = nil
       end
@@ -803,6 +852,40 @@ module RSpec
         @files_to_run ||= get_files_to_run(@files_or_directories_to_run)
       end
 
+      # @private
+      def last_run_statuses
+        @last_run_statuses ||= Hash.new(UNKNOWN_STATUS).tap do |statuses|
+          if (path = example_status_persistence_file_path)
+            begin
+              ExampleStatusPersister.load_from(path).inject(statuses) do |hash, example|
+                hash[example.fetch(:example_id)] = example.fetch(:status)
+                hash
+              end
+            rescue SystemCallError => e
+              RSpec.warning "Could not read from #{path.inspect} (configured as " \
+                            "`config.example_status_persistence_file_path`) due " \
+                            "to a system error: #{e.inspect}. Please check that " \
+                            "the config option is set to an accessible, valid " \
+                            "file path", :call_site => nil
+            end
+          end
+        end
+      end
+
+      # @private
+      UNKNOWN_STATUS = "unknown".freeze
+
+      # @private
+      FAILED_STATUS = "failed".freeze
+
+      # @private
+      def spec_files_with_failures
+        @spec_files_with_failures ||= last_run_statuses.inject(Set.new) do |files, (id, status)|
+          files << id.split(ON_SQUARE_BRACKETS).first if status == FAILED_STATUS
+          files
+        end.to_a
+      end
+
       # Creates a method that delegates to `example` including the submitted
       # `args`. Used internally to add variants of `example` like `pending`:
       # @param name [String] example name alias
@@ -1060,6 +1143,7 @@ module RSpec
       def include(mod, *filters)
         meta = Metadata.build_hash_from(filters, :warn_about_example_group_filtering)
         @include_modules.append(mod, meta)
+        configure_existing_groups(mod, meta, :safe_include)
       end
 
       # Tells RSpec to extend example groups with `mod`. Methods defined in
@@ -1095,6 +1179,7 @@ module RSpec
       def extend(mod, *filters)
         meta = Metadata.build_hash_from(filters, :warn_about_example_group_filtering)
         @extend_modules.append(mod, meta)
+        configure_existing_groups(mod, meta, :safe_extend)
       end
 
       if RSpec::Support::RubyFeatures.module_prepends_supported?
@@ -1133,6 +1218,7 @@ module RSpec
         def prepend(mod, *filters)
           meta = Metadata.build_hash_from(filters, :warn_about_example_group_filtering)
           @prepend_modules.append(mod, meta)
+          configure_existing_groups(mod, meta, :safe_prepend)
         end
       end
 
@@ -1153,17 +1239,30 @@ module RSpec
         end
       end
 
+      # @private
+      def configure_existing_groups(mod, meta, application_method)
+        RSpec.world.all_example_groups.each do |group|
+          next unless meta.empty? || MetadataFilter.apply?(:any?, meta, group.metadata)
+          __send__(application_method, mod, group)
+        end
+      end
+
       # @private
       #
       # Used internally to extend the singleton class of a single example's
       # example group instance with modules using `include` and/or `extend`.
       def configure_example(example)
+        singleton_group = example.example_group_instance.singleton_class
+
         # We replace the metadata so that SharedExampleGroupModule#included
         # has access to the example's metadata[:location].
-        example.example_group_instance.singleton_class.with_replaced_metadata(example.metadata) do
-          @include_modules.items_for(example.metadata).each do |mod|
+        singleton_group.with_replaced_metadata(example.metadata) do
+          modules = @include_modules.items_for(example.metadata)
+          modules.each do |mod|
             safe_include(mod, example.example_group_instance.singleton_class)
           end
+
+          MemoizedHelpers.define_helpers_on(singleton_group) unless modules.empty?
         end
       end
 
@@ -1193,7 +1292,8 @@ module RSpec
         def safe_extend(mod, host)
           host.extend(mod) unless host.singleton_class < mod
         end
-      else
+      else # for 1.8.7
+        # :nocov:
         # @private
         def safe_include(mod, host)
           host.__send__(:include, mod) unless host.included_modules.include?(mod)
@@ -1203,6 +1303,7 @@ module RSpec
         def safe_extend(mod, host)
           host.extend(mod) unless (class << host; self; end).included_modules.include?(mod)
         end
+        # :nocov:
       end
 
       # @private
@@ -1560,10 +1661,15 @@ module RSpec
       end
 
       def get_files_to_run(paths)
-        FlatMap.flat_map(paths_to_check(paths)) do |path|
+        files = FlatMap.flat_map(paths_to_check(paths)) do |path|
           path = path.gsub(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR
           File.directory?(path) ? gather_directories(path) : extract_location(path)
         end.sort.uniq
+
+        return files unless only_failures?
+        relative_files = files.map { |f| Metadata.relative_path(File.expand_path f) }
+        intersection = (relative_files & spec_files_with_failures.to_a)
+        intersection.empty? ? files : intersection
       end
 
       def paths_to_check(paths)
@@ -1592,6 +1698,7 @@ module RSpec
       end
 
       if RSpec::Support::OS.windows?
+        # :nocov:
         def absolute_pattern?(pattern)
           pattern =~ /\A[A-Z]:\\/ || windows_absolute_network_path?(pattern)
         end
@@ -1600,12 +1707,16 @@ module RSpec
           return false unless ::File::ALT_SEPARATOR
           pattern.start_with?(::File::ALT_SEPARATOR + ::File::ALT_SEPARATOR)
         end
+        # :nocov:
       else
         def absolute_pattern?(pattern)
           pattern.start_with?(File::Separator)
         end
       end
 
+      # @private
+      ON_SQUARE_BRACKETS = /[\[\]]/
+
       def extract_location(path)
         match = /^(.*?)((?:\:\d+)+)$/.match(path)
 
@@ -1613,6 +1724,9 @@ module RSpec
           captures = match.captures
           path, lines = captures[0], captures[1][1..-1].split(":").map { |n| n.to_i }
           filter_manager.add_location path, lines
+        else
+          path, scoped_ids = path.split(ON_SQUARE_BRACKETS)
+          filter_manager.add_ids(path, scoped_ids.split(/\s*,\s*/)) if scoped_ids
         end
 
         return [] if path == default_path
@@ -1627,6 +1741,16 @@ module RSpec
         @preferred_options.fetch(key) { yield }
       end
 
+      def define_built_in_hooks
+        around(:example, :aggregate_failures => true) do |procsy|
+          begin
+            aggregate_failures(nil, :hide_backtrace => true, &procsy)
+          rescue Exception => exception
+            procsy.example.set_aggregate_failures_exception(exception)
+          end
+        end
+      end
+
       def assert_no_example_groups_defined(config_option)
         return unless RSpec.world.example_groups.any?
 
@@ -1672,6 +1796,11 @@ module RSpec
         instance_variable_set(:"@#{name}", value)
         @files_to_run = nil
       end
+
+      def clear_values_derived_from_example_status_persistence_file_path
+        @last_run_statuses = nil
+        @spec_files_with_failures = nil
+      end
     end
     # rubocop:enable Style/ClassLength
   end