bundler 1.3.6 → 1.4.0.pre.1

data/lib/bundler/parallel_workers.rb

@@ -0,0 +1,18 @@
+require 'thread'
+
+require "bundler/parallel_workers/worker"
+
+module Bundler
+  module ParallelWorkers
+    autoload :UnixWorker, "bundler/parallel_workers/unix_worker"
+    autoload :ThreadWorker, "bundler/parallel_workers/thread_worker"
+
+    def self.worker_pool(size, job)
+      if Bundler.current_ruby.mswin? || Bundler.current_ruby.jruby?
+        ThreadWorker.new(size, job)
+      else
+        UnixWorker.new(size, job)
+      end
+    end
+  end
+end

data/lib/bundler/parallel_workers/thread_worker.rb

@@ -0,0 +1,27 @@
+module Bundler
+  module ParallelWorkers
+    class ThreadWorker < Worker
+
+      private
+
+      # On platforms where fork is not available
+      # use Threads for parallely downloading gems
+      #
+      # @param size [Integer] Size of thread worker pool
+      # @param func [Proc] Job to be run inside thread worker pool
+      def prepare_workers(size, func)
+        @threads = size.times.map do |i|
+          Thread.start do
+            Thread.current.abort_on_exception = true
+            loop do
+              obj = @request_queue.deq
+              break if obj.equal? POISON
+              @response_queue.enq func.call(obj)
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/bundler/parallel_workers/unix_worker.rb

@@ -0,0 +1,88 @@
+module Bundler
+  module ParallelWorkers
+    # UnixWorker is used only on platforms where fork is available. The way
+    # this code works is, it forks a preconfigured number of workers and then
+    # It starts preconfigured number of threads that write to the connected pipe.
+    class UnixWorker < Worker
+
+      class JobHandler < Struct.new(:pid, :io_r, :io_w)
+        def work(obj)
+          Marshal.dump obj, io_w
+          Marshal.load io_r
+        rescue IOError
+          nil
+        end
+      end
+
+      private
+
+      # Start forked workers for downloading gems. This version of worker
+      # is only used on platforms where fork is available.
+      #
+      # @param size [Integer] Size of worker pool
+      # @param func [Proc] Job that should be executed in the worker
+      def prepare_workers(size, func)
+        @workers = size.times.map do
+          child_read, parent_write = IO.pipe
+          parent_read, child_write = IO.pipe
+
+          pid = Process.fork do
+            begin
+              parent_read.close
+              parent_write.close
+
+              while !child_read.eof?
+                obj = Marshal.load child_read
+                Marshal.dump func.call(obj), child_write
+              end
+            rescue Exception => e
+              begin
+                Marshal.dump WrappedException.new(e), child_write
+              rescue Errno::EPIPE
+                nil
+              end
+            ensure
+              child_read.close
+              child_write.close
+            end
+          end
+
+          child_read.close
+          child_write.close
+          JobHandler.new pid, parent_read, parent_write
+        end
+      end
+
+      # Start the threads whose job is basically to wait for incoming messages
+      # on request queue and write that message to the connected pipe. Also retrieve
+      # messages from child worker via connected pipe and write the message to response queue
+      #
+      # @param size [Integer] Number of threads to be started
+      def prepare_threads(size)
+        @threads = size.times.map do |i|
+          Thread.start do
+            worker = @workers[i]
+            Thread.current.abort_on_exception = true
+            loop do
+              obj = @request_queue.deq
+              break if obj.equal? POISON
+              @response_queue.enq worker.work(obj)
+            end
+          end
+        end
+      end
+
+      # Kill the forked workers by sending SIGINT to them
+      def stop_workers
+        @workers.each do |worker|
+          worker.io_r.close
+          worker.io_w.close
+          Process.kill :INT, worker.pid
+        end
+        @workers.each do |worker|
+          Process.waitpid worker.pid
+        end
+      end
+    end
+  end
+end

data/lib/bundler/parallel_workers/worker.rb

@@ -0,0 +1,68 @@
+module Bundler
+  module ParallelWorkers
+    class Worker
+      POISON = Object.new
+
+      class WrappedException < StandardError
+        attr_reader :exception
+        def initialize(exn)
+          @exception = exn
+        end
+      end
+
+      # Creates a worker pool of specified size
+      #
+      # @param size [Integer] Size of pool
+      # @param func [Proc] job to run in inside the worker pool
+      def initialize(size, func)
+        @request_queue = Queue.new
+        @response_queue = Queue.new
+        prepare_workers size, func
+        prepare_threads size
+      end
+
+      # Enque a request to be executed in the worker pool
+      #
+      # @param obj [String] mostly it is name of spec that should be downloaded
+      def enq(obj)
+        @request_queue.enq obj
+      end
+
+      # Retrieves results of job function being executed in worker pool
+      def deq
+        result = @response_queue.deq
+        if WrappedException === result
+          raise result.exception
+        end
+        result
+      end
+
+      # Stop the forked workers and started threads
+      def stop
+        stop_workers
+        stop_threads
+      end
+
+      private
+      # Stop the worker threads by sending a poison object down the request queue 
+      # so as worker threads after retrieving it, shut themselves down
+      def stop_threads
+        @threads.each do
+          @request_queue.enq POISON
+        end
+        @threads.each do |thread|
+          thread.join
+        end
+      end
+
+      # To be overridden by child classes
+      def prepare_threads(size)
+      end
+
+      # To be overridden by child classes
+      def stop_workers
+      end
+
+    end
+  end
+end

data/lib/bundler/resolver.rb

@@ -1,4 +1,5 @@
 require 'set'
+require 'bundler/safe_catch'
 # This is the latest iteration of the gem dependency resolving algorithm. As of now,
 # it can resolve (as a success or failure) any set of gem dependencies we throw at it
 # in a reasonable amount of time. The most iterations I've seen it take is about 150.
@@ -21,6 +22,9 @@ end
 
 module Bundler
   class Resolver
+    include SafeCatch
+    extend SafeCatch
+
     ALL = Bundler::Dependency::PLATFORM_MAP.values.uniq.freeze
 
     class SpecGroup < Array
@@ -125,7 +129,7 @@ module Bundler
       Bundler.ui.info "Resolving dependencies...", false
       base = SpecSet.new(base) unless base.is_a?(SpecSet)
       resolver = new(index, source_requirements, base)
-      result = catch(:success) do
+      result = safe_catch(:success) do
         resolver.start(requirements)
         raise resolver.version_conflict
         nil
@@ -168,10 +172,10 @@ module Bundler
       resolve(reqs, activated)
     end
 
-    def resolve(reqs, activated)
+    def resolve(reqs, activated, depth = 0)
       # If the requirements are empty, then we are in a success state. Aka, all
       # gem dependencies have been resolved.
-      throw :success, successify(activated) if reqs.empty?
+      safe_throw :success, successify(activated) if reqs.empty?
 
       indicate_progress
 
@@ -197,6 +201,8 @@ module Bundler
       # Pull off the first requirement so that we can resolve it
       current = reqs.shift
 
+      $stderr.puts "#{' ' * depth}#{current}" if ENV['DEBUG_RESOLVER_TREE']
+
       debug { "Attempting:\n  #{current}"}
 
       # Check if the gem has already been activated, if it has, we will make sure
@@ -228,7 +234,7 @@ module Bundler
             @gems_size[dep] ||= gems_size(dep)
           end
 
-          resolve(reqs, activated)
+          resolve(reqs, activated, depth + 1)
         else
           debug { "    * [FAIL] Already activated" }
           @errors[existing.name] = [existing, current]
@@ -248,7 +254,7 @@ module Bundler
           if parent && parent.name != 'bundler'
             debug { "    -> Jumping to: #{parent.name}" }
             required_by = existing.respond_to?(:required_by) && existing.required_by.last
-            throw parent.name, required_by && required_by.name
+            safe_throw parent.name, required_by && required_by.name
           else
             # The original set of dependencies conflict with the base set of specs
             # passed to the resolver. This is by definition an impossible resolve.
@@ -301,7 +307,7 @@ module Bundler
         end
 
         matching_versions.reverse_each do |spec_group|
-          conflict = resolve_requirement(spec_group, current, reqs.dup, activated.dup)
+          conflict = resolve_requirement(spec_group, current, reqs.dup, activated.dup, depth)
           conflicts << conflict if conflict
         end
 
@@ -315,7 +321,7 @@ module Bundler
               # Choose the closest pivot in the stack that will affect the conflict
               errorpivot = (@stack & [req_name, current.required_by.last.name]).last
               debug { "    -> Jumping to: #{errorpivot}" }
-              throw errorpivot, req_name
+              safe_throw errorpivot, req_name
             end
           end
         end
@@ -330,14 +336,14 @@ module Bundler
           @stack.reverse_each do |savepoint|
             if conflicts.include?(savepoint)
               debug { "    -> Jumping to: #{savepoint}" }
-              throw savepoint
+              safe_throw savepoint
             end
           end
         end
       end
     end
 
-    def resolve_requirement(spec_group, requirement, reqs, activated)
+    def resolve_requirement(spec_group, requirement, reqs, activated, depth)
       # We are going to try activating the spec. We need to keep track of stack of
       # requirements that got us to the point of activating this gem.
       spec_group.required_by.replace requirement.required_by
@@ -366,9 +372,9 @@ module Bundler
       # jump back to this point and try another version of the gem.
       length = @stack.length
       @stack << requirement.name
-      retval = catch(requirement.name) do
+      retval = safe_catch(requirement.name) do
         # try to resolve the next option
-        resolve(reqs, activated)
+        resolve(reqs, activated, depth)
       end
 
       # clear the search cache since the catch means we couldn't meet the

data/lib/bundler/rubygems_ext.rb

@@ -51,8 +51,8 @@ module Gem
     end
 
     def git_version
-      if loaded_from && File.exist?(File.join(full_gem_path, ".git"))
-        sha = Dir.chdir(full_gem_path){ `git rev-parse HEAD`.strip }
+      if @loaded_from && File.exist?(File.join(full_gem_path, ".git"))
+        sha = Bundler::SharedHelpers.chdir(full_gem_path){ `git rev-parse HEAD`.strip }
         " #{sha[0..6]}"
       end
     end

data/lib/bundler/rubygems_integration.rb

@@ -6,6 +6,22 @@ require 'rubygems/config_file'
 module Bundler
   class RubygemsIntegration
 
+    def self.version
+      @version ||= Gem::Version.new(Gem::VERSION)
+    end
+
+    def self.provides?(req_str)
+      Gem::Requirement.new(req_str).satisfied_by?(version)
+    end
+
+    def version
+      self.class.version
+    end
+
+    def provides?(req_str)
+      self.class.provides?(req_str)
+    end
+
     def build_args
       Gem::Command.build_args
     end
@@ -79,6 +95,14 @@ module Bundler
       Gem.path
     end
 
+    def spec_cache_dirs
+      @spec_cache_dirs ||= begin
+        dirs = gem_path.map {|dir| File.join(dir, 'specifications')}
+        dirs << Gem.spec_cache_dir if Gem.respond_to?(:spec_cache_dir) # Not in Rubygems 2.0.3 or earlier
+        dirs.uniq.select {|dir| File.directory? dir}
+      end
+    end
+
     def marshal_spec_dir
       Gem::MARSHAL_SPEC_DIR
     end
@@ -153,13 +177,13 @@ module Bundler
       end
     end
 
-    def build(spec, skip_validation = false)
+    def build(spec)
       require 'rubygems/builder'
       Gem::Builder.new(spec).build
     end
 
     def build_gem(gem_dir, spec)
-      Dir.chdir(gem_dir) { build(spec) }
+      SharedHelpers.chdir(gem_dir) { build(spec) }
     end
 
     def download_gem(spec, uri, path)
@@ -170,7 +194,7 @@ module Bundler
       @security_policies ||= begin
         require 'rubygems/security'
         Gem::Security::Policies
-      rescue LoadError, NameError
+      rescue LoadError
         {}
       end
     end
@@ -398,7 +422,7 @@ module Bundler
       end
     end
 
-    # Rubygems 1.8.5-1.8.19
+    # Rubygems ~> 1.8.5
     class Modern < RubygemsIntegration
       def stub_rubygems(specs)
         Gem::Specification.all = specs
@@ -431,24 +455,14 @@ module Bundler
       end
     end
 
-    # Rubygems 1.8.20+
-    class MoreModern < Modern
-      # Rubygems 1.8.20 and adds the skip_validation parameter, so that's
-      # when we start passing it through.
-      def build(spec, skip_validation = false)
-        require 'rubygems/builder'
-        Gem::Builder.new(spec).build(skip_validation)
-      end
-    end
-
     # Rubygems 2.0
     class Future < RubygemsIntegration
       def stub_rubygems(specs)
         Gem::Specification.all = specs
 
-        Gem.post_reset do
+        Gem.post_reset {
           Gem::Specification.all = specs
-        end
+        }
       end
 
       def all_specs
@@ -491,26 +505,24 @@ module Bundler
         return p
       end
 
-      def build(spec, skip_validation = false)
+      def build(spec)
         require 'rubygems/package'
-        Gem::Package.build(spec, skip_validation)
+        Gem::Package.build(spec)
       end
 
     end
 
   end
 
-  if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.99.99')
+  if RubygemsIntegration.provides?(">= 1.99.99")
     @rubygems = RubygemsIntegration::Future.new
-  elsif Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.8.20')
-    @rubygems = RubygemsIntegration::MoreModern.new
-  elsif Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.8.5')
+  elsif RubygemsIntegration.provides?('>= 1.8.5')
     @rubygems = RubygemsIntegration::Modern.new
-  elsif Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.8.0')
+  elsif RubygemsIntegration.provides?('>= 1.8.0')
     @rubygems = RubygemsIntegration::AlmostModern.new
-  elsif Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.7.0')
+  elsif RubygemsIntegration.provides?('>= 1.7.0')
     @rubygems = RubygemsIntegration::Transitional.new
-  elsif Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.4.0')
+  elsif RubygemsIntegration.provides?('>= 1.4.0')
     @rubygems = RubygemsIntegration::Legacy.new
   else # Rubygems 1.3.6 and 1.3.7
     @rubygems = RubygemsIntegration::Ancient.new