peritor-bundler 0.7.0

data/lib/bundler/finder.rb

@@ -0,0 +1,51 @@
+module Bundler
+  # Finder behaves like a rubygems source index in that it responds
+  # to #search. It also resolves a list of dependencies finding the
+  # best possible configuration of gems that satisifes all requirements
+  # without causing any gem activation errors.
+  class Finder
+
+    # Takes an array of gem sources and fetches the full index of
+    # gems from each one. It then combines the indexes together keeping
+    # track of the original source so that any resolved gem can be
+    # fetched from the correct source.
+    #
+    # ==== Parameters
+    # *sources<String>:: URI pointing to the gem repository
+    def initialize(*sources)
+      @cache   = {}
+      @index   = {}
+      @sources = sources
+    end
+
+    # Searches for a gem that matches the dependency
+    #
+    # ==== Parameters
+    # dependency<Gem::Dependency>:: The gem dependency to search for
+    #
+    # ==== Returns
+    # [Gem::Specification]:: A collection of gem specifications
+    #   matching the search
+    def search(dependency)
+      @cache[dependency.hash] ||= begin
+        find_by_name(dependency.name).select do |spec|
+          dependency =~ spec
+        end.sort_by {|s| s.version }
+      end
+    end
+
+  private
+
+    def find_by_name(name)
+      matches = @index[name] ||= begin
+        versions = {}
+        @sources.reverse_each do |source|
+          versions.merge! source.specs[name] || {}
+        end
+        versions
+      end
+      matches.values
+    end
+
+  end
+end

data/lib/bundler/gem_bundle.rb

@@ -0,0 +1,11 @@
+module Bundler
+  class GemBundle < Array
+    def download
+      sort_by {|s| s.full_name.downcase }.each do |spec|
+        spec.source.download(spec)
+      end
+
+      self
+    end
+  end
+end

data/lib/bundler/gem_ext.rb

@@ -0,0 +1,33 @@
+module Gem
+  class Installer
+    remove_method(:app_script_text) if method_defined?(:app_script_text)
+
+    def app_script_text(bin_file_name)
+      path = @gem_home
+      template = File.read(File.join(File.dirname(__FILE__), "templates", "app_script.erb"))
+      erb = ERB.new(template, nil, '-')
+      erb.result(binding)
+    end
+  end
+
+  class Specification
+    attr_accessor :source
+    attr_accessor :location
+
+    remove_method(:specification_version) if method_defined?(:specification_version)
+
+    # Hack to fix github's strange marshal file
+    def specification_version
+      @specification_version && @specification_version.to_i
+    end
+
+    alias full_gem_path_without_location full_gem_path
+    def full_gem_path
+      if defined?(@location) && @location
+        @location
+      else
+        full_gem_path_without_location
+      end
+    end
+  end
+end

data/lib/bundler/remote_specification.rb

@@ -0,0 +1,50 @@
+module Bundler
+  # Represents a lazily loaded gem specification, where the full specification
+  # is on the source server in rubygems' "quick" index. The proxy object is to
+  # be seeded with what we're given from the source's abbreviated index - the
+  # full specification will only be fetched when necesary.
+  class RemoteSpecification
+    attr_reader :name, :version, :platform
+    attr_accessor :source
+
+    def initialize(name, version, platform, source_uri)
+      @name     = name
+      @version  = version
+      @platform = platform
+      @source_uri = source_uri
+    end
+
+    def full_name
+      if platform == Gem::Platform::RUBY or platform.nil? then
+        "#{@name}-#{@version}"
+      else
+        "#{@name}-#{@version}-#{platform}"
+      end
+    end
+
+    # Because Rubyforge cannot be trusted to provide valid specifications
+    # once the remote gem is donwloaded, the backend specification will
+    # be swapped out.
+    def __swap__(spec)
+      @specification = spec
+    end
+
+    private
+
+    def _remote_uri
+      "#{@source_uri}/quick/Marshal.4.8/#{@name}-#{@version}.gemspec.rz"
+    end
+
+    def _remote_specification
+      @specification ||= begin
+        deflated = Gem::RemoteFetcher.fetcher.fetch_path(_remote_uri)
+        inflated = Gem.inflate(deflated)
+        Marshal.load(inflated)
+      end
+    end
+
+    def method_missing(method, *args, &blk)
+      _remote_specification.send(method, *args, &blk)
+    end
+  end
+end

data/lib/bundler/repository.rb

@@ -0,0 +1,256 @@
+module Bundler
+  class InvalidRepository < StandardError ; end
+
+  class Repository
+    attr_reader :path
+
+    def initialize(path, bindir)
+      FileUtils.mkdir_p(path)
+
+      @path   = Pathname.new(path)
+      @bindir = Pathname.new(bindir)
+
+      @cache = GemDirectorySource.new(:location => @path.join("cache"))
+    end
+
+    def install(dependencies, sources, options = {})
+      # TODO: clean this up
+      sources.each do |s|
+        s.repository = self
+        s.local = options[:cached]
+      end
+
+      source_requirements = {}
+      dependencies = dependencies.map do |dep|
+        source_requirements[dep.name] = dep.source if dep.source
+        dep.to_gem_dependency
+      end
+
+      # Check to see whether the existing cache meets all the requirements
+      begin
+        valid = nil
+        # valid = Resolver.resolve(dependencies, [source_index], source_requirements)
+      rescue Bundler::GemNotFound
+      end
+
+      sources = only_local(sources) if options[:cached]
+
+      # Check the remote sources if the existing cache does not meet the requirements
+      # or the user passed --update
+      if options[:update] || !valid
+        Bundler.logger.info "Calculating dependencies..."
+        bundle = Resolver.resolve(dependencies, [@cache] + sources, source_requirements)
+        download(bundle, options)
+        do_install(bundle, options)
+        valid = bundle
+      end
+
+      generate_bins(valid, options)
+      cleanup(valid, options)
+      configure(valid, options)
+    end
+
+    def cache(*gemfiles)
+      FileUtils.mkdir_p(@path.join("cache"))
+      gemfiles.each do |gemfile|
+        Bundler.logger.info "Caching: #{File.basename(gemfile)}"
+        FileUtils.cp(gemfile, @path.join("cache"))
+      end
+    end
+
+    def prune(dependencies, sources)
+      sources.each do |s|
+        s.repository = self
+        s.local = true
+      end
+
+      sources = only_local(sources)
+      bundle = Resolver.resolve(dependencies, [@cache] + sources)
+      @cache.gems.each do |name, specs|
+        specs.each do |spec|
+          unless bundle.any? { |s| s.name == spec.name && s.version == spec.version }
+            Bundler.logger.info "Pruning #{spec.name} (#{spec.version}) from the cache"
+            FileUtils.rm @path.join("cache", "#{spec.full_name}.gem")
+          end
+        end
+      end
+    end
+
+    def gems
+      source_index.gems.values
+    end
+
+    def outdated_gems
+      source_index.outdated.sort
+    end
+
+    def source_index
+      index = Gem::SourceIndex.from_gems_in(@path.join("specifications"))
+      index.each { |n, spec| spec.loaded_from = @path.join("specifications", "#{spec.full_name}.gemspec") }
+      index
+    end
+
+    def download_path_for(type)
+      @repos[type].download_path_for
+    end
+
+  private
+
+    def only_local(sources)
+      sources.select { |s| s.can_be_local? }
+    end
+
+    def download(bundle, options)
+      bundle.sort_by {|s| s.full_name.downcase }.each do |spec|
+        next if options[:no_bundle].include?(spec.name)
+        spec.source.download(spec)
+      end
+    end
+
+    def do_install(bundle, options)
+      bundle.each do |spec|
+        next if options[:no_bundle].include?(spec.name)
+        spec.loaded_from = @path.join("specifications", "#{spec.full_name}.gemspec")
+        # Do nothing if the gem is already expanded
+        next if @path.join("gems", spec.full_name).directory?
+
+        case spec.source
+        when GemSource, GemDirectorySource, SystemGemSource
+          expand_gemfile(spec, options)
+        else
+          expand_vendored_gem(spec, options)
+        end
+      end
+    end
+
+    def generate_bins(bundle, options)
+      bundle.each do |spec|
+        next if options[:no_bundle].include?(spec.name)
+        # HAX -- Generate the bin
+        bin_dir = @bindir
+        path    = @path
+        installer = Gem::Installer.allocate
+        installer.instance_eval do
+          @spec     = spec
+          @bin_dir  = bin_dir
+          @gem_dir  = path.join("gems", "#{spec.full_name}")
+          @gem_home = path
+          @wrappers = true
+          @format_executable = false
+          @env_shebang = false
+        end
+        installer.generate_bin
+      end
+    end
+
+    def expand_gemfile(spec, options)
+      Bundler.logger.info "Installing #{spec.name} (#{spec.version})"
+
+      gemfile = @path.join("cache", "#{spec.full_name}.gem").to_s
+
+      if build_args = options[:build_options] && options[:build_options][spec.name]
+        Gem::Command.build_args = build_args.map {|k,v| "--with-#{k}=#{v}"}
+      end
+
+      installer = Gem::Installer.new(gemfile, options.merge(
+        :install_dir         => @path,
+        :ignore_dependencies => true,
+        :env_shebang         => true,
+        :wrappers            => true,
+        :bin_dir             => @bindir
+      ))
+      installer.install
+    ensure
+      Gem::Command.build_args = []
+    end
+
+    def expand_vendored_gem(spec, options)
+      add_spec(spec)
+      FileUtils.mkdir_p(@path.join("gems"))
+      File.symlink(spec.location, @path.join("gems", spec.full_name))
+    end
+
+    def add_spec(spec)
+      destination = path.join('specifications')
+      destination.mkdir unless destination.exist?
+
+      File.open(destination.join("#{spec.full_name}.gemspec"), 'w') do |f|
+        f.puts spec.to_ruby
+      end
+    end
+
+    def cleanup(valid, options)
+      to_delete = gems
+      to_delete.delete_if do |spec|
+        valid.any? { |other| spec.name == other.name && spec.version == other.version }
+      end
+
+      valid_executables = valid.map { |s| s.executables }.flatten.compact
+
+      to_delete.each do |spec|
+        Bundler.logger.info "Deleting gem: #{spec.name} (#{spec.version})"
+        FileUtils.rm_rf(@path.join("specifications", "#{spec.full_name}.gemspec"))
+        FileUtils.rm_rf(@path.join("gems", spec.full_name))
+        # Cleanup the bin directory
+        spec.executables.each do |bin|
+          next if valid_executables.include?(bin)
+          Bundler.logger.info "Deleting bin file: #{bin}"
+          FileUtils.rm_rf(@bindir.join(bin))
+        end
+      end
+    end
+
+    def expand(options)
+      each_repo do |repo|
+        repo.expand(options)
+      end
+    end
+
+    def configure(specs, options)
+      generate_environment(specs, options)
+    end
+
+    def generate_environment(specs, options)
+      FileUtils.mkdir_p(path)
+
+      load_paths = load_paths_for_specs(specs, options)
+      bindir     = @bindir.relative_path_from(path).to_s
+      filename   = options[:manifest].relative_path_from(path).to_s
+
+      File.open(path.join("environment.rb"), "w") do |file|
+        template = File.read(File.join(File.dirname(__FILE__), "templates", "environment.erb"))
+        erb = ERB.new(template, nil, '-')
+        file.puts erb.result(binding)
+      end
+    end
+
+    def load_paths_for_specs(specs, options)
+      load_paths = []
+      specs.each do |spec|
+        next if options[:no_bundle].include?(spec.name)
+        gem_path = Pathname.new(spec.full_gem_path)
+        load_paths << load_path_for(gem_path, spec.bindir) if spec.bindir
+        spec.require_paths.each do |path|
+          load_paths << load_path_for(gem_path, path)
+        end
+      end
+      load_paths
+    end
+
+    def load_path_for(gem_path, path)
+      gem_path.join(path).relative_path_from(@path).to_s
+    end
+
+    def spec_file_for(spec)
+      spec.loaded_from.relative_path_from(@path).to_s
+    end
+
+    def require_code(file, dep)
+      constraint = case
+      when dep.only   then %{ if #{dep.only.inspect}.include?(env)}
+      when dep.except then %{ unless #{dep.except.inspect}.include?(env)}
+      end
+      "require #{file.inspect}#{constraint}"
+    end
+  end
+end

data/lib/bundler/resolver.rb

@@ -0,0 +1,234 @@
+# This is the latest iteration of the gem dependency resolving algorithm. As of now,
+# it can resolve (as a success of failure) any set of gem dependencies we throw at it
+# in a reasonable amount of time. The most iterations I've seen it take is about 150.
+# The actual implementation of the algorithm is not as good as it could be yet, but that
+# can come later.
+
+# Extending Gem classes to add necessary tracking information
+module Gem
+  class Dependency
+    def required_by
+      @required_by ||= []
+    end
+  end
+  class Specification
+    def required_by
+      @required_by ||= []
+    end
+  end
+end
+
+module Bundler
+  class GemNotFound < StandardError; end
+  class VersionConflict < StandardError; end
+
+  class Resolver
+
+    attr_reader :errors
+
+    # Figures out the best possible configuration of gems that satisfies
+    # the list of passed dependencies and any child dependencies without
+    # causing any gem activation errors.
+    #
+    # ==== Parameters
+    # *dependencies<Gem::Dependency>:: The list of dependencies to resolve
+    #
+    # ==== Returns
+    # <GemBundle>,nil:: If the list of dependencies can be resolved, a
+    #   collection of gemspecs is returned. Otherwise, nil is returned.
+    def self.resolve(requirements, sources, source_requirements = {})
+      resolver = new(sources, source_requirements)
+      result = catch(:success) do
+        resolver.resolve(requirements, {})
+        output = resolver.errors.inject("") do |o, (conflict, (origin, requirement))|
+          o << "  Conflict on: #{conflict.inspect}:\n"
+          o << "    * #{conflict} (#{origin.version}) activated by #{origin.required_by.first}\n"
+          o << "    * #{requirement} required by #{requirement.required_by.first}\n"
+          o << "    All possible versions of origin requirements conflict."
+        end
+        raise VersionConflict, "No compatible versions could be found for required dependencies:\n  #{output}"
+        nil
+      end
+      if result
+        # Order gems in order of dependencies. Every gem's dependency is at
+        # a smaller index in the array.
+        ordered = []
+        result.values.each do |spec1|
+          index = nil
+          place = ordered.detect do |spec2|
+            spec1.dependencies.any? { |d| d.name == spec2.name }
+          end
+          place ?
+            ordered.insert(ordered.index(place), spec1) :
+            ordered << spec1
+        end
+        ordered.reverse
+      end
+    end
+
+    def initialize(sources, source_requirements)
+      @errors = {}
+      @stack  = []
+      @specs  = Hash.new { |h,k| h[k] = [] }
+      @by_gem = source_requirements
+      @cache  = {}
+      @index  = {}
+
+      sources.each do |source|
+        source.gems.each do |name, specs|
+          # Hack to work with a regular Gem::SourceIndex
+          [specs].flatten.compact.each do |spec|
+            next if @specs[spec.name].any? { |s| s.version == spec.version }
+            @specs[spec.name] << spec
+          end
+        end
+      end
+    end
+
+    def debug
+      puts yield if defined?($debug) && $debug
+    end
+
+    def resolve(reqs, activated)
+      # If the requirements are empty, then we are in a success state. Aka, all
+      # gem dependencies have been resolved.
+      throw :success, activated if reqs.empty?
+
+      debug { STDIN.gets ; print "\e[2J\e[f" ; "==== Iterating ====\n\n" }
+
+      # Sort dependencies so that the ones that are easiest to resolve are first.
+      # Easiest to resolve is defined by:
+      #   1) Is this gem already activated?
+      #   2) Do the version requirements include prereleased gems?
+      #   3) Sort by number of gems available in the source.
+      reqs = reqs.sort_by do |a|
+        [ activated[a.name] ? 0 : 1,
+          a.version_requirements.prerelease? ? 0 : 1,
+          @errors[a.name]   ? 0 : 1,
+          activated[a.name] ? 0 : search(a).size ]
+      end
+
+      debug { "Activated:\n" + activated.values.map { |a| "  #{a.name} (#{a.version})" }.join("\n") }
+      debug { "Requirements:\n" + reqs.map { |r| "  #{r.name} (#{r.version_requirements})"}.join("\n") }
+
+      activated = activated.dup
+      # Pull off the first requirement so that we can resolve it
+      current   = reqs.shift
+
+      debug { "Attempting:\n  #{current.name} (#{current.version_requirements})"}
+
+      # Check if the gem has already been activated, if it has, we will make sure
+      # that the currently activated gem satisfies the requirement.
+      if existing = activated[current.name]
+        if current.version_requirements.satisfied_by?(existing.version)
+          debug { "    * [SUCCESS] Already activated" }
+          @errors.delete(existing.name)
+          # Since the current requirement is satisfied, we can continue resolving
+          # the remaining requirements.
+          resolve(reqs, activated)
+        else
+          debug { "    * [FAIL] Already activated" }
+          @errors[existing.name] = [existing, current]
+          debug { current.required_by.map {|d| "      * #{d.name} (#{d.version_requirements})" }.join("\n") }
+          # debug { "    * All current conflicts:\n" + @errors.keys.map { |c| "      - #{c}" }.join("\n") }
+          # Since the current requirement conflicts with an activated gem, we need
+          # to backtrack to the current requirement's parent and try another version
+          # of it (maybe the current requirement won't be present anymore). If the
+          # current requirement is a root level requirement, we need to jump back to
+          # where the conflicting gem was activated.
+          parent = current.required_by.last || existing.required_by.last
+          # We track the spot where the current gem was activated because we need
+          # to keep a list of every spot a failure happened.
+          debug { "    -> Jumping to: #{parent.name}" }
+          throw parent.name, existing.required_by.last.name
+        end
+      else
+        # There are no activated gems for the current requirement, so we are going
+        # to find all gems that match the current requirement and try them in decending
+        # order. We also need to keep a set of all conflicts that happen while trying
+        # this gem. This is so that if no versions work, we can figure out the best
+        # place to backtrack to.
+        conflicts = Set.new
+
+        # Fetch all gem versions matching the requirement
+        #
+        # TODO: Warn / error when no matching versions are found.
+        matching_versions = search(current)
+
+        if matching_versions.empty?
+          if current.required_by.empty?
+            raise GemNotFound, "Could not find gem '#{current}' in any of the sources"
+          end
+          Bundler.logger.warn "Could not find gem '#{current}' (required by '#{current.required_by.last}') in any of the sources"
+        end
+
+        matching_versions.reverse_each do |spec|
+          conflict = resolve_requirement(spec, current, reqs.dup, activated.dup)
+          conflicts << conflict if conflict
+        end
+        # If the current requirement is a root level gem and we have conflicts, we
+        # can figure out the best spot to backtrack to.
+        if current.required_by.empty? && !conflicts.empty?
+          # Check the current "catch" stack for the first one that is included in the
+          # conflicts set. That is where the parent of the conflicting gem was required.
+          # By jumping back to this spot, we can try other version of the parent of
+          # the conflicting gem, hopefully finding a combination that activates correctly.
+          @stack.reverse_each do |savepoint|
+            if conflicts.include?(savepoint)
+              debug { "    -> Jumping to: #{savepoint}" }
+              throw savepoint
+            end
+          end
+        end
+      end
+    end
+
+    def resolve_requirement(spec, requirement, reqs, activated)
+      # We are going to try activating the spec. We need to keep track of stack of
+      # requirements that got us to the point of activating this gem.
+      spec.required_by.replace requirement.required_by
+      spec.required_by << requirement
+
+      activated[spec.name] = spec
+      debug { "  Activating: #{spec.name} (#{spec.version})" }
+      debug { spec.required_by.map { |d| "    * #{d.name} (#{d.version_requirements})" }.join("\n") }
+
+      # Now, we have to loop through all child dependencies and add them to our
+      # array of requirements.
+      debug { "    Dependencies"}
+      spec.dependencies.each do |dep|
+        next if dep.type == :development
+        debug { "    * #{dep.name} (#{dep.version_requirements})" }
+        dep.required_by.replace(requirement.required_by)
+        dep.required_by << requirement
+        reqs << dep
+      end
+
+      # We create a savepoint and mark it by the name of the requirement that caused
+      # the gem to be activated. If the activated gem ever conflicts, we are able to
+      # jump back to this point and try another version of the gem.
+      length = @stack.length
+      @stack << requirement.name
+      retval = catch(requirement.name) do
+        resolve(reqs, activated)
+      end
+      # Since we're doing a lot of throw / catches. A push does not necessarily match
+      # up to a pop. So, we simply slice the stack back to what it was before the catch
+      # block.
+      @stack.slice!(length..-1)
+      retval
+    end
+
+    def search(dependency)
+      @cache[dependency.hash] ||= begin
+        collection = @by_gem[dependency.name].gems if @by_gem[dependency.name]
+        collection ||= @specs
+        collection[dependency.name].select do |spec|
+          match = dependency =~ spec
+          match &= dependency.version_requirements.prerelease? if spec.version.prerelease?
+          match
+        end.sort_by {|s| s.version }
+      end
+    end
+  end
+end