rubygems-update 1.1.1 → 1.2.0

data/lib/rubygems/dependency_installer.rb

@@ -1,9 +1,12 @@
 require 'rubygems'
 require 'rubygems/dependency_list'
 require 'rubygems/installer'
-require 'rubygems/source_info_cache'
+require 'rubygems/spec_fetcher'
 require 'rubygems/user_interaction'
 
+##
+# Installs a gem along with all its dependencies from local and remote gems.
+
 class Gem::DependencyInstaller
 
   include Gem::UserInteraction
@@ -25,36 +28,50 @@ class Gem::DependencyInstaller
   # Creates a new installer instance.
   #
   # Options are:
-  # :env_shebang:: See Gem::Installer::new.
+  # :cache_dir:: Alternate repository path to store .gem files in.
   # :domain:: :local, :remote, or :both.  :local only searches gems in the
   #           current directory.  :remote searches only gems in Gem::sources.
   #           :both searches both.
+  # :env_shebang:: See Gem::Installer::new.
   # :force:: See Gem::Installer#install.
   # :format_executable:: See Gem::Installer#initialize.
-  # :ignore_dependencies: Don't install any dependencies.
-  # :install_dir: See Gem::Installer#install.
-  # :security_policy: See Gem::Installer::new and Gem::Security.
-  # :wrappers: See Gem::Installer::new
+  # :ignore_dependencies:: Don't install any dependencies.
+  # :install_dir:: See Gem::Installer#install.
+  # :security_policy:: See Gem::Installer::new and Gem::Security.
+  # :wrappers:: See Gem::Installer::new
+
   def initialize(options = {})
     options = DEFAULT_OPTIONS.merge options
-    @env_shebang = options[:env_shebang]
+
+    @bin_dir = options[:bin_dir]
+    @development = options[:development]
     @domain = options[:domain]
+    @env_shebang = options[:env_shebang]
     @force = options[:force]
     @format_executable = options[:format_executable]
     @ignore_dependencies = options[:ignore_dependencies]
-    @install_dir = options[:install_dir] || Gem.dir
     @security_policy = options[:security_policy]
     @wrappers = options[:wrappers]
-    @bin_dir = options[:bin_dir]
 
     @installed_gems = []
+
+    @install_dir = options[:install_dir] || Gem.dir
+    @cache_dir = options[:cache_dir] || @install_dir
+
+    if options[:install_dir] then
+      spec_dir = File.join @install_dir, 'specifications'
+      @source_index = Gem::SourceIndex.from_gems_in spec_dir
+    else
+      @source_index = Gem.source_index
+    end
   end
 
   ##
   # Returns a list of pairs of gemspecs and source_uris that match
   # Gem::Dependency +dep+ from both local (Dir.pwd) and remote (Gem.sources)
   # sources.  Gems are sorted with newer gems prefered over older gems, and
-  # local gems prefered over remote gems.
+  # local gems preferred over remote gems.
+
   def find_gems_with_sources(dep)
     gems_and_sources = []
 
@@ -74,8 +91,7 @@ class Gem::DependencyInstaller
         all = requirements.length > 1 ||
                 (requirements.first != ">=" and requirements.first != ">")
 
-        found = Gem::SourceInfoCache.search_with_source dep, true, all
-
+        found = Gem::SpecFetcher.fetcher.fetch dep, all
         gems_and_sources.push(*found)
 
       rescue Gem::RemoteFetcher::FetchError => e
@@ -95,6 +111,7 @@ class Gem::DependencyInstaller
   ##
   # Gathers all dependencies necessary for the installation from local and
   # remote sources unless the ignore_dependencies was given.
+
   def gather_dependencies
     specs = @specs_and_sources.map { |spec,_| spec }
 
@@ -110,14 +127,25 @@ class Gem::DependencyInstaller
         next if spec.nil? or seen[spec.name]
         seen[spec.name] = true
 
-        spec.dependencies.each do |dep|
-          results = find_gems_with_sources(dep).reverse # local gems first
+        deps = spec.runtime_dependencies
+        deps |= spec.development_dependencies if @development
+
+        deps.each do |dep|
+          results = find_gems_with_sources(dep).reverse
+
+          results.reject! do |dep_spec,|
+            to_do.push dep_spec
+
+            @source_index.any? do |_, installed_spec|
+              dep.name == installed_spec.name and
+                dep.version_requirements.satisfied_by? installed_spec.version
+            end
+          end
 
           results.each do |dep_spec, source_uri|
             next if seen[dep_spec.name]
             @specs_and_sources << [dep_spec, source_uri]
             dependency_list.add dep_spec
-            to_do.push dep_spec
           end
         end
       end
@@ -126,6 +154,11 @@ class Gem::DependencyInstaller
     @gems_to_install = dependency_list.dependency_order.reverse
   end
 
+  ##
+  # Finds a spec and the source_uri it came from for gem +gem_name+ and
+  # +version+.  Returns an Array of specs and sources required for
+  # installation of the gem.
+
   def find_spec_by_name_and_version gem_name, version = Gem::Requirement.default
     spec_and_source = nil
 
@@ -160,14 +193,16 @@ class Gem::DependencyInstaller
 
     if spec_and_source.nil? then
       raise Gem::GemNotFoundException,
-        "could not find #{gem_name} locally or in a repository"
+        "could not find gem #{gem_name} locally or in a repository"
     end
 
     @specs_and_sources = [spec_and_source]
   end
 
   ##
-  # Installs the gem and all its dependencies.
+  # Installs the gem and all its dependencies.  Returns an Array of installed
+  # gems specifications.
+
   def install dep_or_name, version = Gem::Requirement.default
     if String === dep_or_name then
       find_spec_by_name_and_version dep_or_name, version
@@ -175,15 +210,14 @@ class Gem::DependencyInstaller
       @specs_and_sources = [find_gems_with_sources(dep_or_name).last]
     end
 
-    gather_dependencies
+    @installed_gems = []
 
-    spec_dir = File.join @install_dir, 'specifications'
-    source_index = Gem::SourceIndex.from_gems_in spec_dir
+    gather_dependencies
 
     @gems_to_install.each do |spec|
       last = spec == @gems_to_install.last
       # HACK is this test for full_name acceptable?
-      next if source_index.any? { |n,_| n == spec.full_name } and not last
+      next if @source_index.any? { |n,_| n == spec.full_name } and not last
 
       # TODO: make this sorta_verbose so other users can benefit from it
       say "Installing gem #{spec.full_name}" if Gem.configuration.really_verbose
@@ -191,7 +225,7 @@ class Gem::DependencyInstaller
       _, source_uri = @specs_and_sources.assoc spec
       begin
         local_gem_path = Gem::RemoteFetcher.fetcher.download spec, source_uri,
-                                                             @install_dir
+                                                             @cache_dir
       rescue Gem::RemoteFetcher::FetchError
         next if @force
         raise
@@ -205,12 +239,15 @@ class Gem::DependencyInstaller
                                 :install_dir => @install_dir,
                                 :security_policy => @security_policy,
                                 :wrappers => @wrappers,
-                                :bin_dir => @bin_dir
+                                :bin_dir => @bin_dir,
+                                :development => @development
 
       spec = inst.install
 
       @installed_gems << spec
     end
+
+    @installed_gems
   end
 
 end

data/lib/rubygems/dependency_list.rb

@@ -69,7 +69,7 @@ class Gem::DependencyList
   # Are all the dependencies in the list satisfied?
   def ok?
     @specs.all? do |spec|
-      spec.dependencies.all? do |dep|
+      spec.runtime_dependencies.all? do |dep|
         @specs.find { |s| s.satisfies_requirement? dep }
       end
     end

data/lib/rubygems/doc_manager.rb

@@ -9,9 +9,9 @@ require 'fileutils'
 module Gem
 
   class DocManager
-  
+
     include UserInteraction
-  
+
     # Create a document manager for the given gem spec.
     #
     # spec::      The Gem::Specification object representing the gem.
@@ -22,12 +22,12 @@ module Gem
       @doc_dir = File.join(spec.installation_path, "doc", spec.full_name)
       @rdoc_args = rdoc_args.nil? ? [] : rdoc_args.split
     end
-    
+
     # Is the RDoc documentation installed?
     def rdoc_installed?
       return File.exist?(File.join(@doc_dir, "rdoc"))
     end
-    
+
     # Generate the RI documents for this gem spec.
     #
     # Note that if both RI and RDoc documents are generated from the
@@ -102,7 +102,7 @@ module Gem
       args << '--quiet'
       args << @spec.require_paths.clone
       args << @spec.extra_rdoc_files
-      args.flatten!
+      args = args.flatten.map do |arg| arg.to_s end
 
       r = RDoc::RDoc.new
 

data/lib/rubygems/gem_openssl.rb

@@ -17,7 +17,7 @@ module Gem
       @ssl_available
     end
 
-    # Set the value of the ssl_avilable flag.
+    # Set the value of the ssl_available flag.
     attr_writer :ssl_available
 
     # Ensure that SSL is available.  Throw an exception if it is not.

data/lib/rubygems/indexer.rb

@@ -1,5 +1,6 @@
 require 'fileutils'
 require 'tmpdir'
+require 'zlib'
 
 require 'rubygems'
 require 'rubygems/format'
@@ -40,116 +41,304 @@ class Gem::Indexer
 
     marshal_name = "Marshal.#{Gem.marshal_version}"
 
-    @master_index = Gem::Indexer::MasterIndexBuilder.new "yaml", @directory
-    @marshal_index = Gem::Indexer::MarshalIndexBuilder.new marshal_name, @directory
-    @quick_index = Gem::Indexer::QuickIndexBuilder.new 'index', @directory
+    @master_index = File.join @directory, 'yaml'
+    @marshal_index = File.join @directory, marshal_name
 
-    quick_dir = File.join @directory, 'quick'
-    @latest_index = Gem::Indexer::LatestIndexBuilder.new 'latest_index', quick_dir
+    @quick_dir = File.join @directory, 'quick'
+
+    @quick_marshal_dir = File.join @quick_dir, marshal_name
+
+    @quick_index = File.join @quick_dir, 'index'
+    @latest_index = File.join @quick_dir, 'latest_index'
+
+    @specs_index = File.join @directory, "specs.#{Gem.marshal_version}"
+    @latest_specs_index = File.join @directory,
+                                    "latest_specs.#{Gem.marshal_version}"
+
+    files = [
+      @specs_index,
+      "#{@specs_index}.gz",
+      @latest_specs_index,
+      "#{@latest_specs_index}.gz",
+      @quick_dir,
+      @master_index,
+      "#{@master_index}.Z",
+      @marshal_index,
+      "#{@marshal_index}.Z",
+    ]
+
+    @files = files.map do |path|
+      path.sub @directory, ''
+    end
+  end
+
+  ##
+  # Abbreviate the spec for downloading.  Abbreviated specs are only used for
+  # searching, downloading and related activities and do not need deployment
+  # specific information (e.g. list of files).  So we abbreviate the spec,
+  # making it much smaller for quicker downloads.
+
+  def abbreviate(spec)
+    spec.files = []
+    spec.test_files = []
+    spec.rdoc_options = []
+    spec.extra_rdoc_files = []
+    spec.cert_chain = []
+    spec
+  end
+
+  ##
+  # Build various indicies
+
+  def build_indicies(index)
+    progress = ui.progress_reporter index.size,
+                                    "Generating quick index gemspecs for #{index.size} gems",
+                                    "Complete"
+
+    index.each do |original_name, spec|
+      spec_file_name = "#{original_name}.gemspec.rz"
+      yaml_name = File.join @quick_dir, spec_file_name
+      marshal_name = File.join @quick_marshal_dir, spec_file_name
+
+      yaml_zipped = Gem.deflate spec.to_yaml
+      open yaml_name, 'wb' do |io| io.write yaml_zipped end
+
+      marshal_zipped = Gem.deflate Marshal.dump(spec)
+      open marshal_name, 'wb' do |io| io.write marshal_zipped end
+
+      progress.updated original_name
+    end
+
+    progress.done
+
+    say "Generating specs index"
+
+    open @specs_index, 'wb' do |io|
+      specs = index.sort.map do |_, spec|
+        platform = spec.original_platform
+        platform = Gem::Platform::RUBY if platform.nil?
+        [spec.name, spec.version, platform]
+      end
+
+      specs = compact_specs specs
+
+      Marshal.dump specs, io
+    end
+
+    say "Generating latest specs index"
+
+    open @latest_specs_index, 'wb' do |io|
+      specs = index.latest_specs.sort.map do |spec|
+        [spec.name, spec.version, spec.original_platform]
+      end
+
+      specs = compact_specs specs
+
+      Marshal.dump specs, io
+    end
+
+    say "Generating quick index"
+
+    quick_index = File.join @quick_dir, 'index'
+    open quick_index, 'wb' do |io|
+      io.puts index.sort.map { |_, spec| spec.original_name }
+    end
+
+    say "Generating latest index"
+
+    latest_index = File.join @quick_dir, 'latest_index'
+    open latest_index, 'wb' do |io|
+      io.puts index.latest_specs.sort.map { |spec| spec.original_name }
+    end
+
+    say "Generating Marshal master index"
+
+    open @marshal_index, 'wb' do |io|
+      io.write index.dump
+    end
+
+    progress = ui.progress_reporter index.size,
+                                    "Generating YAML master index for #{index.size} gems (this may take a while)",
+                                    "Complete"
+
+    open @master_index, 'wb' do |io|
+      io.puts "--- !ruby/object:#{index.class}"
+      io.puts "gems:"
+
+      gems = index.sort_by { |name, gemspec| gemspec.sort_obj }
+      gems.each do |original_name, gemspec|
+        yaml = gemspec.to_yaml.gsub(/^/, '    ')
+        yaml = yaml.sub(/\A    ---/, '') # there's a needed extra ' ' here
+        io.print "  #{original_name}:"
+        io.puts yaml
+
+        progress.updated original_name
+      end
+    end
+
+    progress.done
+
+    say "Compressing indicies"
+    # use gzip for future files.
+
+    compress quick_index, 'rz'
+    paranoid quick_index, 'rz'
+
+    compress latest_index, 'rz'
+    paranoid latest_index, 'rz'
+
+    compress @marshal_index, 'Z'
+    paranoid @marshal_index, 'Z'
+
+    compress @master_index, 'Z'
+    paranoid @master_index, 'Z'
+
+    gzip @specs_index
+    gzip @latest_specs_index
   end
 
   ##
-  # Build the index.
-
-  def build_index
-    @master_index.build do
-      @quick_index.build do
-        @marshal_index.build do
-          @latest_index.build do
-            progress = ui.progress_reporter gem_file_list.size,
-                                          "Generating index for #{gem_file_list.size} gems in #{@dest_directory}",
-                                          "Loaded all gems"
-
-                                          gem_file_list.each do |gemfile|
-              if File.size(gemfile.to_s) == 0 then
-                alert_warning "Skipping zero-length gem: #{gemfile}"
-                next
-              end
-
-              begin
-                spec = Gem::Format.from_file_by_path(gemfile).spec
-
-                unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then
-                  alert_warning "Skipping misnamed gem: #{gemfile} => #{spec.full_name} (#{spec.original_name})"
-                  next
-                end
-
-                abbreviate spec
-                sanitize spec
-
-                @master_index.add spec
-                @quick_index.add spec
-                @marshal_index.add spec
-                @latest_index.add spec
-
-                progress.updated spec.original_name
-
-              rescue SignalException => e
-                alert_error "Received signal, exiting"
-                raise
-              rescue Exception => e
-                alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}"
-              end
-                                          end
-
-            progress.done
-
-            say "Generating master indexes (this may take a while)"
-          end
+  # Collect specifications from .gem files from the gem directory.
+
+  def collect_specs
+    index = Gem::SourceIndex.new
+
+    progress = ui.progress_reporter gem_file_list.size,
+                                    "Loading #{gem_file_list.size} gems from #{@dest_directory}",
+                                    "Loaded all gems"
+
+    gem_file_list.each do |gemfile|
+      if File.size(gemfile.to_s) == 0 then
+        alert_warning "Skipping zero-length gem: #{gemfile}"
+        next
+      end
+
+      begin
+        spec = Gem::Format.from_file_by_path(gemfile).spec
+
+        unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then
+          alert_warning "Skipping misnamed gem: #{gemfile} => #{spec.full_name} (#{spec.original_name})"
+          next
         end
+
+        abbreviate spec
+        sanitize spec
+
+        index.gems[spec.original_name] = spec
+
+        progress.updated spec.original_name
+
+      rescue SignalException => e
+        alert_error "Received signal, exiting"
+        raise
+      rescue Exception => e
+        alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}"
       end
     end
+
+    progress.done
+
+    index
   end
 
-  def install_index
-    verbose = Gem.configuration.really_verbose
+  ##
+  # Compacts Marshal output for the specs index data source by using identical
+  # objects as much as possible.
 
-    say "Moving index into production dir #{@dest_directory}" if verbose
+  def compact_specs(specs)
+    names = {}
+    versions = {}
+    platforms = {}
 
-    files = @master_index.files + @quick_index.files + @marshal_index.files +
-            @latest_index.files
+    specs.map do |(name, version, platform)|
+      names[name] = name unless names.include? name
+      versions[version] = version unless versions.include? version
+      platforms[platform] = platform unless platforms.include? platform
 
-    files.each do |file|
-      src_name = File.join @directory, file
-      dst_name = File.join @dest_directory, file
+      [names[name], versions[version], platforms[platform]]
+    end
+  end
 
-      FileUtils.rm_rf dst_name, :verbose => verbose
-      FileUtils.mv src_name, @dest_directory, :verbose => verbose
+  ##
+  # Compress +filename+ with +extension+.
+
+  def compress(filename, extension)
+    data = Gem.read_binary filename
+
+    zipped = Gem.deflate data
+
+    open "#{filename}.#{extension}", 'wb' do |io|
+      io.write zipped
     end
   end
 
+  ##
+  # List of gem file names to index.
+
+  def gem_file_list
+    Dir.glob(File.join(@dest_directory, "gems", "*.gem"))
+  end
+
+  ##
+  # Builds and installs indexicies.
+
   def generate_index
     FileUtils.rm_rf @directory
     FileUtils.mkdir_p @directory, :mode => 0700
+    FileUtils.mkdir_p @quick_marshal_dir
 
-    build_index
-    install_index
+    index = collect_specs
+    build_indicies index
+    install_indicies
   rescue SignalException
   ensure
     FileUtils.rm_rf @directory
   end
 
-  # List of gem file names to index.
-  def gem_file_list
-    Dir.glob(File.join(@dest_directory, "gems", "*.gem"))
+   ##
+  # Zlib::GzipWriter wrapper that gzips +filename+ on disk.
+
+  def gzip(filename)
+    Zlib::GzipWriter.open "#{filename}.gz" do |io|
+      io.write Gem.read_binary(filename)
+    end
   end
 
-  # Abbreviate the spec for downloading.  Abbreviated specs are only
-  # used for searching, downloading and related activities and do not
-  # need deployment specific information (e.g. list of files).  So we
-  # abbreviate the spec, making it much smaller for quicker downloads.
-  def abbreviate(spec)
-    spec.files = []
-    spec.test_files = []
-    spec.rdoc_options = []
-    spec.extra_rdoc_files = []
-    spec.cert_chain = []
-    spec
+  ##
+  # Install generated indicies into the destination directory.
+
+  def install_indicies
+    verbose = Gem.configuration.really_verbose
+
+    say "Moving index into production dir #{@dest_directory}" if verbose
+
+    @files.each do |file|
+      src_name = File.join @directory, file
+      dst_name = File.join @dest_directory, file
+
+      FileUtils.rm_rf dst_name, :verbose => verbose
+      FileUtils.mv src_name, @dest_directory, :verbose => verbose,
+                   :force => true
+    end
+  end
+
+  ##
+  # Ensure +path+ and path with +extension+ are identical.
+
+  def paranoid(path, extension)
+    data = Gem.read_binary path
+    compressed_data = Gem.read_binary "#{path}.#{extension}"
+
+    unless data == Gem.inflate(compressed_data) then
+      raise "Compressed file #{compressed_path} does not match uncompressed file #{path}"
+    end
   end
 
+  ##
   # Sanitize the descriptive fields in the spec.  Sometimes non-ASCII
   # characters will garble the site index.  Non-ASCII characters will
   # be replaced by their XML entity equivalent.
+
   def sanitize(spec)
     spec.summary = sanitize_string(spec.summary)
     spec.description = sanitize_string(spec.description)
@@ -158,7 +347,9 @@ class Gem::Indexer
     spec
   end
 
+  ##
   # Sanitize a single string.
+
   def sanitize_string(string)
     # HACK the #to_s is in here because RSpec has an Array of Arrays of
     # Strings for authors.  Need a way to disallow bad values on gempsec
@@ -168,9 +359,3 @@ class Gem::Indexer
 
 end
 
-require 'rubygems/indexer/abstract_index_builder'
-require 'rubygems/indexer/master_index_builder'
-require 'rubygems/indexer/quick_index_builder'
-require 'rubygems/indexer/marshal_index_builder'
-require 'rubygems/indexer/latest_index_builder'
-