rubygems-update 0.9.0 → 0.9.1

data/lib/rubygems/source_index.rb

@@ -5,6 +5,7 @@
 #++
 
 require 'rubygems/user_interaction'
+require 'rubygems/remote_fetcher'
 
 require 'forwardable'
 require 'digest/sha2'
@@ -24,8 +25,13 @@ module Gem
   #
   class SourceIndex
     extend Forwardable
+
     include Enumerable
 
+    include Gem::UserInteraction
+
+    INCREMENTAL_THRESHHOLD = 50
+
     # Class Methods. -------------------------------------------------
     class << self
       include Gem::UserInteraction
@@ -57,7 +63,7 @@ module Gem
       #   List of directory paths (all ending in "../specifications").
       #
       def installed_spec_directories
-	Gem.path.collect { |dir| File.join(dir, "specifications") }
+        Gem.path.collect { |dir| File.join(dir, "specifications") }
       end
 
       # Factory method to construct a source index instance for a
@@ -72,7 +78,7 @@ module Gem
       #   SourceIndex instance
       #
       def from_gems_in(*spec_dirs)
-	self.new.load_gems_in(*spec_dirs)
+        self.new.load_gems_in(*spec_dirs)
       end
       
       # Load a specification from a file (eval'd Ruby code)
@@ -81,22 +87,22 @@ module Gem
       # return:: Specification instance or nil if an error occurs
       #
       def load_specification(file_name)
-	begin
-	  spec_code = File.read(file_name).untaint
-	  gemspec = eval(spec_code)
-	  if gemspec.is_a?(Gem::Specification)
-	    gemspec.loaded_from = file_name
-	    return gemspec
-	  end
-	  alert_warning "File '#{file_name}' does not evaluate to a gem specification"
-	rescue SyntaxError => e
-	  alert_warning e
-	  alert_warning spec_code
-	rescue Exception => e
-	  alert_warning(e.inspect.to_s + "\n" + spec_code)
-	  alert_warning "Invalid .gemspec format in '#{file_name}'"
-	end
-	return nil
+        begin
+          spec_code = File.read(file_name).untaint
+          gemspec = eval(spec_code)
+          if gemspec.is_a?(Gem::Specification)
+            gemspec.loaded_from = file_name
+            return gemspec
+          end
+          alert_warning "File '#{file_name}' does not evaluate to a gem specification"
+        rescue SyntaxError => e
+          alert_warning e
+          alert_warning spec_code
+        rescue Exception => e
+          alert_warning(e.inspect.to_s + "\n" + spec_code)
+          alert_warning "Invalid .gemspec format in '#{file_name}'"
+        end
+        return nil
       end
       
     end
@@ -117,13 +123,31 @@ module Gem
     # directories.
     def load_gems_in(*spec_dirs)
       @gems.clear
-      Dir.glob("{#{spec_dirs.join(',')}}/*.gemspec").each do |file_name|
+      specs = Dir.glob File.join("{#{spec_dirs.join(',')}}", "*.gemspec")
+      specs.each do |file_name|
         gemspec = self.class.load_specification(file_name.untaint)
-	add_spec(gemspec) if gemspec
+        add_spec(gemspec) if gemspec
       end
       self
     end
 
+    # Returns a Hash of name => Specification of the latest versions of each
+    # gem in this index.
+    def latest_specs
+      thin = {}
+
+      each do |full_name, spec|
+        name = spec.name
+        if thin.has_key? name then
+          thin[name] = spec if spec.version > thin[name].version
+        else
+          thin[name] = spec
+        end
+      end
+
+      thin
+    end
+
     # Add a gem specification to the source index.
     def add_spec(gem_spec)
       @gems[gem_spec.full_name] = gem_spec
@@ -178,7 +202,6 @@ module Gem
     #   order.  Empty if not found.
     #
     def search(gem_pattern, version_requirement=Version::Requirement.new(">= 0"))
-      #FIXME - remove duplication between this and RemoteInstaller.search
       gem_pattern = /#{ gem_pattern }/i if String === gem_pattern
       version_requirement = Gem::Version::Requirement.create(version_requirement)
       result = []
@@ -197,7 +220,149 @@ module Gem
     def refresh!
       load_gems_in(self.class.installed_spec_directories)
     end
+
+    # Returns an Array of Gem::Specifications that are not up to date.
+    #
+    def outdated
+      remotes = Gem::SourceInfoCache.search(//)
+      outdateds = []
+
+      latest_specs.each do |_, local|
+        name = local.name
+        remote = remotes.select  { |spec| spec.name == name }.
+                         sort_by { |spec| spec.version }.
+                         last
+        outdateds << name if remote and local.version < remote.version
+      end
+
+      outdateds
+    end
+
+    def update(source_uri)
+      use_incremental = false
+
+      begin
+        gem_names = fetch_quick_index source_uri
+        remove_extra gem_names
+        missing_gems = find_missing gem_names
+        use_incremental = missing_gems.size <= INCREMENTAL_THRESHHOLD
+      rescue Gem::OperationNotSupportedError => ex
+        use_incremental = false
+      end
+
+      if use_incremental then
+        update_with_missing source_uri, missing_gems
+      else
+        new_index = fetch_bulk_index source_uri
+        @gems.replace new_index.gems
+      end
+
+      self
+    end
     
+    protected
+
+    attr_reader :gems
+
+    private
+
+    # Convert the yamlized string spec into a real spec (actually, these are
+    # hashes of specs.).
+    def convert_specs(yaml_spec)
+      YAML.load(reduce_specs(yaml_spec)) or
+      raise "Didn't get a valid YAML document"
+    end
+
+    def fetcher
+      Gem::RemoteFetcher.fetcher
+    end
+
+    def fetch_bulk_index(source_uri)
+      say "Bulk updating Gem source index for: #{source_uri}"
+
+      begin
+        yaml_spec = fetcher.fetch_path source_uri + '/yaml.Z'
+        yaml_spec = unzip yaml_spec
+      rescue
+        begin
+          yaml_spec = fetcher.fetch_path source_uri + '/yaml'
+        rescue => e
+          raise Gem::RemoteSourceException,
+                "Error fetching remote gem cache: #{e}"
+        end
+      end
+
+      convert_specs yaml_spec
+    end
+
+    # Get the quick index needed for incremental updates.
+    def fetch_quick_index(source_uri)
+      zipped_index = fetcher.fetch_path source_uri + '/quick/index.rz'
+      unzip(zipped_index).split("\n")
+    rescue ::Exception => ex
+      raise Gem::OperationNotSupportedError,
+            "No quick index found: " + ex.message
+    end
+
+    # Make a list of full names for all the missing gemspecs.
+    def find_missing(spec_names)
+      spec_names.find_all { |full_name|
+        specification(full_name).nil?
+      }
+    end
+
+    # This reduces the source spec in size so that YAML bugs with large data
+    # sets will be dodged.  Obviously this is a workaround, but it allows Gems
+    # to continue to work until the YAML bug is fixed.  
+    def reduce_specs(yaml_spec)
+      result = ""
+      state = :copy
+      yaml_spec.each do |line|
+        if state == :copy && line =~ /^\s+files:\s*$/
+          state = :skip
+          result << line.sub(/$/, " []")
+        elsif state == :skip
+          if line !~ /^\s+-/
+            state = :copy
+          end
+        end
+        result << line if state == :copy
+      end
+      result
+    end
+
+    def remove_extra(spec_names)
+      dictionary = spec_names.inject({}) { |h, k| h[k] = true; h }
+      each do |name, spec|
+        remove_spec name unless dictionary.include? name
+      end
+    end
+
+    # Unzip the given string.
+    def unzip(string)
+      require 'zlib'
+      Zlib::Inflate.inflate(string)
+    end
+
+    # Update the cached source index with the missing names.
+    def update_with_missing(source_uri, missing_names)
+      progress = ui.progress_reporter(missing_names.size,
+        "Need to update #{missing_names.size} gems from #{source_uri}")
+      missing_names.each do |spec_name|
+        begin
+          spec_uri = source_uri + "/quick/#{spec_name}.gemspec.rz"
+          zipped_yaml = fetcher.fetch_path spec_uri
+          gemspec = YAML.load unzip(zipped_yaml)
+          add_spec gemspec
+          progress.updated spec_name
+        rescue RuntimeError => ex
+          ui.say "Failed to download spec for #{spec_name} from #{source_uri}"
+        end
+      end
+      progress.done
+      progress.count
+    end
+
   end
 
   # Cache is an alias for SourceIndex to allow older YAMLized source
@@ -205,3 +370,4 @@ module Gem
   Cache = SourceIndex
 
 end
+

data/lib/rubygems/source_info_cache.rb

@@ -0,0 +1,153 @@
+require 'fileutils'
+require 'rubygems'
+require 'rubygems/remote_fetcher'
+require 'rubygems/source_info_cache_entry'
+
+require 'sources'
+
+##
+# SourceInfoCache stores a copy of the gem index for each gem source.
+#
+# There are two possible cache locations, the system cache and the user cache:
+# * The system cache is prefered if it is writable or can be created.
+# * The user cache is used otherwise
+#
+# Once a cache is selected, it will be used for all operations.
+# SourceInfoCache will not switch between cache files dynamically.
+#
+# Cache data is a Hash mapping a source URI to a SourceInfoCacheEntry.
+#
+#--
+# To keep things straight, this is how the cache objects all fit together:
+#
+#   Gem::SourceInfoCache
+#     @cache_data = {
+#       source_uri => Gem::SourceInfoCacheEntry
+#         @size => source index size
+#         @source_index => Gem::SourceIndex
+#       ...
+#     }
+#
+class Gem::SourceInfoCache
+
+  include Gem::UserInteraction
+
+  @cache = nil
+
+  def self.cache
+    return @cache if @cache
+    @cache = new
+    @cache.refresh
+    @cache
+  end
+
+  def self.cache_data
+    cache.cache_data
+  end
+
+  # Search all source indexes for +pattern+.
+  def self.search(pattern)
+    cache.search(pattern)
+  end
+
+  def initialize # :nodoc:
+    @cache_data = nil
+    @cache_file = nil
+    @dirty = false
+
+    @system_cache_file = nil
+    @user_cache_file = nil
+  end
+
+  # The most recent cache data.
+  def cache_data
+    return @cache_data if @cache_data
+    @dirty = false
+    cache_file # HACK writable check
+    # Marshal loads 30-40% faster from a String, and 2MB on 20061116 is small
+    @cache_data = Marshal.load(File.read(cache_file)) rescue {}
+  end
+
+  # The name of the cache file to be read
+  def cache_file
+    return @cache_file if @cache_file
+    @cache_file = (try_file(system_cache_file) or
+                   try_file(user_cache_file) or
+                   raise "unable to locate a writable cache file")
+  end
+
+  # Write the cache to a local file (if it is dirty).
+  def flush
+    write_cache if @dirty
+    @dirty = false
+  end
+
+  # Refreshes each source in the cache from its repository.
+  def refresh
+    Gem.sources.each do |source_uri|
+      cache_entry = cache_data[source_uri]
+      if cache_entry.nil? then
+        cache_entry = Gem::SourceInfoCacheEntry.new nil, 0
+        cache_data[source_uri] = cache_entry
+      end
+
+      cache_entry.refresh source_uri
+    end
+    update
+    flush
+  end
+
+  # Searches all source indexes for +pattern+.
+  def search(pattern)
+    cache_data.map do |source, sic_entry|
+      sic_entry.source_index.search pattern
+    end.flatten
+  end
+
+  # The name of the system cache file.
+  def system_cache_file
+    @system_cache_file ||= File.join(Gem.dir, "source_cache")
+  end
+
+  # Mark the cache as updated (i.e. dirty).
+  def update
+    @dirty = true
+  end
+
+  # The name of the user cache file.
+  def user_cache_file
+    @user_cache_file ||=
+      ENV['GEMCACHE'] || File.join(Gem.user_home, ".gem", "source_cache")
+  end
+
+  # Write data to the proper cache.
+  def write_cache
+    open cache_file, "wb" do |f|
+      f.write Marshal.dump(cache_data)
+    end
+  end
+
+  private
+
+  # Determine if +fn+ is a candidate for a cache file.  Return fn if
+  # it is.  Return nil if it is not.
+  def try_file(fn)
+    return fn if File.writable?(fn)
+    return nil if File.exist?(fn)
+    dir = File.dirname(fn)
+    if ! File.exist? dir
+      begin
+        FileUtils.mkdir_p(dir)
+      rescue RuntimeError
+        return nil
+      end
+    end
+    if File.writable?(dir)
+      FileUtils.touch fn
+      return fn
+    end
+    nil
+  end
+
+end
+

data/lib/rubygems/source_info_cache_entry.rb

@@ -0,0 +1,31 @@
+require 'rubygems'
+require 'rubygems/source_index'
+require 'rubygems/remote_fetcher'
+
+##
+# Entrys held by a SourceInfoCache.
+
+class Gem::SourceInfoCacheEntry
+
+  # The source index for this cache entry.
+  attr_reader :source_index
+
+  # The size of the of the source entry.  Used to determine if the
+  # source index has changed.
+  attr_reader :size
+
+  # Create a cache entry.
+  def initialize(si, size)
+    @source_index = si || Gem::SourceIndex.new({})
+    @size = size
+  end
+
+  def refresh(source_uri)
+    remote_size = Gem::RemoteFetcher.fetcher.fetch_size source_uri + '/yaml'
+    return if @size == remote_size
+    @source_index.update source_uri 
+    @size = remote_size
+  end
+
+end
+