shoesgem 0.1424.0

data/shoes/ruby/lib/rubygems/source_info_cache.rb

@@ -0,0 +1,393 @@
+require 'fileutils'
+
+require 'rubygems'
+require 'rubygems/source_info_cache_entry'
+require 'rubygems/user_interaction'
+
+##
+# SourceInfoCache stores a copy of the gem index for each gem source.
+#
+# There are two possible cache locations, the system cache and the user cache:
+# * The system cache is preferred if it is writable or can be created.
+# * The user cache is used otherwise
+#
+# Once a cache is selected, it will be used for all operations.
+# SourceInfoCache will not switch between cache files dynamically.
+#
+# Cache data is a Hash mapping a source URI to a SourceInfoCacheEntry.
+#
+#--
+# To keep things straight, this is how the cache objects all fit together:
+#
+#   Gem::SourceInfoCache
+#     @cache_data = {
+#       source_uri => Gem::SourceInfoCacheEntry
+#         @size = source index size
+#         @source_index = Gem::SourceIndex
+#       ...
+#     }
+
+class Gem::SourceInfoCache
+
+  include Gem::UserInteraction
+
+  ##
+  # The singleton Gem::SourceInfoCache.  If +all+ is true, a full refresh will
+  # be performed if the singleton instance is being initialized.
+
+  def self.cache(all = false)
+    return @cache if @cache
+    @cache = new
+    @cache.refresh all if Gem.configuration.update_sources
+    @cache
+  end
+
+  def self.cache_data
+    cache.cache_data
+  end
+
+  ##
+  # The name of the system cache file.
+
+  def self.latest_system_cache_file
+    File.join File.dirname(system_cache_file),
+              "latest_#{File.basename system_cache_file}"
+  end
+
+  ##
+  # The name of the latest user cache file.
+
+  def self.latest_user_cache_file
+    File.join File.dirname(user_cache_file),
+              "latest_#{File.basename user_cache_file}"
+  end
+
+  ##
+  # Reset all singletons, discarding any changes.
+
+  def self.reset
+    @cache = nil
+    @system_cache_file = nil
+    @user_cache_file = nil
+  end
+
+  ##
+  # Search all source indexes.  See Gem::SourceInfoCache#search.
+
+  def self.search(*args)
+    cache.search(*args)
+  end
+
+  ##
+  # Search all source indexes returning the source_uri.  See
+  # Gem::SourceInfoCache#search_with_source.
+
+  def self.search_with_source(*args)
+    cache.search_with_source(*args)
+  end
+
+  ##
+  # The name of the system cache file. (class method)
+
+  def self.system_cache_file
+    @system_cache_file ||= Gem.default_system_source_cache_dir
+  end
+
+  ##
+  # The name of the user cache file.
+
+  def self.user_cache_file
+    @user_cache_file ||=
+      ENV['GEMCACHE'] || Gem.default_user_source_cache_dir
+  end
+
+  def initialize # :nodoc:
+    @cache_data = nil
+    @cache_file = nil
+    @dirty = false
+    @only_latest = true
+  end
+
+  ##
+  # The most recent cache data.
+
+  def cache_data
+    return @cache_data if @cache_data
+    cache_file # HACK writable check
+
+    @only_latest = true
+
+    @cache_data = read_cache_data latest_cache_file
+
+    @cache_data
+  end
+
+  ##
+  # The name of the cache file.
+
+  def cache_file
+    return @cache_file if @cache_file
+    @cache_file = (try_file(system_cache_file) or
+      try_file(user_cache_file) or
+      raise "unable to locate a writable cache file")
+  end
+
+  ##
+  # Write the cache to a local file (if it is dirty).
+
+  def flush
+    write_cache if @dirty
+    @dirty = false
+  end
+
+  def latest_cache_data
+    latest_cache_data = {}
+
+    cache_data.each do |repo, sice|
+      latest = sice.source_index.latest_specs
+
+      new_si = Gem::SourceIndex.new
+      new_si.add_specs(*latest)
+
+      latest_sice = Gem::SourceInfoCacheEntry.new new_si, sice.size
+      latest_cache_data[repo] = latest_sice
+    end
+
+    latest_cache_data
+  end
+
+  ##
+  # The name of the latest cache file.
+
+  def latest_cache_file
+    File.join File.dirname(cache_file), "latest_#{File.basename cache_file}"
+  end
+
+  ##
+  # The name of the latest system cache file.
+
+  def latest_system_cache_file
+    self.class.latest_system_cache_file
+  end
+
+  ##
+  # The name of the latest user cache file.
+
+  def latest_user_cache_file
+    self.class.latest_user_cache_file
+  end
+
+  ##
+  # Merges the complete cache file into this Gem::SourceInfoCache.
+
+  def read_all_cache_data
+    if @only_latest then
+      @only_latest = false
+      all_data = read_cache_data cache_file
+
+      cache_data.update all_data do |source_uri, latest_sice, all_sice|
+        all_sice.source_index.gems.update latest_sice.source_index.gems
+
+        Gem::SourceInfoCacheEntry.new all_sice.source_index, latest_sice.size
+      end
+
+      begin
+        refresh true
+      rescue Gem::RemoteFetcher::FetchError
+      end
+    end
+  end
+
+  ##
+  # Reads cached data from +file+.
+
+  def read_cache_data(file)
+    # Marshal loads 30-40% faster from a String, and 2MB on 20061116 is small
+    data = open file, 'rb' do |fp| fp.read end
+    cache_data = Marshal.load data
+
+    cache_data.each do |url, sice|
+      next unless sice.is_a?(Hash)
+      update
+
+      cache = sice['cache']
+      size  = sice['size']
+
+      if cache.is_a?(Gem::SourceIndex) and size.is_a?(Numeric) then
+        new_sice = Gem::SourceInfoCacheEntry.new cache, size
+        cache_data[url] = new_sice
+      else # irreperable, force refetch.
+        reset_cache_for url, cache_data
+      end
+    end
+
+    cache_data
+  rescue Errno::ENOENT
+    {}
+  rescue => e
+    if Gem.configuration.really_verbose then
+      say "Exception during cache_data handling: #{e.class} - #{e}"
+      say "Cache file was: #{file}"
+      say "\t#{e.backtrace.join "\n\t"}"
+    end
+
+    {}
+  end
+
+  ##
+  # Refreshes each source in the cache from its repository.  If +all+ is
+  # false, only latest gems are updated.
+
+  def refresh(all)
+    Gem.sources.each do |source_uri|
+      cache_entry = cache_data[source_uri]
+      if cache_entry.nil? then
+        cache_entry = Gem::SourceInfoCacheEntry.new nil, 0
+        cache_data[source_uri] = cache_entry
+      end
+
+      update if cache_entry.refresh source_uri, all
+    end
+
+    flush
+  end
+
+  def reset_cache_for(url, cache_data)
+    say "Reseting cache for #{url}" if Gem.configuration.really_verbose
+
+    sice = Gem::SourceInfoCacheEntry.new Gem::SourceIndex.new, 0
+    sice.refresh url, false # HACK may be unnecessary, see ::cache and #refresh
+
+    cache_data[url] = sice
+    cache_data
+  end
+
+  def reset_cache_data
+    @cache_data = nil
+    @only_latest = true
+  end
+
+  ##
+  # Force cache file to be reset, useful for integration testing of rubygems
+
+  def reset_cache_file
+    @cache_file = nil
+  end
+
+  ##
+  # Searches all source indexes.  See Gem::SourceIndex#search for details on
+  # +pattern+ and +platform_only+.  If +all+ is set to true, the full index
+  # will be loaded before searching.
+
+  def search(pattern, platform_only = false, all = false)
+    read_all_cache_data if all
+
+    cache_data.map do |source_uri, sic_entry|
+      next unless Gem.sources.include? source_uri
+      # TODO - Remove this gunk after 2008/11
+      unless pattern.kind_of?(Gem::Dependency)
+        pattern = Gem::Dependency.new(pattern, Gem::Requirement.default) 
+      end
+      sic_entry.source_index.search pattern, platform_only
+    end.flatten.compact
+  end
+
+  # Searches all source indexes for +pattern+.  If +only_platform+ is true,
+  # only gems matching Gem.platforms will be selected.  Returns an Array of
+  # pairs containing the Gem::Specification found and the source_uri it was
+  # found at.
+  def search_with_source(pattern, only_platform = false, all = false)
+    read_all_cache_data if all
+
+    results = []
+
+    cache_data.map do |source_uri, sic_entry|
+      next unless Gem.sources.include? source_uri
+
+      # TODO - Remove this gunk after 2008/11
+      unless pattern.kind_of?(Gem::Dependency)
+        pattern = Gem::Dependency.new(pattern, Gem::Requirement.default) 
+      end
+
+      sic_entry.source_index.search(pattern, only_platform).each do |spec|
+        results << [spec, source_uri]
+      end
+    end
+
+    results
+  end
+
+  ##
+  # Set the source info cache data directly.  This is mainly used for unit
+  # testing when we don't want to read a file system to grab the cached source
+  # index information.  The +hash+ should map a source URL into a
+  # SourceInfoCacheEntry.
+
+  def set_cache_data(hash)
+    @cache_data = hash
+    update
+  end
+
+  ##
+  # The name of the system cache file.
+
+  def system_cache_file
+    self.class.system_cache_file
+  end
+
+  ##
+  # Determine if +path+ is a candidate for a cache file.  Returns +path+ if
+  # it is, nil if not.
+
+  def try_file(path)
+    return path if File.writable? path
+    return nil if File.exist? path
+
+    dir = File.dirname path
+
+    unless File.exist? dir then
+      begin
+        FileUtils.mkdir_p dir
+      rescue RuntimeError, SystemCallError
+        return nil
+      end
+    end
+
+    return path if File.writable? dir
+
+    nil
+  end
+
+  ##
+  # Mark the cache as updated (i.e. dirty).
+
+  def update
+    @dirty = true
+  end
+
+  ##
+  # The name of the user cache file.
+
+  def user_cache_file
+    self.class.user_cache_file
+  end
+
+  ##
+  # Write data to the proper cache files.
+
+  def write_cache
+    if not File.exist?(cache_file) or not @only_latest then
+      open cache_file, 'wb' do |io|
+        io.write Marshal.dump(cache_data)
+      end
+    end
+
+    open latest_cache_file, 'wb' do |io|
+      io.write Marshal.dump(latest_cache_data)
+    end
+  end
+
+  reset
+
+end
+

data/shoes/ruby/lib/rubygems/source_info_cache_entry.rb

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'rubygems/source_index'
+require 'rubygems/remote_fetcher'
+
+##
+# Entries held by a SourceInfoCache.
+
+class Gem::SourceInfoCacheEntry
+
+  ##
+  # The source index for this cache entry.
+
+  attr_reader :source_index
+
+  ##
+  # The size of the of the source entry.  Used to determine if the
+  # source index has changed.
+
+  attr_reader :size
+
+  ##
+  # Create a cache entry.
+
+  def initialize(si, size)
+    @source_index = si || Gem::SourceIndex.new({})
+    @size = size
+    @all = false
+  end
+
+  def refresh(source_uri, all)
+    begin
+      marshal_uri = URI.join source_uri.to_s, "Marshal.#{Gem.marshal_version}"
+      remote_size = Gem::RemoteFetcher.fetcher.fetch_size marshal_uri
+    rescue Gem::RemoteSourceException
+      yaml_uri = URI.join source_uri.to_s, 'yaml'
+      remote_size = Gem::RemoteFetcher.fetcher.fetch_size yaml_uri
+    end
+
+    # TODO Use index_signature instead of size?
+    return false if @size == remote_size and @all
+
+    updated = @source_index.update source_uri, all
+    @size = remote_size
+    @all = all
+
+    updated
+  end
+
+  def ==(other) # :nodoc:
+    self.class === other and
+    @size == other.size and
+    @source_index == other.source_index
+  end
+
+end
+

data/shoes/ruby/lib/rubygems/spec_fetcher.rb

@@ -0,0 +1,249 @@
+require 'zlib'
+
+require 'rubygems'
+require 'rubygems/remote_fetcher'
+require 'rubygems/user_interaction'
+
+##
+# SpecFetcher handles metadata updates from remote gem repositories.
+
+class Gem::SpecFetcher
+
+  include Gem::UserInteraction
+
+  ##
+  # The SpecFetcher cache dir.
+
+  attr_reader :dir # :nodoc:
+
+  ##
+  # Cache of latest specs
+
+  attr_reader :latest_specs # :nodoc:
+
+  ##
+  # Cache of all spces
+
+  attr_reader :specs # :nodoc:
+
+  @fetcher = nil
+
+  def self.fetcher
+    @fetcher ||= new
+  end
+
+  def self.fetcher=(fetcher) # :nodoc:
+    @fetcher = fetcher
+  end
+
+  def initialize
+    @dir = File.join Gem.user_home, '.gem', 'specs'
+    @update_cache = File.stat(Gem.user_home).uid == Process.uid
+
+    @specs = {}
+    @latest_specs = {}
+
+    @fetcher = Gem::RemoteFetcher.fetcher
+  end
+
+  ##
+  # Retuns the local directory to write +uri+ to.
+
+  def cache_dir(uri)
+    File.join @dir, "#{uri.host}%#{uri.port}", File.dirname(uri.path)
+  end
+
+  ##
+  # Fetch specs matching +dependency+.  If +all+ is true, all matching
+  # versions are returned.  If +matching_platform+ is false, all platforms are
+  # returned.
+
+  def fetch(dependency, all = false, matching_platform = true)
+    specs_and_sources = find_matching dependency, all, matching_platform
+
+    specs_and_sources.map do |spec_tuple, source_uri|
+      [fetch_spec(spec_tuple, URI.parse(source_uri)), source_uri]
+    end
+
+  rescue Gem::RemoteFetcher::FetchError => e
+    raise unless warn_legacy e do
+      require 'rubygems/source_info_cache'
+
+      return Gem::SourceInfoCache.search_with_source(dependency,
+                                                     matching_platform, all)
+    end
+  end
+
+  def fetch_spec(spec, source_uri)
+    spec = spec - [nil, 'ruby', '']
+    spec_file_name = "#{spec.join '-'}.gemspec"
+
+    uri = source_uri + "#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}"
+
+    cache_dir = cache_dir uri
+
+    local_spec = File.join cache_dir, spec_file_name
+
+    if File.exist? local_spec then
+      spec = Gem.read_binary local_spec
+    else
+      uri.path << '.rz'
+
+      spec = @fetcher.fetch_path uri
+      spec = Gem.inflate spec
+
+      if @update_cache then
+        FileUtils.mkdir_p cache_dir
+
+        open local_spec, 'wb' do |io|
+          io.write spec
+        end
+      end
+    end
+
+    # TODO: Investigate setting Gem::Specification#loaded_from to a URI
+    Marshal.load spec
+  end
+
+  ##
+  # Find spec names that match +dependency+.  If +all+ is true, all matching
+  # versions are returned.  If +matching_platform+ is false, gems for all
+  # platforms are returned.
+
+  def find_matching(dependency, all = false, matching_platform = true)
+    found = {}
+
+    list(all).each do |source_uri, specs|
+      found[source_uri] = specs.select do |spec_name, version, spec_platform|
+        dependency =~ Gem::Dependency.new(spec_name, version) and
+          (not matching_platform or Gem::Platform.match(spec_platform))
+      end
+    end
+
+    specs_and_sources = []
+
+    found.each do |source_uri, specs|
+      uri_str = source_uri.to_s
+      specs_and_sources.push(*specs.map { |spec| [spec, uri_str] })
+    end
+
+    specs_and_sources
+  end
+
+  ##
+  # Returns Array of gem repositories that were generated with RubyGems less
+  # than 1.2.
+
+  def legacy_repos
+    Gem.sources.reject do |source_uri|
+      source_uri = URI.parse source_uri
+      spec_path = source_uri + "specs.#{Gem.marshal_version}.gz"
+
+      begin
+        @fetcher.fetch_size spec_path
+      rescue Gem::RemoteFetcher::FetchError
+        begin
+          @fetcher.fetch_size(source_uri + 'yaml') # re-raise if non-repo
+        rescue Gem::RemoteFetcher::FetchError
+          alert_error "#{source_uri} does not appear to be a repository"
+          raise
+        end
+        false
+      end
+    end
+  end
+
+  ##
+  # Returns a list of gems available for each source in Gem::sources.  If
+  # +all+ is true, all versions are returned instead of only latest versions.
+
+  def list(all = false)
+    list = {}
+
+    file = all ? 'specs' : 'latest_specs'
+
+    Gem.sources.each do |source_uri|
+      source_uri = URI.parse source_uri
+
+      if all and @specs.include? source_uri then
+        list[source_uri] = @specs[source_uri]
+      elsif not all and @latest_specs.include? source_uri then
+        list[source_uri] = @latest_specs[source_uri]
+      else
+        specs = load_specs source_uri, file
+
+        cache = all ? @specs : @latest_specs
+
+        cache[source_uri] = specs
+        list[source_uri] = specs
+      end
+    end
+
+    list
+  end
+
+  ##
+  # Loads specs in +file+, fetching from +source_uri+ if the on-disk cache is
+  # out of date.
+
+  def load_specs(source_uri, file)
+    file_name  = "#{file}.#{Gem.marshal_version}"
+    spec_path  = source_uri + "#{file_name}.gz"
+    cache_dir  = cache_dir spec_path
+    local_file = File.join(cache_dir, file_name)
+    loaded     = false
+
+    if File.exist? local_file then
+      spec_dump = @fetcher.fetch_path spec_path, File.mtime(local_file)
+
+      if spec_dump.nil? then
+        spec_dump = Gem.read_binary local_file
+      else
+        loaded = true
+      end
+    else
+      spec_dump = @fetcher.fetch_path spec_path
+      loaded = true
+    end
+
+    specs = Marshal.load spec_dump
+
+    if loaded and @update_cache then
+      begin
+        FileUtils.mkdir_p cache_dir
+
+        open local_file, 'wb' do |io|
+          Marshal.dump specs, io
+        end
+      rescue
+      end
+    end
+
+    specs
+  end
+
+  ##
+  # Warn about legacy repositories if +exception+ indicates only legacy
+  # repositories are available, and yield to the block.  Returns false if the
+  # exception indicates some other FetchError.
+
+  def warn_legacy(exception)
+    uri = exception.uri.to_s
+    if uri =~ /specs\.#{Regexp.escape Gem.marshal_version}\.gz$/ then
+      alert_warning <<-EOF
+RubyGems 1.2+ index not found for:
+\t#{legacy_repos.join "\n\t"}
+
+RubyGems will revert to legacy indexes degrading performance.
+      EOF
+
+      yield
+
+      return true
+    end
+
+    false
+  end
+
+end
+