stickler 2.0.2 → 2.1.0

data/lib/stickler/middleware/index.rb

@@ -7,27 +7,33 @@ require 'stickler/logable'
 require 'stickler/paths'
 
 module Stickler::Middleware
-  # Index is a Rack middleware that passes all requests through except for those
-  # matching these two urls:
+  # Index is a Rack middleware that passes all requests through except for the
+  # following urls:
   #
   # <b>/specs.#{Gem.marhsal_version}.gz</b>::         The [ name, version, platform ] index
   #                                                   of <b>all<b> the gems in the
   #                                                   entire repository
   #
-  # <b>/latest_specs.#{Gem.marshal_version}.gz</b>::  The [ name, version, # platform ] index
+  # <b>/latest_specs.#{Gem.marshal_version}.gz</b>::  The [ name, version, platform ] index
   #                                                   of the <b>most recent</b> version of each
   #                                                   gem in the repository.
   #
-  # For these 2 urls, it respond reponds with the summation of all the specs
-  # that are in <tt>env['stickler.specs']</tt>.  If there are no specs in that
-  # environment variable, then it returns with an empty index.
+  # <b>/prerelease_specs.#{Gem.marshal-version}.gz</b>:: The [ name, version, platform ] index
+  #                                                      of the <b>prerelease</b> versions of the
+  #                                                      prerelease gems in the repository
+  #
+  # <b>/quick/Marshal.#{Gem.marshal_version}/*.gemspec.rz</b>:: The gemspec of each gem
+  #
+  # <b>/gems/*.gem</b>::                              The actual .gem file to serve
+  #
+  # For the <b>specs</b> urls, it responds with the summation of all the specs
+  # that are in all the repositories served by stickler
   #
   # == Options
   #
   # This class is also the base class for all the other GemServer type
   # middlewares, so there is an optional behavior to NOT respond to the index
-  # url requests and just append the spec, or latest_specs to
-  # env['stickler.specs'] instead of serving the values out of there.
+  # url requests.
   #
   # <b>:serve_indexes</b>::     +true+ or +false+ it defaults to +true+.  This
   #                             option is used when Index is used in a stack
@@ -72,7 +78,6 @@ module Stickler::Middleware
     end
 
     get '/' do
-      append_specs
       if @serve_indexes then 
         erb :index
       else
@@ -83,24 +88,29 @@ module Stickler::Middleware
     #
     # Respond to the requests for the <b>all gems</b> index
     #
-    get %r{\A/specs.#{Gem.marshal_version}(\.gz)?\Z} do |with_compression|
-      append_specs
-      serve_indexes( with_compression )
+    get %r{\A/specs.#{Gem.marshal_version}(\.gz)?\Z} do |compression|
+      serve_indexes( released_specs, compression )
     end
 
     #
     # Respond to the requests for the <b>latest gems</b> index
     #
-    get %r{\A/latest_specs.#{Gem.marshal_version}(\.gz)?\Z} do |with_compression|
-      append_latest_specs
-      serve_indexes( with_compression )
+    get %r{\A/latest_specs.#{Gem.marshal_version}(\.gz)?\Z} do |compression|
+      serve_indexes( latest_specs, compression)
+    end
+
+    #
+    # Respond to the request for the <b>pre release gems</b> index
+    #
+    get %r{\A/prerelease_specs.#{Gem.marshal_version}(\.gz)?\Z} do |compression|
+      serve_indexes( prerelease_specs, compression )
     end
 
     #
     # Serve the indexes up as the response if @serve_indexes is true.  Otherwise
     # return false
     #
-    def serve_indexes( with_compression = :none )
+    def serve_indexes( specs, with_compression = :none )
       if @serve_indexes then
         self.compression = to_compression_flag( with_compression )
         return marshalled_specs( specs )
@@ -128,8 +138,10 @@ module Stickler::Middleware
     # Serve up a gemspec.  This is really only used by the child classes.
     # an Index instance will never have any gemspecs to return
     #
-    get %r{\A/quick/Marshal.#{Gem.marshal_version}/(.*?)-([0-9.]+)(-.*?)?\.gemspec\.rz\Z} do
-      name, version, platform, with_compression = *params[:captures]
+    # To support pre-releases the a-z has been added to the version
+    #
+    get %r{\A/quick/Marshal.#{Gem.marshal_version}/(.*?)-([0-9.]+[0-9a-z.]*)(-.*?)?\.gemspec\.rz\Z} do
+      name, version, platform = *params[:captures]
       spec = Stickler::SpecLite.new( name, version, platform )
       full_path = @repo.full_path_to_specification( spec )
       if full_path and File.exist?( full_path ) then
@@ -145,7 +157,27 @@ module Stickler::Middleware
     # everywhere
     #
     def marshalled_specs( spec_a )
-      marshal( spec_a.collect { |s| s.to_rubygems_a } )
+      a = spec_a.collect { |s| s.to_rubygems_a }
+      marshal( optimize_specs( a ) )
+    end
+
+    #
+    # Optimize the specs marshalling by using identical objects as much as possible.  this
+    # is take directly from RubyGems source code.  See rubygems/indexer.rb
+    # #compact_specs
+    #
+    def optimize_specs( specs )
+      names     = {}
+      versions  = {}
+      platforms = {}
+
+      specs.collect do |(name, version, platform)|
+        names[name]         = name     unless names.include?( name )
+        versions[version]   = version  unless versions.include?( version )
+        platforms[platform] = platform unless platforms.include?( platform )
+
+        [ names[name], versions[version], platforms[platform] ]
+      end
     end
 
     def marshal( data )

data/lib/stickler/repository/index.rb

@@ -18,11 +18,18 @@ module Stickler::Repository
     # The directory the specs live
     attr_reader :spec_dir
 
-    # 
+    # The last time the repository directory was modified
+    attr_reader :last_modified_time
+
+    # The number of entries in the spec directory
+    attr_reader :last_entry_count
 
     def initialize( spec_dir )
-      @specs = []
-      @spec_dir = spec_dir
+      @specs              = []
+      @spec_dir           = spec_dir
+      @last_modified_time = nil
+      @last_entry_count   = nil
+      @spec_glob          = File.join( @spec_dir, "*.gemspec" )
       load_specs
     end
 
@@ -31,9 +38,14 @@ module Stickler::Repository
       return @specs
     end
 
+    #
+    # return all the latest specs in the repository, do not include pre-release
+    # gems
+    #
     def latest_specs
       latest = {}
       specs.each do |s|
+        next if s.prerelease?
         if old_spec = latest[s.name] then
           if old_spec.version < s.version then
             latest[s.name] = s
@@ -45,25 +57,49 @@ module Stickler::Repository
       latest.values
     end
 
+    #
+    # return just the list of pre-release specs
+    #
+    def prerelease_specs
+      specs.select { |s| s.prerelease? }
+    end
+
+    #
+    # return just the list of release specs
+    #
+    def released_specs
+      specs.select { |s| not s.prerelease? }
+    end
+
     def load_specs
       load_specs_in_dir( self.spec_dir )
     end
 
     def reload_necessary?
-      return true
-      # return true unless @last_modified_time
-      # current_modified_time = File.stat( self.spec_dir ).mtime
-      # return (current_modified_time > @last_modified_time )
+      return true unless @last_modified_time
+      return true unless @last_entry_count
+      return true if (self.current_modified_time > @last_modified_time )
+      return true if (self.current_entry_count  != @last_entry_count   )
+      return false
     end
 
-    def last_modified_time
+    def current_modified_time
       File.stat( self.spec_dir ).mtime
     end
 
+    def current_entry_count
+      Dir.glob( @spec_glob ).size
+    end
+
     def spec_dir=( d )
       raise Error, "#{d} is not a directory" unless File.directory?( d )
       @spec_dir = d
-      @last_modified_time = File.stat( d ).mtime
+      update_reload_conditions
+    end
+
+    def update_reload_conditions
+      @last_modified_time = self.current_modified_time
+      @last_entry_count   = self.current_entry_count
     end
 
     def load_specs_in_dir( spec_dir )
@@ -75,6 +111,7 @@ module Stickler::Repository
         next unless entry =~ /\.gemspec\Z/
         add_spec_from_file( File.join( spec_dir, entry ) )
       end
+      update_reload_conditions
     end
 
     def add_spec_from_file( path )

data/lib/stickler/repository/local.rb

@@ -1,9 +1,11 @@
 require 'stickler/spec_lite'
+require 'stickler/logable'
 require 'stickler/repository'
 require 'stickler/repository/api'
 require 'stickler/repository/index'
 require 'addressable/uri'
 require 'tempfile'
+require 'forwardable'
 
 module Stickler::Repository
   #
@@ -18,6 +20,10 @@ module Stickler::Repository
   #
   class Local
     class Error < ::Stickler::Repository::Error; end
+    include Stickler::Logable
+
+    # The name to give to this repository
+    attr_reader :name
 
     # the root directory of the repository
     attr_reader :root_dir
@@ -34,13 +40,73 @@ module Stickler::Repository
     # the index of the repository
     attr_reader :index
 
-    def initialize( root_dir )
+    # mutex for synchronizing local instance allocations
+    @mutex = Mutex.new
+
+    # The list of repos keyd by root_dir
+    @repos = Hash.new
+
+    #
+    # Return the list of all the Local repositories
+    #
+    def self.repos
+      return @repos
+    end
+
+    #
+    # Purge the list of local repos that are known.  This is mainly used in
+    # testing.
+    #
+    def self.purge
+      @mutex.synchronize { @repos.clear }
+    end
+
+    #
+    # :call-seq:
+    #     Local.new( '/tmp/repo' )
+    #     Local.new( '/tmp/repo', "Temporary Repo" )
+    #
+    # Create a new Local repository.  Local repository instances
+    # are shared if the +root_dir+ is the same.  That is
+    #
+    #   repo1 = Local.new( '/foo/bar' )
+    #   repo2 = Local.new( '/foo/bar' )
+    #
+    # repo1 and repo2 will be references to the sname object
+    #
+    # If a new is called for an already existing repo, and the +name+
+    # of the repo is not nil, and different than the existing repo
+    # an exception is raised.
+    #
+    def self.new( root_dir, name = nil )
+      repo = nil
+      @mutex.synchronize do
+        local_key = File.expand_path( root_dir ) + File::SEPARATOR
+        repo = @repos[local_key]
+        if repo.nil? then
+          repo = super( local_key, name )
+          @repos[local_key] = repo
+        else
+          if name and (repo.name != name) then
+            raise Error, "A repository already exists for #{root_dir} with name has the name #{repo.name} which conflicts with the given name #{name}"
+          end
+        end
+      end
+      repo
+    end
+
+    def initialize( root_dir, name = nil )
       @root_dir = File.expand_path( root_dir ) + File::SEPARATOR
+      @name     = name || @root_dir
       @gems_dir = File.join( @root_dir, 'gems/' )
       @specifications_dir = File.join( @root_dir, 'specifications/' )
       @temp_dir = File.join( @root_dir, "tmp/" )
-      @index = ::Stickler::Repository::Index.new( @specifications_dir )
+
+      # setup the dirs before doing the index because the @specifications_dir
+      # may not exist yet.
       setup_dirs
+
+      @index = ::Stickler::Repository::Index.new( @specifications_dir )
     end
 
     #
@@ -66,25 +132,14 @@ module Stickler::Repository
     end
 
     #
-    # A list of all the specs in the repo
+    # Forward some calls directly to the index
     #
-    def specs
-      @index.specs
-    end
-
-    #
-    # A list of just the latests specs in the repo
-    #
-    def latest_specs
-      @index.latest_specs
-    end
-
-    #
-    # The last time this index was modified
-    #
-    def last_modified_time
-      @index.last_modified_time
-    end
+    extend Forwardable
+    def_delegators :@index, :specs,
+                            :released_specs,
+                            :latest_specs,
+                            :prerelease_specs,
+                            :last_modified_time
 
     #
     # See Api#search_for

data/lib/stickler/repository/null.rb

@@ -45,8 +45,6 @@ module Stickler::Repository
     alias :latest_specs :empty_array
     alias :search_for :empty_array
 
-    def
-
     def specs
       Array.new
     end

data/lib/stickler/repository/remote.rb

@@ -1,5 +1,6 @@
-require 'resourceful'
+require 'excon'
 require 'stickler/repository'
+require 'stickler/version'
 require 'stickler/repository/api'
 require 'stickler/repository/rubygems_authenticator'
 require 'stringio'
@@ -11,19 +12,13 @@ module ::Stickler::Repository
   # cutter api (push/yank/unyank).  The legacy gem server api is not utilized.
   #
   class Remote
-    # the http client
-    attr_reader :http
 
-    def initialize( repo_uri, options = {}   )
-      options[:authenticator] ||= Stickler::Repository::RubygemsAuthenticator.new
-      options[:cache_manager] ||= Resourceful::InMemoryCacheManager.new
-      if options.delete(:debug) then
-        options[:logger]      ||= Resourceful::StdOutLogger.new
-      end
+    attr_reader :authenticator
 
-      @uri        = Addressable::URI.parse( ensure_http( ensure_trailing_slash( repo_uri ) ) )
-      @http       = Resourceful::HttpAccessor.new( options )
-      @specs_list = nil
+    def initialize( repo_uri, options = {}   )
+      @authenticator = options[:authenticator] || Stickler::Repository::RubygemsAuthenticator.new
+      @uri           = Addressable::URI.parse( ensure_http( ensure_trailing_slash( repo_uri ) ) )
+      @specs_list    = nil
     end
 
     #
@@ -80,13 +75,11 @@ module ::Stickler::Repository
     def push( path )
       spec = speclite_from_gem_file( path )
       raise Stickler::Repository::Error, "gem #{spec.full_name} already exists in remote repository" if remote_gem_file_exist?( spec )
-      begin
-        resp = push_resource.post( IO.read( path ) )
-      rescue Resourceful::UnsuccessfulHttpRequestError => e
-        msg = "Failure pushing #{path} to remote repository : response code => #{e.http_response.code}, response message => '#{e.http_response.body}'"
-        raise Stickler::Repository::Error, msg
-      end
+      resp = resource_request( push_resource, :body => IO.read( path ) )
       return spec
+    rescue Excon::Errors::Error => e
+      msg = "Failure pushing #{path} to remote repository : response code => #{e.response.status}, response message => '#{e.response.body}'"
+      raise Stickler::Repository::Error, msg
     end
 
     #
@@ -94,15 +87,11 @@ module ::Stickler::Repository
     #
     def yank( spec )
       return nil unless remote_gem_file_exist?( spec )
-      begin
-        form_data = Resourceful::UrlencodedFormData.new
-        form_data.add( "gem_name", spec.name )
-        form_data.add( "version", spec.version.to_s )
-        yank_resource.request( :delete, form_data.read, {'Content-Type' => form_data.content_type } )
-        return full_uri_to_gem( spec )
-      rescue Resourceful::UnsuccessfulHttpRequestError => e
-        raise Stickler::Repository::Error, "Failure yanking: #{e.inspect}"
-      end
+      query = { :gem_name => spec.name, :version => spec.version.to_s }
+      resp  = resource_request( yank_resource, :query => query  )
+      return full_uri_to_gem( spec )
+    rescue Excon::Errors::Error => e
+      raise Stickler::Repository::Error, "Failure yanking: #{e.inspect}"
     end
 
     #
@@ -110,12 +99,10 @@ module ::Stickler::Repository
     #
     def delete( spec )
       return false unless remote_gem_file_exist?( spec )
-      begin
-        gem_resource( spec ).delete
-        return true
-      rescue Resourceful::UnsuccessfulHttpRequestError => e
-        return false
-      end
+      resource_request( gem_resource( spec ), :method => :delete )
+      return true
+    rescue Excon::Errors::Error => e
+      return false
     end
 
     #
@@ -123,22 +110,21 @@ module ::Stickler::Repository
     #
     def open( spec, &block )
       return nil unless remote_gem_file_exist?( spec )
-      begin
-        data = download_resource( gem_resource( spec ) )
-        io = StringIO.new( data , "rb" )
-        if block_given? then
-          begin
-            yield io
-          ensure
-            io.close
-          end
-        else
-          return io
+      data = download_resource( gem_resource( spec ) )
+      io = StringIO.new( data , "rb" )
+      if block_given? then
+        begin
+          yield io
+        ensure
+          io.close
         end
-      rescue Resourceful::UnsuccessfulHttpRequestError => e
-        return nil
+      else
+        return io
       end
       nil
+    rescue Excon::Errors::Error => e
+      $stderr.puts e.inspect
+      return nil
     end
 
     private
@@ -162,7 +148,7 @@ module ::Stickler::Repository
     end
 
     def specs_list_resource
-      @specs_list_resource ||= @http.resource( specs_list_uri )
+      @specs_list_resource ||= Excon.new( specs_list_uri.to_s, :method => :get, :expects => [200] )
     end
 
     def push_uri
@@ -170,7 +156,11 @@ module ::Stickler::Repository
     end
 
     def push_resource
-      @push_resource ||= @http.resource( push_uri, { 'Content-Type' => 'application/octet-stream' } )
+      unless @push_resource then
+        params = { :method => :post, :headers => { 'Content-Type' => 'application/octet-stream' }, :expects => [ 201, 200 ] }
+        @push_resource = Excon.new( push_uri.to_s, params )
+      end
+      return @push_resource
     end
 
     def yank_uri
@@ -178,11 +168,17 @@ module ::Stickler::Repository
     end
 
     def yank_resource
-      @yank_resource ||= @http.resource( yank_uri )
+      unless @yank_resource then
+        params = { :method => :delete,
+                   :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
+                   :expects => [200] }
+        @yank_resource = Excon.new( yank_uri.to_s, params )
+      end
+      return @yank_resource
     end
 
     def gem_resource( spec )
-      @http.resource( full_uri_to_gem( spec ) )
+      Excon.new( full_uri_to_gem( spec ), :method => :get, :expects => [200] )
     end
 
     def download_specs_list
@@ -198,15 +194,14 @@ module ::Stickler::Repository
     end
 
     def download_uri( uri )
-      download_resource( http.resource( uri ) )
+      download_resource( Excon.new( uri ) )
     end
 
     def download_resource( resource )
-      begin
-        resource.get.body
-      rescue Resourceful::UnsuccessfulHttpRequestError => e
-        return false
-      end
+      resource_request( resource, :method => :get, :expects => [200] ).body
+    rescue Excon::Errors::Error => e
+      puts e.inspect
+      return false
     end
 
     def remote_gem_file_exist?( spec )
@@ -215,14 +210,10 @@ module ::Stickler::Repository
     end
 
     def remote_uri_exist?( uri )
-      begin
-        # FIXME: bug in Resourceful that uses cached HEAD responses
-        #        to satisfy later GET requests.
-        rc = http.resource( uri ).head( 'Cache-Control' => 'no-store' ).successful?
-        return rc
-      rescue Resourceful::UnsuccessfulHttpRequestError => e
-        return false
-      end
+      rc = resource_request( Excon.new( uri.to_s ),  :method => :head, :expects => [200] )
+      return true
+    rescue Excon::Errors::Error => e
+      return false
     end
 
     def speclite_from_gem_file( path )
@@ -237,5 +228,31 @@ module ::Stickler::Repository
     def speclite_from_specification( spec )
       Stickler::SpecLite.new( spec.name, spec.version.to_s, spec.platform )
     end
+
+    def resource_request( resource, params = {} )
+      trys = 0
+      begin
+        resource.connection[:headers]['User-Agent'] = "Stickler Client v#{Stickler::VERSION}"
+        resource.connection[:headers].delete('Authorization')
+        if authenticator.handles?( resource.connection[:scheme], resource.connection[:host] ) then
+          resource.connection[:headers]['Authorization'] = authenticator.credentials
+        end
+        trys += 1
+        #puts "Making request #{resource.connection.inspect} with extra params #{params.inspect}"
+        resource.request( params )
+      rescue Excon::Errors::MovedPermanently, Excon::Errors::Found,
+             Excon::Errors::SeeOther, Excon::Errors::TemporaryRedirect => redirect
+        # follow a redirect, it is only allowable to follow redirects from a GET or
+        # HEAD request.  Only follow a few times though.
+        raise redirect unless [ :get, :head ].include?( redirect.request[:method] )
+        raise redirect if trys > 5
+        #puts "Redirecting to #{redirect.response.headers['Location']}"
+        resource = Excon::Connection.new( redirect.response.headers['Location'],
+                                          { :headers => resource.connection[:headers],
+                                            :query   => resource.connection[:headers],
+                                            :method  => resource.connection[:method] } )
+        retry
+      end
+    end
   end
 end