stickler 0.1.1 → 2.0.0a

data/HISTORY.rdoc

@@ -1,9 +1,12 @@
 = stickler Changelog
+== Version 0.1.2
 
-=== Version 0.1.1
+* fix compatibility with gems 1.3.1
+
+== Version 0.1.1
 * 1 minor bugfix
   * remove unnecessary require 'progressbar' 
 
-=== Version 0.1.0
+== Version 0.1.0
 * {Managing a Gem Repository with Stickler}[http://copiousfreetime.org/articles/2008/10/09/managing-a-gem-repository-with-stickler.html]
 * Initial public release

data/Rakefile

@@ -0,0 +1,31 @@
+begin
+  require 'bones'
+rescue LoadError
+  abort '### Please install the "bones" gem ###'
+end
+
+task :default => 'spec:run'
+task 'gem:release' => 'spec:run'
+
+Bones {
+  name 'stickler'
+  authors 'Jeremy Hinegardner'
+  email   'jeremy@hinegardner.org'
+  url     'http://rubygems.org/gems/stickler'
+
+  ruby_opts      %w[-W0 -rubygems]
+  readme_file    'README.rdoc'
+  ignore_file    '.gitignore'
+  history_file   'HISTORY.rdoc'
+  rubyforge.name 'copiousfreetime'
+
+  spec.opts << "--color" << "--format specdoc"
+
+  depend_on 'sinatra', '~> 1.0.0'
+  depend_on 'addressable', '~> 2.1.2'
+  depend_on 'resourceful', '~> 1.0.1'
+
+  depend_on 'bones'       , '~> 3.4.6', :development => true
+  depend_on 'rack-test'   , '~> 0.5.4', :development => true
+  depend_on 'bones-extras', '~> 1.2.4', :development => true
+}

data/examples/config.ru

@@ -0,0 +1,9 @@
+#-----------------------------------------------------------------------
+# Example rackup file for an entire stickler stack
+#-----------------------------------------------------------------------
+$:.unshift File.expand_path( File.join( File.dirname(__FILE__), "..", "lib" ) )
+
+require 'stickler/web'
+
+use Stickler::Web
+run Sinatra::Base

data/examples/gemcutter_repo.ru

@@ -0,0 +1,19 @@
+#-----------------------------------------------------------------------
+# An Example remote repository that implements all the methods that are
+# required to satisfy being talked to by a Respository::Remote client.
+# This means it needs to speak:
+# - the gem cutter api
+# - the modern gem server ai
+# 
+#-----------------------------------------------------------------------
+$:.unshift File.expand_path( File.join( File.dirname(__FILE__), "..", "lib" ) )
+
+require 'stickler/middleware/gemcutter'
+require 'stickler/middleware/compression'
+
+gem_dir = File.join( File.expand_path( File.dirname( __FILE__ ) ), "..", "spec", "tmp" )
+
+use ::Stickler::Middleware::Compression
+use ::Stickler::Middleware::Gemcutter, :repo_root => gem_dir
+run ::Sinatra::Base
+

data/examples/index_repo.ru

@@ -0,0 +1,15 @@
+#-----------------------------------------------------------------------
+# Example rackup file for serving up a null repository.  This really 
+# would never be used in the wild, but it shows the basics of what 
+# is required to setup a stickler webstack
+#-----------------------------------------------------------------------
+$:.unshift File.expand_path( File.join( File.dirname(__FILE__), "..", "lib" ) )
+
+require 'stickler/middleware/compression'
+require 'stickler/middleware/index'
+
+gem_dir = File.join( File.expand_path( File.dirname( __FILE__ ) ), "tmp" )
+
+use ::Stickler::Middleware::Compression
+use ::Stickler::Middleware::Index, :repo_root => gem_dir
+run ::Sinatra::Base

data/examples/local_repo.ru

@@ -0,0 +1,14 @@
+#-----------------------------------------------------------------------
+# Example rackup file for serving up a single repository.  This repository
+# will respond to index and gem requests.
+#-----------------------------------------------------------------------
+$:.unshift File.expand_path( File.join( File.dirname(__FILE__), "..", "lib" ) )
+
+require 'stickler/middleware/compression'
+require 'stickler/middleware/repo_local'
+
+gem_dir = File.join( File.expand_path( File.dirname( __FILE__ ) ), "data" )
+
+use ::Stickler::Middleware::Compression
+use ::Stickler::Middleware::RepoLocal, :repo_root => gem_dir
+run ::Sinatra::Base

data/examples/mirror_repo.ru

@@ -0,0 +1,16 @@
+#-----------------------------------------------------------------------
+# Example repository for serving up a Mirror repository.  This repo
+# will respond to mirroring requests and mirror gems from a remote
+# repository locally.
+#-----------------------------------------------------------------------
+$:.unshift File.expand_path( File.join( File.dirname(__FILE__), "..", "lib" ) )
+
+require 'stickler/middleware/compression'
+require 'stickler/middleware/mirror'
+
+gem_dir = File.join( File.expand_path( File.dirname( __FILE__ ) ), "tmp" )
+
+use ::Stickler::Middleware::Compression
+use ::Stickler::Middleware::Mirror, :repo_root => gem_dir
+run ::Sinatra::Base
+

data/examples/not_found.ru

@@ -0,0 +1,8 @@
+#-----------------------------------------------------------------------
+# Serve up the not found page for everything
+#-----------------------------------------------------------------------
+$:.unshift File.expand_path( File.join( File.dirname(__FILE__), "..", "lib" ) )
+
+require 'stickler/middleware/not_found'
+
+run ::Stickler::Middleware::NotFound.new

data/lib/stickler/error.rb

@@ -0,0 +1,3 @@
+module Stickler
+  class Error < ::StandardError ; end 
+end

data/lib/stickler/middleware/compression.rb

@@ -0,0 +1,30 @@
+require 'rack/utils'
+require 'stickler/middleware'
+module Stickler::Middleware
+  class Compression
+    def initialize( app )
+      @app = app 
+    end
+
+    def call( env )
+      status, headers, body = @app.call( env )
+      return [ status, headers, body ] unless status == 200
+
+      headers = ::Rack::Utils::HeaderHash.new( headers )
+      stream  = body
+
+      if compress_method = env['stickler.compression'] then
+        headers.delete('Content-Length')
+        case compress_method
+        when :gzip
+          headers['Content-Type'] = 'application/x-gzip'
+          stream = Gem.gzip( body.first )
+        when :deflate
+          headers['Content-Type'] = 'application/x-deflate'
+          stream = Gem.deflate( body.first )
+        end
+      end
+      return [ status, headers, stream ]
+    end
+  end
+end

data/lib/stickler/middleware/gemcutter.rb

@@ -0,0 +1,62 @@
+require 'sinatra/base'
+require 'stickler/middleware'
+require 'stickler/middleware/local'
+require 'stickler/repository/local'
+
+module Stickler::Middleware
+  # 
+  # A rack middleware for implementing the gemcutter api
+  #
+  # == Options
+  #
+  # <b>:serve_indexes</b>::   the same as the Index middleware
+  #
+  # <b>:repo_root</b>::       the same as the Local middleware
+  #
+  # The <b>:repo_root</b> option is required.
+  #
+  # == Usage
+  #
+  #   use Stickler::Middleware::Gemcutter, :repo_root => '/path/to/repository'
+  #
+  #   use Stickler::Middleware::Gemcutter, :repo_root => '/path/to/repository',
+  #                                        :serve_indexes => true
+  #
+  #
+  class Gemcutter < ::Stickler::Middleware::Local
+
+    def initialize( app = nil, options = {}  )
+      super( app, options )
+    end
+
+    # gemcutter push
+    post '/api/v1/gems' do
+      begin
+        spec = @repo.add( request.body )
+        return spec.to_s
+      rescue Stickler::Repository::Error => e
+        error( 500, "Error adding gem to repo: #{e}" )
+      end
+    end
+
+    # gemcutter yank
+    delete '/api/v1/gems/yank' do
+      spec = Stickler::SpecLite.new( params[:gem_name], params[:version] )
+      if s = @repo.yank( spec ) then
+        return "Yanked #{s.to_s}"
+      else
+        error( 503, "Did not Yank #{spec.to_s}" )
+      end
+    end
+
+    # direct delete
+    delete %r{\A/gems/((.*?)-([0-9.]+)(-.*?)?)\.gem\Z} do
+      full_name, name, version, platform = *params[:captures]
+      spec = Stickler::SpecLite.new( name, version, platform )
+      @repo.delete( spec )
+      return "deleted gem #{spec.full_name}"
+    end
+  end
+end
+
+

data/lib/stickler/middleware/helpers.rb

@@ -0,0 +1,84 @@
+require 'stickler/middleware'
+module Stickler::Middleware
+  module Helpers
+    #
+    # set what, if any kind of compression to use on the response This is a Gem
+    # server specific type compressions, as it does not set the http headers and
+    # such in the same manner as normal compressed HTTP responses
+    #
+    # compression may be set to one of the following, all others will be
+    # ignored.
+    #
+    # <b>:gzip</b>::    use Gem.gzip
+    # <b>:deflate</b>:: use Gem.deflate
+    # <b>nil</b>::      no compression
+    #
+    module Compression
+      def compression=( type ) env['stickler.compression'] = type end
+      def compression()        env['stickler.compression']        end
+    end
+
+    #
+    # Manage the contents of the <tt>stickler.specs</tt> environment variable.
+    # It is used as as communcation method between the various types of
+    # middlewares managing gem repositories.  The Index server will use the
+    # values in this variable in generating the responses to gem index requests
+    #
+    # env['stickler.specs'] is a Hash itself, the key being the return value of
+    # +root_dir+ from the Class it is included in, the value for each key is
+    # the Array of SpecLite's.
+    #
+    #
+    module Specs
+      #
+      # The specs by repository
+      #
+      def specs_by_repo
+        env['stickler.specs'] ||= Hash.new{ |h,k| h[k] = Array.new }
+      end
+
+      #
+      # return the flattened array of all the values in
+      # <tt>#specs_by_repo</tt>
+      #
+      def specs
+        [ specs_by_repo.values ].flatten.sort
+      end
+
+      #
+      # Append spec or array of specs to the current list of specs for this key.
+      #
+      def append_spec( key, spec_or_array_of_specs )
+        if Array === spec_or_array_of_specs then
+          specs_by_repo[key].concat(  spec_or_array_of_specs )
+        else
+          specs_by_repo[key] << spec_or_array_of_specs
+        end
+      end
+
+      #
+      # Automatically append the specs from the included class into the specs
+      # environment variable.
+      #
+      # The Class that includes this module and wants to use +append_specs+
+      # MUST have a +repo+ method. The +repo+ method must +respond_to+ both
+      # +root_dir+ and +specs+.
+      #
+      def append_specs
+        append_spec( self.repo.root_dir, self.repo.specs )
+      end
+
+      #
+      # Automatically append the latest_specs from the included class into the
+      # specs environment variable.
+      #
+      # The Class that includes this module and wants to use +append_specs+ MUST
+      # have a +repo+ method. The +repo+ method must +respond_to+ both
+      # +root_dir+ and +specs+.
+      #
+      def append_latest_specs
+        append_spec( self.repo.root_dir, self.repo.latest_specs )
+      end
+    end
+  end
+end

data/lib/stickler/middleware/index.rb

@@ -0,0 +1,137 @@
+require 'sinatra'
+require 'stickler/middleware'
+require 'stickler/middleware/helpers'
+require 'stickler/repository/null'
+
+module Stickler::Middleware
+  # Index is a Rack middleware that passes all requests through except for those
+  # matching these two urls:
+  #
+  # <b>/specs.#{Gem.marhsal_version}.gz</b>::         The [ name, version, platform ] index
+  #                                                   of <b>all<b> the gems in the
+  #                                                   entire repository
+  #
+  # <b>/latest_specs.#{Gem.marshal_version}.gz</b>::  The [ name, version, # platform ] index
+  #                                                   of the <b>most recent</b> version of each
+  #                                                   gem in the repository.
+  #
+  # For these 2 urls, it respond reponds with the summation of all the specs
+  # that are in <tt>env['stickler.specs']</tt>.  If there are no specs in that
+  # environment variable, then it returns with an empty index.
+  #
+  # == Options
+  #
+  # This class is also the base class for all the other GemServer type
+  # middlewares, so there is an optional behavior to NOT respond to the index
+  # url requests and just append the spec, or latest_specs to
+  # env['stickler.specs'] instead of serving the values out of there.
+  #
+  # <b>:serve_indexes</b>::     +true+ or +false+ it defaults to +true+.  This
+  #                             option is used when Index is used in a stack
+  #                             with other Index derived  middlewares.  In this
+  #                             case, all of the Index derived middlewares
+  #                             should set <b>:serve_indexes => false</b> except
+  #                             for the bottom one.  It should set
+  #                             <b>:serve_indexes => true</b>.  This allows all
+  #                             the Index derived middlewares to cooperatively
+  #                             respond to the <b>/specs</b> and
+  #                             </b>/latests_specs</b> urls.
+  #
+  # == Usage
+  #
+  # use Stickler::Middleware::Index, :serve_indexes => true
+  # use Stickler::Middleware::Index, :serve_indexes => false
+  #
+  class Index < ::Sinatra::Base
+    include Stickler::Middleware::Helpers::Compression
+    include Stickler::Middleware::Helpers::Specs
+
+    # The respository of the Index is a Repository::Null
+    attr_reader :repo
+
+    def initialize( app, opts = {} )
+      @app           = app
+      @repo          = ::Stickler::Repository::Null.new
+      @serve_indexes = opts.has_key?( :serve_indexes ) ? opts[:serve_indexes] : true
+      super( app )
+    end
+
+    before do
+      response["Date"] = @repo.last_modified_time.rfc2822
+      cache_control( 'no-cache' )
+    end
+
+    get '/' do
+      if @serve_indexes then 
+        erb :index
+      else
+        not_found
+      end
+    end
+
+    #
+    # Respond to the requests for the <b>all gems</b> index
+    #
+    get %r{\A/specs.#{Gem.marshal_version}(\.gz)?\Z} do |with_compression|
+      append_specs
+      serve_indexes( with_compression )
+    end
+
+    #
+    # Respond to the requests for the <b>latest gems</b> index
+    #
+    get %r{\A/latest_specs.#{Gem.marshal_version}(\.gz)?\Z} do |with_compression|
+      append_latest_specs
+      serve_indexes( with_compression )
+    end
+
+    #
+    # Serve the indexes up as the response if @serve_indexes is true.  Otherwise
+    # return false
+    #
+    def serve_indexes( with_compression = :none )
+      if @serve_indexes then
+        self.compression = to_compression_flag( with_compression )
+        return marshalled_specs( specs )
+      else
+        pass
+      end
+    end
+
+    #
+    # Actually serve up the gem.  This is really only used by the child classes.
+    # an Index instance will never have any gems to return.
+    #
+    get %r{\A/gems/(.*?)-([0-9.]+)(-.*?)?\.gem\Z} do
+      name, version, platform = *params[:captures]
+      spec = Stickler::SpecLite.new( name, version, platform )
+      full_path = @repo.full_path_to_gem( spec )
+      if full_path then
+        content_type 'application/x-tar'
+        send_file( full_path )
+      else
+        pass
+      end
+    end
+
+    #
+    # Convert to the array format used by gem servers
+    # everywhere
+    #
+    def marshalled_specs( spec_a )
+      marshal( spec_a.collect { |s| s.to_rubygems_a } )
+    end
+
+    def marshal( data )
+      content_type 'application/octet-stream'
+      ::Marshal.dump( data )
+    end
+
+    def to_compression_flag( with_compression )
+      return with_compression if [ :gzip, :deflate, :none ].include?( with_compression )
+      return :gzip            if with_compression =~ /\.gz\Z/i
+      return :deflate         if with_compression =~ /\.(Z|rz)\Z/i
+      return :none
+    end
+  end
+end

data/lib/stickler/middleware/local.rb

@@ -0,0 +1,38 @@
+require 'sinatra'
+require 'stickler/middleware'
+require 'stickler/middleware/index'
+require 'stickler/repository/local'
+
+module Stickler::Middleware
+  #
+  # A Sinatra middleware that implements the HTTP portions of a Modern gem server.  
+  # It sits on top of a Repository::Local and serves up the gems in it.
+  #
+  # It utilizies a Stickler::Repository::Local, and the :repo_root option
+  # is passed directly to it.
+  #
+  # == Options
+  #
+  # <b>:serve_indexes</b>::   the same as the Index middleware
+  #
+  # <b>:repo_root</b>::       The path that is to be the root of the
+  #                           Repository instance managed by this server.
+  #
+  # The <b>:repo_root</b> option is required.
+  #
+  # == Usage
+  #
+  #   use Stickler::Middleware::Local, :repo_root => '/path/to/repository'
+  #
+  #   use Stickler::Middleware::Local, :repo_root => '/path/to/repository',
+  #                                    :serve_indexes => true
+  #
+  class Local < Index
+    def initialize( app = nil, opts = {} )
+      super( app, opts )
+      # overwrite the repo that is set in the parent
+      @repo      = ::Stickler::Repository::Local.new( opts[:repo_root] )
+    end
+
+  end
+end

data/lib/stickler/middleware/mirror.rb

@@ -0,0 +1,60 @@
+require 'sinatra/base'
+require 'stickler/middleware/index'
+require 'stickler/repository/mirror'
+
+module Stickler::Middleware
+  #
+  # A Mirror server keeps gems from one or more upstream gem servers in local
+  # repositories.
+  #
+  # == Options
+  #
+  # <b>:serve_indexes</b>::   the same as the Index middleware
+  #
+  # <b>:repo_root</b>::       The path that is to be the root of the
+  #                           Repository instance managed by this server.
+  #
+  # The <b>:repo_root</b> option is required.
+  #
+  # == Usage
+  #
+  #   use Stickler::Middleware::Mirror, :repo_root => '/path/to/repository'
+  #
+  #   use Stickler::Middleware::Mirror, :repo_root => '/path/to/repository',
+  #                                     :serve_indexes => true
+  #
+  class Mirror < ::Stickler::Middleware::Index
+
+    def initialize( app, options = {} )
+      super( app )
+      @repo = ::Stickler::Repository::Mirror.new( options[:repo_root] )
+    end
+
+    def manage( params )
+      host = params[:source]
+      spec = Stickler::SpecLite.new( params[:name], params[:version], params[:platform] )
+
+      begin
+        if spec = @repo.mirror( host , spec ) then
+          status 201
+          response["Location"] = "/gems/#{spec.file_name}"
+          nil
+        else
+          not_found "Unable to find gem [#{spec.full_name}] at source #{host}"
+        end
+      rescue ::Stickler::Repository::Mirror::Error => e
+        error( 409, e.message )
+      end
+    end
+
+    post '/:source/:name/:version/:platform' do
+      manage( params )
+    end
+
+    post '/:source/:name/:version' do
+      manage( params )
+    end
+  end
+end
+
+

data/lib/stickler/middleware/not_found.rb

@@ -0,0 +1,62 @@
+require 'stickler/middleware'
+module Stickler::Middleware
+  #
+  # Idea completely taken from rack-contrib
+  #
+  class NotFound
+    def initialize
+      @body = <<-_
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
+"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+  <head>
+    <meta content="text/html; charset=utf-8" http-equiv="Content-Type" />
+    <style type="text/css" media="screen">
+      * { margin: 0; padding: 0; border: 0; outline: 0; }
+      div.clear { clear: both; }
+      body { background: #eeeeee; margin: 0; padding: 0; }
+      #wrap { width: 1000px; margin: 0 auto; padding: 30px 50px 20px 50px;
+              background: #fff; border-left: 1px solid #DDD;
+              border-right: 1px solid #DDD; }
+      #header { margin: 0 auto 25px auto; }
+      h1 { margin: 0; font-size: 36px; color: #981919; text-align: center; }
+      h2 { margin: 0; font-size: 22px; color: #333333; }
+      table.gem { width: 980px; text-align: left; font-size: 12px;
+                  color: #666666; padding: 0; border-spacing: 0;
+                  border: 1px solid #EEEEEE; border-bottom: 0;
+                  border-left: 0;
+                  clear:both;}
+      table.gem tr th { padding: 2px 10px; font-weight: bold;
+        font-size: 22px; background: #f7f7f7; text-align: center;
+        border-left: 1px solid #eeeeee;
+        border-bottom: 1px solid #eeeeee; }
+      table.gem tr td { padding: 2px 20px 2px 10px;
+                        border-bottom: 1px solid #eeeeee;
+                        border-left: 1px solid #eeeeee; }
+
+    </style>
+    <title>Stickler - Not Found</title>
+  </head>
+  <body>
+    <div id="wrap">
+      <div id="header">
+        <h1>Nothing Found</h1>
+      </div>
+      <h2>Try <a href="/">Over Here</a></h2>
+    </div>
+  </body>
+</html>
+      _
+      @size = @body.size.to_s
+    end
+
+    def call( env )
+      [ 404, 
+        { 'Content-Type' => 'text/html', 'Content-Length' => @size },
+        [ @body ]
+      ]
+    end
+  end
+end

data/lib/stickler/middleware.rb

@@ -0,0 +1,4 @@
+module Stickler
+  module Middleware
+  end
+end