gemstash 1.0.0.pre.1

data/lib/gemstash/gem_fetcher.rb

@@ -0,0 +1,50 @@
+require "gemstash"
+require "set"
+
+module Gemstash
+  #:nodoc:
+  class GemFetcher
+    def initialize(http_client)
+      @http_client = http_client
+      @valid_headers = Set.new(["etag", "content-type", "content-length", "last-modified"])
+    end
+
+    def fetch(gem_id, type, &block)
+      @http_client.get(path_for(gem_id, type)) do |body, headers|
+        properties = filter_headers(headers)
+        validate_download(body, properties)
+        yield body, properties
+      end
+    end
+
+  private
+
+    def path_for(gem_id, type)
+      case type
+      when :gem
+        "gems/#{gem_id}"
+      when :spec
+        "quick/Marshal.4.8/#{gem_id}"
+      else
+        raise "Invalid type #{type.inspect}"
+      end
+    end
+
+    def filter_headers(headers)
+      headers.inject({}) do|properties, (key, value)|
+        properties[key.downcase] = value if @valid_headers.include?(key.downcase)
+        properties
+      end
+    end
+
+    def validate_download(content, headers)
+      expected_size = content_length(headers)
+      raise "Incomplete download, only #{body.length} was downloaded out of #{expected_size}" \
+        if content.length < expected_size
+    end
+
+    def content_length(headers)
+      headers["content-length"].to_i
+    end
+  end
+end

data/lib/gemstash/gem_pusher.rb

@@ -0,0 +1,125 @@
+require "gemstash"
+require "rubygems/package"
+require "stringio"
+
+#:nodoc:
+module Gemstash
+  # Class that supports pushing a new gem to the private repository of gems.
+  class GemPusher
+    include Gemstash::Env::Helper
+
+    # This error is thrown when pushing to an existing version.
+    class ExistingVersionError < StandardError
+    end
+
+    # This error is thrown when pushing to a yanked version.
+    class YankedVersionError < ExistingVersionError
+    end
+
+    def initialize(auth_key, content)
+      @auth_key = auth_key
+      @content = content
+    end
+
+    def push
+      check_auth
+      store_gem
+      store_gemspec
+      save_to_database
+      invalidate_cache
+    end
+
+  private
+
+    def gem
+      @gem ||= Gem::Package.new(StringIO.new(@content))
+    end
+
+    def storage
+      @storage ||= Gemstash::Storage.for("private").for("gems")
+    end
+
+    def full_name
+      @full_name ||= gem.spec.full_name
+    end
+
+    def check_auth
+      Gemstash::Authorization.check(@auth_key, "push")
+    end
+
+    def store_gem
+      storage.resource(full_name).save({ gem: @content }, indexed: true)
+    end
+
+    def store_gemspec
+      spec = gem.spec
+      spec = Marshal.dump(spec)
+      spec = Zlib::Deflate.deflate(spec)
+      storage.resource(full_name).save(spec: spec)
+    end
+
+    def save_to_database
+      spec = gem.spec
+
+      gemstash_env.db.transaction do
+        gem_id = Gemstash::DB::Rubygem.find_or_insert(spec)
+        existing = Gemstash::DB::Version.find_by_spec(gem_id, spec)
+
+        if existing
+          if existing.indexed
+            raise ExistingVersionError, "Cannot push to an existing version!"
+          else
+            raise YankedVersionError, "Cannot push to a yanked version!"
+          end
+        end
+
+        version_id = Gemstash::DB::Version.insert_by_spec(gem_id, spec)
+        Gemstash::DB::Dependency.insert_by_spec(version_id, spec)
+      end
+    end
+
+    def invalidate_cache
+      gemstash_env.cache.invalidate_gem("private", gem.spec.name)
+    end
+  end
+
+  unless Gem::Requirement.new(">= 2.4").satisfied_by?(Gem::Version.new(Gem::VERSION))
+    require "tempfile"
+
+    # Adds support for legacy versions of RubyGems
+    module LegacyRubyGemsSupport
+      def self.included(base)
+        base.class_eval do
+          alias_method :push_without_cleanup, :push
+          remove_method :push
+          remove_method :gem
+        end
+      end
+
+      def push
+        push_without_cleanup
+      ensure
+        cleanup
+      end
+
+    private
+
+      def gem
+        @gem ||= begin
+          @tempfile = Tempfile.new("gemstash-gem")
+          @tempfile.write(@content)
+          @tempfile.flush
+          Gem::Package.new(@tempfile.path)
+        end
+      end
+
+      def cleanup
+        return unless @tempfile
+        @tempfile.close
+        @tempfile.unlink
+      end
+    end
+
+    GemPusher.send(:include, LegacyRubyGemsSupport)
+  end
+end

data/lib/gemstash/gem_source.rb

@@ -0,0 +1,37 @@
+require "gemstash"
+require "forwardable"
+
+module Gemstash
+  #:nodoc:
+  module GemSource
+    autoload :DependencyCaching, "gemstash/gem_source/dependency_caching"
+    autoload :PrivateSource,     "gemstash/gem_source/private_source"
+    autoload :RackMiddleware,    "gemstash/gem_source/rack_middleware"
+    autoload :RedirectSource,    "gemstash/gem_source/upstream_source"
+    autoload :RubygemsSource,    "gemstash/gem_source/upstream_source"
+    autoload :UpstreamSource,    "gemstash/gem_source/upstream_source"
+
+    def self.sources
+      @sources ||= [
+        Gemstash::GemSource::PrivateSource,
+        Gemstash::GemSource::RedirectSource,
+        Gemstash::GemSource::UpstreamSource,
+        Gemstash::GemSource::RubygemsSource
+      ]
+    end
+
+    # Base GemSource for some common utilities.
+    class Base
+      extend Forwardable
+      extend Gemstash::Logging
+      include Gemstash::Logging
+
+      def_delegators :@app, :cache_control, :content_type, :env, :halt,
+        :headers, :http_client_for, :params, :redirect, :request
+
+      def initialize(app)
+        @app = app
+      end
+    end
+  end
+end

data/lib/gemstash/gem_source/dependency_caching.rb

@@ -0,0 +1,40 @@
+module Gemstash
+  module GemSource
+    # Module for caching dependencies in a GemSource.
+    module DependencyCaching
+      API_REQUEST_LIMIT = 200
+
+      def serve_dependencies
+        gems = gems_from_params
+
+        if gems.length > API_REQUEST_LIMIT
+          halt 422, "Too many gems (use --full-index instead)"
+        end
+
+        content_type "application/octet-stream"
+        Marshal.dump dependencies.fetch(gems)
+      end
+
+      def serve_dependencies_json
+        gems = gems_from_params
+
+        if gems.length > API_REQUEST_LIMIT
+          halt 422, {
+            "error" => "Too many gems (use --full-index instead)",
+            "code"  => 422
+          }.to_json
+        end
+
+        content_type "application/json;charset=UTF-8"
+        dependencies.fetch(gems).to_json
+      end
+
+    private
+
+      def gems_from_params
+        halt(200) if params[:gems].nil? || params[:gems].empty?
+        params[:gems].split(",").uniq
+      end
+    end
+  end
+end

data/lib/gemstash/gem_source/private_source.rb

@@ -0,0 +1,139 @@
+require "gemstash"
+
+module Gemstash
+  module GemSource
+    # GemSource for privately stored gems.
+    class PrivateSource < Gemstash::GemSource::Base
+      include Gemstash::GemSource::DependencyCaching
+      include Gemstash::Env::Helper
+
+      def self.rack_env_rewriter
+        @rack_env_rewriter ||= Gemstash::RackEnvRewriter.new(%r{\A/private})
+      end
+
+      def self.matches?(env)
+        rewriter = rack_env_rewriter.for(env)
+        return false unless rewriter.matches?
+        rewriter.rewrite
+        true
+      end
+
+      def serve_root
+        halt 403, "Not yet supported"
+      end
+
+      def serve_add_gem
+        authenticated("Gemstash Private Gems") do
+          auth = request.env["HTTP_AUTHORIZATION"]
+          gem = request.body.read
+          Gemstash::GemPusher.new(auth, gem).push
+        end
+      end
+
+      def serve_yank
+        authenticated("Gemstash Private Gems") do
+          auth = request.env["HTTP_AUTHORIZATION"]
+          gem_name = params[:gem_name]
+          Gemstash::GemYanker.new(auth, gem_name, slug_param).yank
+        end
+      end
+
+      def serve_unyank
+        authenticated("Gemstash Private Gems") do
+          auth = request.env["HTTP_AUTHORIZATION"]
+          gem_name = params[:gem_name]
+          Gemstash::GemUnyanker.new(auth, gem_name, slug_param).unyank
+        end
+      end
+
+      def serve_add_spec_json
+        halt 403, "Not yet supported"
+      end
+
+      def serve_remove_spec_json
+        halt 403, "Not yet supported"
+      end
+
+      def serve_names
+        halt 403, "Not yet supported"
+      end
+
+      def serve_versions
+        halt 403, "Not yet supported"
+      end
+
+      def serve_info(name)
+        halt 403, "Not yet supported"
+      end
+
+      def serve_marshal(id)
+        gem_full_name = id.sub(/\.gemspec\.rz\z/, "")
+        gem = fetch_gem(gem_full_name)
+        halt 404 unless gem.exist?(:spec)
+        content_type "application/octet-stream"
+        gem.load(:spec).content(:spec)
+      end
+
+      def serve_actual_gem(id)
+        halt 403, "Not yet supported"
+      end
+
+      def serve_gem(id)
+        gem_full_name = id.sub(/\.gem\z/, "")
+        gem = fetch_gem(gem_full_name)
+        content_type "application/octet-stream"
+        gem.content(:gem)
+      end
+
+      def serve_latest_specs
+        halt 403, "Not yet supported"
+      end
+
+      def serve_specs
+        content_type "application/octet-stream"
+        Gemstash::SpecsBuilder.all
+      end
+
+      def serve_prerelease_specs
+        content_type "application/octet-stream"
+        Gemstash::SpecsBuilder.prerelease
+      end
+
+    private
+
+      def slug_param
+        version = params[:version]
+        platform = params[:platform]
+
+        if platform.to_s.empty?
+          version
+        else
+          "#{version}-#{platform}"
+        end
+      end
+
+      def authenticated(realm)
+        yield
+      rescue Gemstash::NotAuthorizedError => e
+        headers["WWW-Authenticate"] = "Basic realm=\"#{realm}\""
+        halt 401, e.message
+      end
+
+      def dependencies
+        @dependencies ||= Gemstash::Dependencies.for_private
+      end
+
+      def storage
+        @storage ||= Gemstash::Storage.for("private").for("gems")
+      end
+
+      def fetch_gem(gem_full_name)
+        gem = storage.resource(gem_full_name)
+        halt 404 unless gem.exist?(:gem)
+        gem.load(:gem)
+        halt 403, "That gem has been yanked" unless gem.properties[:indexed]
+        gem
+      end
+    end
+  end
+end

data/lib/gemstash/gem_source/rack_middleware.rb

@@ -0,0 +1,22 @@
+require "gemstash"
+
+module Gemstash
+  module GemSource
+    # Rack middleware to detect the gem source from the URL.
+    class RackMiddleware
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        Gemstash::GemSource.sources.each do |source|
+          next unless source.matches?(env)
+          env["gemstash.gem_source"] = source
+          break
+        end
+
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/gemstash/gem_source/upstream_source.rb

@@ -0,0 +1,183 @@
+require "gemstash"
+require "cgi"
+
+module Gemstash
+  module GemSource
+    # GemSource that purely redirects to the upstream server.
+    class RedirectSource < Gemstash::GemSource::Base
+      def self.rack_env_rewriter
+        @rack_env_rewriter ||= Gemstash::RackEnvRewriter.new(%r{\A/redirect/(?<upstream_url>[^/]+)})
+      end
+
+      def self.matches?(env)
+        rewriter = rack_env_rewriter.for(env)
+        return false unless rewriter.matches?
+        rewriter.rewrite
+        env["gemstash.upstream"] = rewriter.captures["upstream_url"]
+        capture_user_agent(env)
+        true
+      end
+
+      def self.capture_user_agent(env)
+        env["gemstash.user-agent"] = env["HTTP_USER_AGENT"]
+      end
+
+      def serve_root
+        cache_control :public, :max_age => 31_536_000
+        redirect upstream.url(nil, request.query_string)
+      end
+
+      def serve_add_gem
+        halt 403, "Cannot add gem to an upstream server!"
+      end
+
+      def serve_yank
+        halt 403, "Cannot yank from an upstream server!"
+      end
+
+      def serve_unyank
+        halt 403, "Cannot unyank from an upstream server!"
+      end
+
+      def serve_add_spec_json
+        halt 403, "Cannot add spec to an upstream server!"
+      end
+
+      def serve_remove_spec_json
+        halt 403, "Cannot remove spec from an upstream server!"
+      end
+
+      def serve_dependencies
+        redirect upstream.url("api/v1/dependencies", request.query_string)
+      end
+
+      def serve_dependencies_json
+        redirect upstream.url("api/v1/dependencies.json", request.query_string)
+      end
+
+      def serve_names
+        redirect upstream.url("names", request.query_string)
+      end
+
+      def serve_versions
+        redirect upstream.url("versions", request.query_string)
+      end
+
+      def serve_info(name)
+        redirect upstream.url("info/#{name}", request.query_string)
+      end
+
+      def serve_marshal(id)
+        redirect upstream.url("quick/Marshal.4.8/#{id}", request.query_string)
+      end
+
+      def serve_actual_gem(id)
+        redirect upstream.url("fetch/actual/gem/#{id}", request.query_string)
+      end
+
+      def serve_gem(id)
+        redirect upstream.url("gems/#{id}", request.query_string)
+      end
+
+      def serve_latest_specs
+        redirect upstream.url("latest_specs.4.8.gz", request.query_string)
+      end
+
+      def serve_specs
+        redirect upstream.url("specs.4.8.gz", request.query_string)
+      end
+
+      def serve_prerelease_specs
+        redirect upstream.url("prerelease_specs.4.8.gz", request.query_string)
+      end
+
+    private
+
+      def upstream
+        @upstream ||= Gemstash::Upstream.new(env["gemstash.upstream"],
+          user_agent: env["gemstash.user-agent"])
+      end
+    end
+
+    # GemSource for gems in an upstream server.
+    class UpstreamSource < Gemstash::GemSource::RedirectSource
+      include Gemstash::GemSource::DependencyCaching
+      include Gemstash::Env::Helper
+
+      def self.rack_env_rewriter
+        @rack_env_rewriter ||= Gemstash::RackEnvRewriter.new(%r{\A/upstream/(?<upstream_url>[^/]+)})
+      end
+
+      def serve_marshal(id)
+        serve_cached(id, :spec)
+      end
+
+      def serve_gem(id)
+        serve_cached(id, :gem)
+      end
+
+    private
+
+      def serve_cached(id, key)
+        gem = fetch_gem(id, key)
+        headers.update(gem.properties[:headers][key]) if gem.properties[:headers] && gem.properties[:headers][key]
+        gem.content(key)
+      rescue Gemstash::WebError => e
+        halt e.code
+      end
+
+      def dependencies
+        @dependencies ||= begin
+          http_client = http_client_for(upstream)
+          Gemstash::Dependencies.for_upstream(upstream, http_client)
+        end
+      end
+
+      def storage
+        @storage ||= Gemstash::Storage.for("gem_cache")
+        @storage.for(upstream.host_id)
+      end
+
+      def gem_fetcher
+        @gem_fetcher ||= Gemstash::GemFetcher.new(http_client_for(upstream))
+      end
+
+      def fetch_gem(id, key)
+        gem_name = Gemstash::Upstream::GemName.new(upstream, id)
+        gem_resource = storage.resource(gem_name.name)
+        if gem_resource.exist?(key)
+          fetch_local_gem(gem_name, gem_resource, key)
+        else
+          fetch_remote_gem(gem_name, gem_resource, key)
+        end
+      end
+
+      def fetch_local_gem(gem_name, gem_resource, key)
+        log.info "Gem #{gem_name.name} exists, returning cached #{key}"
+        gem_resource.load(key)
+      end
+
+      def fetch_remote_gem(gem_name, gem_resource, key)
+        log.info "Gem #{gem_name.name} is not cached, fetching #{key}"
+        gem_fetcher.fetch(gem_name.id, key) do |content, properties|
+          gem_resource.save({ key => content }, headers: { key => properties })
+        end
+      end
+    end
+
+    # GemSource for https://rubygems.org (specifically when defined by using the
+    # default upstream).
+    class RubygemsSource < Gemstash::GemSource::UpstreamSource
+      def self.matches?(env)
+        if env["HTTP_X_GEMFILE_SOURCE"].to_s.empty?
+          env["gemstash.upstream"] = env["gemstash.env"].config[:rubygems_url]
+        else
+          env["gemstash.upstream"] = env["HTTP_X_GEMFILE_SOURCE"]
+        end
+        capture_user_agent(env)
+
+        true
+      end
+    end
+  end
+end