gemstash 1.0.0.pre.1-java → 2.6.0-java

data/lib/gemstash/env.rb

@@ -1,8 +1,12 @@
+# frozen_string_literal: true
+
 require "gemstash"
+require "active_support/core_ext/file/atomic"
 require "dalli"
 require "fileutils"
 require "sequel"
 require "uri"
+require "pathname"
 
 module Gemstash
   # Storage for application-wide variables and configuration.
@@ -14,7 +18,7 @@ module Gemstash
 
     # Little module to provide easy access to the current Gemstash::Env.
     module Helper
-    private
+      private # rubocop:disable Layout/AccessModifierIndentation
 
       def gemstash_env
         Gemstash::Env.current
@@ -30,6 +34,7 @@ module Gemstash
 
       def call(env)
         env["gemstash.env"] = @gemstash_env
+        Gemstash::Env.current = @gemstash_env
         @app.call(env)
       end
     end
@@ -46,6 +51,7 @@ module Gemstash
 
     def self.current
       raise EnvNotSetError unless Thread.current[:gemstash_env]
+
       Thread.current[:gemstash_env]
     end
 
@@ -54,11 +60,9 @@ module Gemstash
     end
 
     def self.daemonized?
-      if @daemonized.nil?
-        raise "Daemonized hasn't been set yet!"
-      else
-        @daemonized
-      end
+      raise "Daemonized hasn't been set yet!" if @daemonized.nil?
+
+      @daemonized
     end
 
     def self.daemonized=(value)
@@ -98,8 +102,33 @@ module Gemstash
       File.join(base_path, path)
     end
 
+    def log_file
+      if config[:log_file] == :stdout
+        $stdout
+      else
+        base_file(config[:log_file] || "server.log")
+      end
+    end
+
+    def pidfile
+      if config[:pidfile]
+        pathname = Pathname.new(config[:pidfile])
+        if pathname.relative?
+          base_file(pathname.to_s)
+        else
+          pathname.to_s
+        end
+      else
+        base_file("puma.pid")
+      end
+    end
+
+    def atomic_write(file, &block)
+      File.atomic_write(file, File.dirname(file), &block)
+    end
+
     def rackup
-      File.expand_path("../config.ru", __FILE__)
+      File.expand_path("config.ru", __dir__)
     end
 
     def db
@@ -108,13 +137,13 @@ module Gemstash
         when "sqlite3"
           db_path = base_file("gemstash.db")
 
-          if RUBY_PLATFORM == "java"
-            db = Sequel.connect("jdbc:sqlite:#{db_path}", max_connections: 1)
+          db = if RUBY_PLATFORM == "java"
+            Sequel.connect("jdbc:sqlite:#{db_path}", config.database_connection_config)
           else
-            db = Sequel.connect("sqlite://#{URI.escape(db_path)}", max_connections: 1)
+            Sequel.connect("sqlite://#{CGI.escape(db_path)}", config.database_connection_config)
           end
-        when "postgres"
-          db = Sequel.connect(config[:db_url])
+        when "postgres", "mysql", "mysql2"
+          db = Sequel.connect(config[:db_url], config.database_connection_config)
         else
           raise "Unsupported DB adapter: '#{config[:db_adapter]}'"
         end
@@ -126,7 +155,7 @@ module Gemstash
 
     def self.migrate(db)
       Sequel.extension :migration
-      migrations_dir = File.expand_path("../migrations", __FILE__)
+      migrations_dir = File.expand_path("migrations", __dir__)
       Sequel::Migrator.run(db, migrations_dir, :use_transactions => true)
     end
 
@@ -141,6 +170,8 @@ module Gemstash
           Gemstash::LruReduxClient.new
         when "memcached"
           Dalli::Client.new(config[:memcached_servers])
+        when "redis"
+          Gemstash::RedisClient.new(config[:redis_servers])
         else
           raise "Invalid cache client: '#{config[:cache_type]}'"
         end

data/lib/gemstash/gem_fetcher.rb

@@ -1,12 +1,14 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "set"
 
 module Gemstash
-  #:nodoc:
+  # :nodoc:
   class GemFetcher
     def initialize(http_client)
       @http_client = http_client
-      @valid_headers = Set.new(["etag", "content-type", "content-length", "last-modified"])
+      @valid_headers = Set.new(%w[etag content-type content-length last-modified])
     end
 
     def fetch(gem_id, type, &block)
@@ -31,7 +33,7 @@ module Gemstash
     end
 
     def filter_headers(headers)
-      headers.inject({}) do|properties, (key, value)|
+      headers.inject({}) do |properties, (key, value)|
         properties[key.downcase] = value if @valid_headers.include?(key.downcase)
         properties
       end

data/lib/gemstash/gem_pusher.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "rubygems/package"
 require "stringio"
 
-#:nodoc:
+# :nodoc:
 module Gemstash
   # Class that supports pushing a new gem to the private repository of gems.
   class GemPusher
@@ -16,12 +18,17 @@ module Gemstash
     class YankedVersionError < ExistingVersionError
     end
 
-    def initialize(auth_key, content)
-      @auth_key = auth_key
+    def self.serve(app)
+      gem = app.request.body.read
+      new(app.auth, gem).serve
+    end
+
+    def initialize(auth, content)
+      @auth = auth
       @content = content
     end
 
-    def push
+    def serve
       check_auth
       store_gem
       store_gemspec
@@ -44,10 +51,15 @@ module Gemstash
     end
 
     def check_auth
-      Gemstash::Authorization.check(@auth_key, "push")
+      @auth.check("push")
     end
 
     def store_gem
+      resource_exist = storage.resource(full_name).exist?
+      resource_is_indexed = storage.resource(full_name).properties[:indexed] if resource_exist
+
+      raise ExistingVersionError, "Cannot push to an existing version!" if resource_exist && resource_is_indexed
+
       storage.resource(full_name).save({ gem: @content }, indexed: true)
     end
 
@@ -64,14 +76,8 @@ module Gemstash
       gemstash_env.db.transaction do
         gem_id = Gemstash::DB::Rubygem.find_or_insert(spec)
         existing = Gemstash::DB::Version.find_by_spec(gem_id, spec)
-
-        if existing
-          if existing.indexed
-            raise ExistingVersionError, "Cannot push to an existing version!"
-          else
-            raise YankedVersionError, "Cannot push to a yanked version!"
-          end
-        end
+        raise ExistingVersionError, "Cannot push to an existing version!" if existing && existing.indexed
+        raise YankedVersionError, "Cannot push to a yanked version!" if existing && !existing.indexed
 
         version_id = Gemstash::DB::Version.insert_by_spec(gem_id, spec)
         Gemstash::DB::Dependency.insert_by_spec(version_id, spec)
@@ -90,14 +96,14 @@ module Gemstash
     module LegacyRubyGemsSupport
       def self.included(base)
         base.class_eval do
-          alias_method :push_without_cleanup, :push
-          remove_method :push
+          alias_method :serve_without_cleanup, :serve
+          remove_method :serve
           remove_method :gem
         end
       end
 
-      def push
-        push_without_cleanup
+      def serve
+        serve_without_cleanup
       ensure
         cleanup
       end
@@ -115,11 +121,12 @@ module Gemstash
 
       def cleanup
         return unless @tempfile
+
         @tempfile.close
         @tempfile.unlink
       end
     end
 
-    GemPusher.send(:include, LegacyRubyGemsSupport)
+    GemPusher.include LegacyRubyGemsSupport
   end
 end

data/lib/gemstash/gem_source/dependency_caching.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gemstash
   module GemSource
     # Module for caching dependencies in a GemSource.
@@ -7,9 +9,7 @@ module Gemstash
       def serve_dependencies
         gems = gems_from_params
 
-        if gems.length > API_REQUEST_LIMIT
-          halt 422, "Too many gems (use --full-index instead)"
-        end
+        halt 422, "Too many gems (use --full-index instead)" if gems.length > API_REQUEST_LIMIT
 
         content_type "application/octet-stream"
         Marshal.dump dependencies.fetch(gems)
@@ -21,7 +21,7 @@ module Gemstash
         if gems.length > API_REQUEST_LIMIT
           halt 422, {
             "error" => "Too many gems (use --full-index instead)",
-            "code"  => 422
+            "code" => 422
           }.to_json
         end
 

data/lib/gemstash/gem_source/private_source.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
 module Gemstash
@@ -6,6 +8,7 @@ module Gemstash
     class PrivateSource < Gemstash::GemSource::Base
       include Gemstash::GemSource::DependencyCaching
       include Gemstash::Env::Helper
+      attr_accessor :auth
 
       def self.rack_env_rewriter
         @rack_env_rewriter ||= Gemstash::RackEnvRewriter.new(%r{\A/private})
@@ -14,6 +17,7 @@ module Gemstash
       def self.matches?(env)
         rewriter = rack_env_rewriter.for(env)
         return false unless rewriter.matches?
+
         rewriter.rewrite
         true
       end
@@ -23,27 +27,11 @@ module Gemstash
       end
 
       def serve_add_gem
-        authenticated("Gemstash Private Gems") do
-          auth = request.env["HTTP_AUTHORIZATION"]
-          gem = request.body.read
-          Gemstash::GemPusher.new(auth, gem).push
-        end
+        protected(Gemstash::GemPusher)
       end
 
       def serve_yank
-        authenticated("Gemstash Private Gems") do
-          auth = request.env["HTTP_AUTHORIZATION"]
-          gem_name = params[:gem_name]
-          Gemstash::GemYanker.new(auth, gem_name, slug_param).yank
-        end
-      end
-
-      def serve_unyank
-        authenticated("Gemstash Private Gems") do
-          auth = request.env["HTTP_AUTHORIZATION"]
-          gem_name = params[:gem_name]
-          Gemstash::GemUnyanker.new(auth, gem_name, slug_param).unyank
-        end
+        protected(Gemstash::GemYanker)
       end
 
       def serve_add_spec_json
@@ -59,7 +47,7 @@ module Gemstash
       end
 
       def serve_versions
-        halt 403, "Not yet supported"
+        halt 404, "Not yet supported"
       end
 
       def serve_info(name)
@@ -67,11 +55,14 @@ module Gemstash
       end
 
       def serve_marshal(id)
-        gem_full_name = id.sub(/\.gemspec\.rz\z/, "")
-        gem = fetch_gem(gem_full_name)
-        halt 404 unless gem.exist?(:spec)
-        content_type "application/octet-stream"
-        gem.load(:spec).content(:spec)
+        authorization.protect(self) do
+          auth.check("fetch") if gemstash_env.config[:protected_fetch]
+          gem_full_name = id.delete_suffix(".gemspec.rz")
+          gem = fetch_gem(gem_full_name)
+          halt 404 unless gem.exist?(:spec)
+          content_type "application/octet-stream"
+          gem.content(:spec)
+        end
       end
 
       def serve_actual_gem(id)
@@ -79,44 +70,38 @@ module Gemstash
       end
 
       def serve_gem(id)
-        gem_full_name = id.sub(/\.gem\z/, "")
-        gem = fetch_gem(gem_full_name)
-        content_type "application/octet-stream"
-        gem.content(:gem)
+        authorization.protect(self) do
+          auth.check("fetch") if gemstash_env.config[:protected_fetch]
+          gem_full_name = id.delete_suffix(".gem")
+          gem = fetch_gem(gem_full_name)
+          content_type "application/octet-stream"
+          gem.content(:gem)
+        end
       end
 
-      def serve_latest_specs
-        halt 403, "Not yet supported"
+      def serve_specs
+        params[:prerelease] = false
+        protected(Gemstash::SpecsBuilder)
       end
 
-      def serve_specs
-        content_type "application/octet-stream"
-        Gemstash::SpecsBuilder.all
+      def serve_latest_specs
+        params[:latest] = true
+        protected(Gemstash::SpecsBuilder)
       end
 
       def serve_prerelease_specs
-        content_type "application/octet-stream"
-        Gemstash::SpecsBuilder.prerelease
+        params[:prerelease] = true
+        protected(Gemstash::SpecsBuilder)
       end
 
     private
 
-      def slug_param
-        version = params[:version]
-        platform = params[:platform]
-
-        if platform.to_s.empty?
-          version
-        else
-          "#{version}-#{platform}"
-        end
+      def protected(servable)
+        authorization.protect(self) { servable.serve(self) }
       end
 
-      def authenticated(realm)
-        yield
-      rescue Gemstash::NotAuthorizedError => e
-        headers["WWW-Authenticate"] = "Basic realm=\"#{realm}\""
-        halt 401, e.message
+      def authorization
+        Gemstash::ApiKeyAuthorization
       end
 
       def dependencies
@@ -130,7 +115,6 @@ module Gemstash
       def fetch_gem(gem_full_name)
         gem = storage.resource(gem_full_name)
         halt 404 unless gem.exist?(:gem)
-        gem.load(:gem)
         halt 403, "That gem has been yanked" unless gem.properties[:indexed]
         gem
       end

data/lib/gemstash/gem_source/rack_middleware.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
 module Gemstash
@@ -11,6 +13,7 @@ module Gemstash
       def call(env)
         Gemstash::GemSource.sources.each do |source|
           next unless source.matches?(env)
+
           env["gemstash.gem_source"] = source
           break
         end

data/lib/gemstash/gem_source/upstream_source.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "cgi"
 
@@ -12,6 +14,7 @@ module Gemstash
       def self.matches?(env)
         rewriter = rack_env_rewriter.for(env)
         return false unless rewriter.matches?
+
         rewriter.rewrite
         env["gemstash.upstream"] = rewriter.captures["upstream_url"]
         capture_user_agent(env)
@@ -35,10 +38,6 @@ module Gemstash
         halt 403, "Cannot yank from an upstream server!"
       end
 
-      def serve_unyank
-        halt 403, "Cannot unyank from an upstream server!"
-      end
-
       def serve_add_spec_json
         halt 403, "Cannot add spec to an upstream server!"
       end
@@ -60,11 +59,11 @@ module Gemstash
       end
 
       def serve_versions
-        redirect upstream.url("versions", request.query_string)
+        redirect index_upstream.url("versions", request.query_string)
       end
 
       def serve_info(name)
-        redirect upstream.url("info/#{name}", request.query_string)
+        redirect index_upstream.url("info/#{name}", request.query_string)
       end
 
       def serve_marshal(id)
@@ -94,8 +93,21 @@ module Gemstash
     private
 
       def upstream
-        @upstream ||= Gemstash::Upstream.new(env["gemstash.upstream"],
-          user_agent: env["gemstash.user-agent"])
+        @upstream ||= Gemstash::Upstream.new(
+          env["gemstash.upstream"],
+          user_agent: env["gemstash.user-agent"]
+        )
+      end
+
+      def index_upstream
+        @index_upstream ||=
+          if upstream.uri.host == "rubygems.org"
+            uri = upstream.uri.dup
+            uri.host = "index.rubygems.org"
+            Gemstash::Upstream.new(uri, user_agent: upstream.user_agent)
+          else
+            upstream
+          end
       end
     end
 
@@ -116,16 +128,40 @@ module Gemstash
         serve_cached(id, :gem)
       end
 
+      def serve_latest_specs
+        http_client = http_client_for(upstream)
+        http_client.get("latest_specs.4.8.gz")
+      end
+
+      def serve_prerelease_specs
+        http_client = http_client_for(upstream)
+        http_client.get("prerelease_specs.4.8.gz")
+      end
+
+      def serve_specs
+        http_client = http_client_for(upstream)
+        http_client.get("specs.4.8.gz")
+      end
+
     private
 
-      def serve_cached(id, key)
-        gem = fetch_gem(id, key)
-        headers.update(gem.properties[:headers][key]) if gem.properties[:headers] && gem.properties[:headers][key]
-        gem.content(key)
+      def serve_cached(id, resource_type)
+        gem = fetch_gem(id, resource_type)
+        set_gem_headers(gem, resource_type)
+        gem.content(resource_type)
       rescue Gemstash::WebError => e
         halt e.code
       end
 
+      def set_gem_headers(gem, resource_type)
+        return unless gem.property?(:headers, resource_type)
+
+        gem_headers = gem.properties[:headers][resource_type]
+        headers["Content-Type"] = gem_headers["content-type"] if gem_headers.include?("content-type")
+        headers["Last-Modified"] = gem_headers["last-modified"] if gem_headers.include?("last-modified")
+        headers["ETag"] = gem_headers["etag"] if gem_headers.include?("etag")
+      end
+
       def dependencies
         @dependencies ||= begin
           http_client = http_client_for(upstream)
@@ -142,25 +178,33 @@ module Gemstash
         @gem_fetcher ||= Gemstash::GemFetcher.new(http_client_for(upstream))
       end
 
-      def fetch_gem(id, key)
+      def fetch_gem(id, resource_type)
         gem_name = Gemstash::Upstream::GemName.new(upstream, id)
         gem_resource = storage.resource(gem_name.name)
-        if gem_resource.exist?(key)
-          fetch_local_gem(gem_name, gem_resource, key)
+        if gem_resource.exist?(resource_type)
+          fetch_local_gem(gem_name, gem_resource, resource_type)
         else
-          fetch_remote_gem(gem_name, gem_resource, key)
+          fetch_remote_gem(gem_name, gem_resource, resource_type)
         end
       end
 
-      def fetch_local_gem(gem_name, gem_resource, key)
-        log.info "Gem #{gem_name.name} exists, returning cached #{key}"
-        gem_resource.load(key)
+      def fetch_local_gem(gem_name, gem_resource, resource_type)
+        log.info "Gem #{gem_name.name} exists, returning cached #{resource_type}"
+        gem_resource
       end
 
-      def fetch_remote_gem(gem_name, gem_resource, key)
-        log.info "Gem #{gem_name.name} is not cached, fetching #{key}"
-        gem_fetcher.fetch(gem_name.id, key) do |content, properties|
-          gem_resource.save({ key => content }, headers: { key => properties })
+      def fetch_remote_gem(gem_name, gem_resource, resource_type)
+        log.info "Gem #{gem_name.name} is not cached, fetching #{resource_type}"
+        gem_fetcher.fetch(gem_name.id, resource_type) do |content, properties|
+          resource_properties = {
+            upstream: upstream.to_s,
+            gem_name: gem_name.name,
+            headers: { resource_type => properties }
+          }
+
+          gem = gem_resource.save({ resource_type => content }, resource_properties)
+          Gemstash::DB::CachedRubygem.store(upstream, gem_name, resource_type)
+          gem
         end
       end
     end
@@ -169,13 +213,13 @@ module Gemstash
     # default upstream).
     class RubygemsSource < Gemstash::GemSource::UpstreamSource
       def self.matches?(env)
-        if env["HTTP_X_GEMFILE_SOURCE"].to_s.empty?
-          env["gemstash.upstream"] = env["gemstash.env"].config[:rubygems_url]
+        env["gemstash.upstream"] = if env["gemstash.env"].config[:ignore_gemfile_source] || env["HTTP_X_GEMFILE_SOURCE"].to_s.empty?
+          env["gemstash.env"].config[:rubygems_url]
         else
-          env["gemstash.upstream"] = env["HTTP_X_GEMFILE_SOURCE"]
+          env["HTTP_X_GEMFILE_SOURCE"]
         end
-        capture_user_agent(env)
 
+        capture_user_agent(env)
         true
       end
     end

data/lib/gemstash/gem_source.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "forwardable"
 
 module Gemstash
-  #:nodoc:
+  # :nodoc:
   module GemSource
     autoload :DependencyCaching, "gemstash/gem_source/dependency_caching"
     autoload :PrivateSource,     "gemstash/gem_source/private_source"
@@ -27,7 +29,7 @@ module Gemstash
       include Gemstash::Logging
 
       def_delegators :@app, :cache_control, :content_type, :env, :halt,
-        :headers, :http_client_for, :params, :redirect, :request
+                     :headers, :http_client_for, :params, :redirect, :request
 
       def initialize(app)
         @app = app

data/lib/gemstash/gem_yanker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
 module Gemstash
@@ -17,13 +19,19 @@ module Gemstash
     class YankedVersionError < StandardError
     end
 
-    def initialize(auth_key, gem_name, slug)
-      @auth_key = auth_key
+    def self.serve(app)
+      gem_name = app.params[:gem_name]
+      slug = Gemstash::DB::Version.slug(app.params)
+      new(app.auth, gem_name, slug).serve
+    end
+
+    def initialize(auth, gem_name, slug)
+      @auth = auth
       @gem_name = gem_name
       @slug = slug
     end
 
-    def yank
+    def serve
       check_auth
       update_database
       invalidate_cache
@@ -40,15 +48,17 @@ module Gemstash
     end
 
     def check_auth
-      Gemstash::Authorization.check(@auth_key, "yank")
+      @auth.check("yank")
     end
 
     def update_database
       gemstash_env.db.transaction do
         raise UnknownGemError, "Cannot yank an unknown gem!" unless Gemstash::DB::Rubygem[name: @gem_name]
+
         version = Gemstash::DB::Version.find_by_full_name(full_name)
         raise UnknownVersionError, "Cannot yank an unknown version!" unless version
         raise YankedVersionError, "Cannot yank an already yanked version!" unless version.indexed
+
         version.deindex
         storage.resource(version.storage_id).update_properties(indexed: false)
       end