gemstash 1.0.4 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4c86611f04efabc000653d70884cd2cad313b98a
-  data.tar.gz: 9d720ecfb8eafa60ec05829d8beb2c92b50b88c6
+SHA256:
+  metadata.gz: 653074694a6da8d9a0d09699993e21229de4676097e5cc930ffbd6110898a6cf
+  data.tar.gz: 62e548f1804f1da0d2709fb725186ad8a811f608c8e9d6a5f2567307295136f6
 SHA512:
-  metadata.gz: 99decbbafb9c1062cbdd600f7e45cec081121f6925dd4171ed2304899e102cb134805e1dd23182c2f6987406346663a19c9db49cbc3d95c6cbb781027a21bc6a
-  data.tar.gz: 05de14fa477ee16db9c0c1f249cf6c1efad0e5f1785a5154ae9e1ad23fcc3b5cde465cc9b24218132ca39fcda7cf28ebe1f243abf3d53cc2a67aae07204ff431
+  metadata.gz: d378ab9b7674da573af0fd96fd0c22d8070cdc0f0e7e02c9efd4f22cff6723e28892b28067c64ed740ad18f2add456542ff16274a5f6c318840afb3d35eb2737
+  data.tar.gz: 61d2a82b279c024f9e89d3aa037c967af91a47ae99b4f28da26e93a759cf9e7459d09738385f9715b29191fd92046c56f05cbc01428a0a63977ff01bbc89084b

data/CHANGELOG.md

@@ -1,3 +1,36 @@
+## 1.1.0 (2017-07-31)
+
+### Bugfixes
+
+  - Gracefully handle empty configuration files ([#97](https://github.com/bundler/gemstash/pull/97), [@rjocoleman](https://github.com/rjocoleman))
+  - Remove bundler-audit since we don't commit our Gemfile.lock ([#98](https://github.com/bundler/gemstash/pull/98), [@smellsblue](https://github.com/smellsblue))
+  - Clarify what is being cached for 30 minutes ([#108](https://github.com/bundler/gemstash/pull/108), [@Nowaker](https://github.com/Nowaker))
+  - Update documentation ([#112](https://github.com/bundler/gemstash/pull/112), [@smellsblue](https://github.com/smellsblue))
+  - Integration tests for searching for gems ([#113](https://github.com/bundler/gemstash/pull/113), [@smellsblue](https://github.com/smellsblue))
+  - Add `ruby-head` Gemfile for Travis ([#121](https://github.com/bundler/gemstash/pull/121), [@olleolleolle](https://github.com/olleolleolle))
+  - Make all Pandoc filters executable ([#122](https://github.com/bundler/gemstash/pull/122), [@olleolleolle](https://github.com/olleolleolle))
+  - Fix `rake doc` to work with newer versions of Pandoc ([#124](https://github.com/bundler/gemstash/pull/124), [@smellsblue](https://github.com/smellsblue))
+  - Use `curl` since `unyank` is removed in newer RubyGems versions ([#125](https://github.com/bundler/gemstash/pull/125), [@smellsblue](https://github.com/smellsblue))
+  - Touch up docs and update CHANGELOG ([#128](https://github.com/bundler/gemstash/pull/128), [@smellsblue](https://github.com/smellsblue))
+  - Update Travis to use ruby `2.3.3` ([#127](https://github.com/bundler/gemstash/pull/127), [@olleolleolle](https://github.com/olleolleolle))
+  - Add Ruby `2.4.0` to Travis ([#132](https://github.com/bundler/gemstash/pull/132), [@thedrow](https://github.com/thedrow))
+
+### Features
+
+  - Add support for mysql2 adapter ([#71](https://github.com/bundler/gemstash/pull/71), [@chriseckhardt](https://github.com/chriseckhardt))
+  - Allow logging to a different file ([#74](https://github.com/bundler/gemstash/pull/74), [@mrchucho](https://github.com/mrchucho))
+  - Document temporary protected fetch solution ([#80](https://github.com/bundler/gemstash/pull/80), [@taoza](https://github.com/taoza))
+  - Make gem fetch timeout configurable ([#81](https://github.com/bundler/gemstash/pull/81), [@midwire](https://github.com/midwire))
+  - Document fallback timeout for when Gemstash is down ([#88](https://github.com/bundler/gemstash/pull/88), [@parndt](https://github.com/parndt))
+  - Allow ERB parsed config file via `.erb` extension ([#90](https://github.com/bundler/gemstash/pull/90), [@jiexinhuang](https://github.com/jiexinhuang), [@rjocoleman](https://github.com/rjocoleman))
+  - Improve code climate ([#92](https://github.com/bundler/gemstash/pull/92), [@smellsblue](https://github.com/smellsblue))
+  - Refactor authorization ([#93](https://github.com/bundler/gemstash/pull/93), [@smellsblue](https://github.com/smellsblue), [@rjocoleman](https://github.com/rjocoleman))
+  - Add protected fetch for private gems ([#94](https://github.com/bundler/gemstash/pull/94), [@rjocoleman](https://github.com/rjocoleman))
+  - Add Ruby Together call to action ([#116](https://github.com/bundler/gemstash/pull/116), [@mrb](https://github.com/mrb))
+  - Configurable options to `Sequel.connect` ([#123](https://github.com/bundler/gemstash/pull/123), [@olleolleolle](https://github.com/olleolleolle))
+  - Support the `latest_specs.4.8.gz` endpoint for private gems ([#131](https://github.com/bundler/gemstash/pull/131), [@randycoulman](https://github.com/randycoulman))
+  - Merge in the latest 1.0 stable ([#136](https://github.com/bundler/gemstash/pull/136), [@smellsblue](https://github.com/smellsblue), [@randycoulman](https://github.com/randycoulman))
+
 ## 1.0.4 (2017-01-27)
 
 ### Features

data/lib/gemstash.rb

@@ -1,5 +1,6 @@
 #:nodoc:
 module Gemstash
+  autoload :ApiKeyAuthorization, "gemstash/api_key_authorization"
   autoload :Authorization,       "gemstash/authorization"
   autoload :DB,                  "gemstash/db"
   autoload :Cache,               "gemstash/cache"
@@ -12,6 +13,7 @@ module Gemstash
   autoload :GemSource,           "gemstash/gem_source"
   autoload :GemUnyanker,         "gemstash/gem_unyanker"
   autoload :GemYanker,           "gemstash/gem_yanker"
+  autoload :Health,              "gemstash/health"
   autoload :HTTPClient,          "gemstash/http_client"
   autoload :Logging,             "gemstash/logging"
   autoload :LruReduxClient,      "gemstash/cache"

data/lib/gemstash/api_key_authorization.rb

@@ -0,0 +1,29 @@
+require "gemstash"
+
+module Gemstash
+  # Authorize actions via an API key and Gemstash::Authorization.
+  class ApiKeyAuthorization
+    def initialize(key)
+      @key = key
+    end
+
+    def self.protect(app, &block)
+      key = parse_authorization(app.request.env)
+      app.auth = new(key)
+      yield
+    rescue Gemstash::NotAuthorizedError => e
+      app.headers["WWW-Authenticate"] = "Basic realm=\"Gemstash Private Gems\""
+      app.halt 401, e.message
+    end
+
+    def self.parse_authorization(request_env)
+      http_auth = Rack::Auth::Basic::Request.new(request_env)
+      return http_auth.credentials.first if http_auth.provided? && http_auth.basic?
+      request_env["HTTP_AUTHORIZATION"]
+    end
+
+    def check(permission)
+      Gemstash::Authorization.check(@key, permission)
+    end
+  end
+end

data/lib/gemstash/authorization.rb

@@ -9,7 +9,7 @@ module Gemstash
   class Authorization
     extend Gemstash::Env::Helper
     extend Gemstash::Logging
-    VALID_PERMISSIONS = %w(push yank unyank).freeze
+    VALID_PERMISSIONS = %w(push yank unyank fetch).freeze
 
     def self.authorize(auth_key, permissions)
       raise "Authorization key is required!" if auth_key.to_s.strip.empty?
@@ -83,5 +83,9 @@ module Gemstash
     def unyank?
       can?("unyank")
     end
+
+    def fetch?
+      can?("fetch")
+    end
   end
 end

data/lib/gemstash/cli/authorize.rb

@@ -20,7 +20,7 @@ module Gemstash
     private
 
       def setup_logging
-        Gemstash::Logging.setup_logger(gemstash_env.base_file("server.log"))
+        Gemstash::Logging.setup_logger(gemstash_env.log_file)
       end
 
       def remove_authorization

data/lib/gemstash/cli/setup.rb

@@ -22,6 +22,8 @@ module Gemstash
         ask_storage
         ask_cache
         ask_database
+        ask_protected_fetch
+        ask_timeout
         check_cache
         check_storage
         check_database
@@ -45,6 +47,22 @@ module Gemstash
         @cli.say "#{label}: #{gemstash_env.config[option]}"
       end
 
+      def ask_with_default(prompt, options, default)
+        raise "The options must all be lower case" if options.any? {|x| x.downcase != x }
+        result = nil
+        displayed_options = options.map {|x| x == default ? x.upcase : x }
+        prompt = "#{prompt} [#{displayed_options.join(", ")}]"
+
+        until result
+          result = @cli.ask prompt
+          result = result.downcase
+          result = default if result.empty?
+          result = nil unless options.include?(result)
+        end
+
+        result
+      end
+
       def ask_storage
         say_current_config(:base_path, "Current base path")
         path = @cli.ask "Where should files go? [~/.gemstash]", path: true
@@ -54,18 +72,8 @@ module Gemstash
 
       def ask_cache
         say_current_config(:cache_type, "Current cache")
-        options = %w(memory memcached)
-        cache = nil
-
-        until cache
-          cache = @cli.ask "Cache with what? [MEMORY, memcached]"
-          cache = cache.downcase
-          cache = "memory" if cache.empty?
-          cache = nil unless options.include?(cache)
-        end
-
-        @config[:cache_type] = cache
-        ask_memcached_details if cache == "memcached"
+        @config[:cache_type] = ask_with_default("Cache with what?", %w(memory memcached), "memory")
+        ask_memcached_details if @config[:cache_type] == "memcached"
       end
 
       def ask_memcached_details
@@ -77,27 +85,17 @@ module Gemstash
 
       def ask_database
         say_current_config(:db_adapter, "Current database adapter")
-        options = %w(sqlite3 postgres mysql)
-        database = nil
-
-        until database
-          database = @cli.ask "What database adapter? [SQLITE3, postgres, mysql]"
-          database = database.downcase
-          database = "sqlite3" if database.empty?
-          database = nil unless options.include?(database)
-        end
-
-        @config[:db_adapter] = database
-        ask_database_details(database) unless database == "sqlite3"
+        @config[:db_adapter] = ask_with_default("What database adapter?", %w(sqlite3 postgres mysql mysql2), "sqlite3")
+        ask_database_details(@config[:db_adapter]) unless @config[:db_adapter] == "sqlite3"
       end
 
       def ask_database_details(database)
         say_current_config(:db_url, "Current database url")
 
-        if RUBY_PLATFORM == "java"
-          default_value = "jdbc:#{database}:///gemstash"
+        default_value = if RUBY_PLATFORM == "java"
+          "jdbc:#{database}:///gemstash"
         else
-          default_value = "#{database}:///gemstash"
+          "#{database}:///gemstash"
         end
 
         url = @cli.ask "Where is the database? [#{default_value}]"
@@ -105,20 +103,27 @@ module Gemstash
         @config[:db_url] = url
       end
 
+      def ask_protected_fetch
+        say_current_config(:protected_fetch, "Protected Fetch enabled")
+
+        value = @cli.yes? "Use Protected Fetch for Private Gems? [y/N]"
+        value = Gemstash::Configuration::DEFAULTS[:protected_fetch] if value.is_a?(String) && value.empty?
+        @config[:protected_fetch] = value
+      end
+
+      def ask_timeout
+        say_current_config(:fetch_timeout, "Fetch timeout")
+        timeout = @cli.ask "How many seconds to wait when fetching a gem? [20]"
+        timeout = Gemstash::Configuration::DEFAULTS[:fetch_timeout] if timeout.to_i < 1
+        @config[:fetch_timeout] = timeout.to_i
+      end
+
       def check_cache
-        @cli.say "Checking that the cache is available"
-        with_new_config { gemstash_env.cache_client.alive! }
-      rescue => e
-        say_error "Cache error", e
-        raise Gemstash::CLI::Error.new(@cli, "The cache is not available")
+        try("cache") { gemstash_env.cache_client.alive! }
       end
 
       def check_database
-        @cli.say "Checking that the database is available"
-        with_new_config { gemstash_env.db.test_connection }
-      rescue => e
-        say_error "Database error", e
-        raise Gemstash::CLI::Error.new(@cli, "The database is not available")
+        try("database") { gemstash_env.db.test_connection }
       end
 
       def check_storage
@@ -168,6 +173,14 @@ module Gemstash
       ensure
         gemstash_env.reset
       end
+
+      def try(thing)
+        @cli.say "Checking that the #{thing} is available"
+        with_new_config { yield }
+      rescue => e
+        say_error "Error checking #{thing}", e
+        raise Gemstash::CLI::Error.new(@cli, "The #{thing} is not available")
+      end
     end
   end
 end

data/lib/gemstash/cli/start.rb

@@ -18,7 +18,7 @@ module Gemstash
 
       def setup_logging
         return unless daemonize?
-        Gemstash::Logging.setup_logger(gemstash_env.base_file("server.log"))
+        Gemstash::Logging.setup_logger(gemstash_env.log_file)
       end
 
       def store_daemonized

data/lib/gemstash/config.ru

@@ -10,4 +10,5 @@ end
 
 use Gemstash::Env::RackMiddleware, Gemstash::Env.current
 use Gemstash::GemSource::RackMiddleware
+use Gemstash::Health::RackMiddleware
 run Gemstash::Web.new(gemstash_env: Gemstash::Env.current)

data/lib/gemstash/configuration.rb

@@ -1,4 +1,5 @@
 require "yaml"
+require "erb"
 
 module Gemstash
   #:nodoc:
@@ -8,7 +9,12 @@ module Gemstash
       base_path: File.expand_path("~/.gemstash"),
       db_adapter: "sqlite3",
       bind: "tcp://0.0.0.0:9292",
-      rubygems_url: "https://rubygems.org"
+      rubygems_url: "https://rubygems.org",
+      protected_fetch: false,
+      fetch_timeout: 20,
+      # Actual default for db_connection_options is dynamic based on the adapter
+      db_connection_options: {},
+      puma_threads: 16
     }.freeze
 
     DEFAULT_FILE = File.expand_path("~/.gemstash/config.yml").freeze
@@ -28,10 +34,10 @@ module Gemstash
       end
 
       raise MissingFileError, file if file && !File.exist?(file)
-      file ||= DEFAULT_FILE
+      file ||= default_file
 
       if File.exist?(file)
-        @config = YAML.load_file(file)
+        @config = parse_config(file)
         @config = DEFAULTS.merge(@config)
         @config.freeze
       else
@@ -46,5 +52,31 @@ module Gemstash
     def [](key)
       @config[key]
     end
+
+    # @return [Hash] Sequel connection configuration hash
+    def database_connection_config
+      case self[:db_adapter]
+      when "sqlite3"
+        { max_connections: 1 }.merge(self[:db_connection_options])
+      when "postgres", "mysql", "mysql2"
+        { max_connections: self[:puma_threads] + 1 }.merge(self[:db_connection_options])
+      else
+        raise "Unsupported DB adapter: '#{self[:db_adapter]}'"
+      end
+    end
+
+  private
+
+    def default_file
+      File.exist?("#{DEFAULT_FILE}.erb") ? "#{DEFAULT_FILE}.erb" : DEFAULT_FILE
+    end
+
+    def parse_config(file)
+      if file.end_with?(".erb")
+        YAML.load(ERB.new(File.read(file)).result) || {}
+      else
+        YAML.load_file(file) || {}
+      end
+    end
   end
 end

data/lib/gemstash/db/version.rb

@@ -19,6 +19,17 @@ module Gemstash
         [rubygem.name, Gem::Version.new(number), platform]
       end
 
+      def self.slug(params)
+        version = params[:version]
+        platform = params[:platform]
+
+        if platform.to_s.empty?
+          version
+        else
+          "#{version}-#{platform}"
+        end
+      end
+
       def self.for_spec_collection(prerelease: false, latest: false)
         versions = where(indexed: true, prerelease: prerelease).association_join(:rubygem)
         latest ? select_latest(versions) : versions

data/lib/gemstash/env.rb

@@ -31,6 +31,7 @@ module Gemstash
 
       def call(env)
         env["gemstash.env"] = @gemstash_env
+        Gemstash::Env.current = @gemstash_env
         @app.call(env)
       end
     end
@@ -55,11 +56,8 @@ module Gemstash
     end
 
     def self.daemonized?
-      if @daemonized.nil?
-        raise "Daemonized hasn't been set yet!"
-      else
-        @daemonized
-      end
+      raise "Daemonized hasn't been set yet!" if @daemonized.nil?
+      @daemonized
     end
 
     def self.daemonized=(value)
@@ -99,6 +97,14 @@ module Gemstash
       File.join(base_path, path)
     end
 
+    def log_file
+      if config[:log_file] == :stdout
+        $stdout
+      else
+        base_file(config[:log_file] || "server.log")
+      end
+    end
+
     def atomic_write(file, &block)
       File.atomic_write(file, File.dirname(file), &block)
     end
@@ -113,13 +119,13 @@ module Gemstash
         when "sqlite3"
           db_path = base_file("gemstash.db")
 
-          if RUBY_PLATFORM == "java"
-            db = Sequel.connect("jdbc:sqlite:#{db_path}", max_connections: 1)
+          db = if RUBY_PLATFORM == "java"
+            Sequel.connect("jdbc:sqlite:#{db_path}", config.database_connection_config)
           else
-            db = Sequel.connect("sqlite://#{URI.escape(db_path)}", max_connections: 1)
+            Sequel.connect("sqlite://#{URI.escape(db_path)}", config.database_connection_config)
           end
-        when "postgres", "mysql"
-          db = Sequel.connect(config[:db_url])
+        when "postgres", "mysql", "mysql2"
+          db = Sequel.connect(config[:db_url], config.database_connection_config)
         else
           raise "Unsupported DB adapter: '#{config[:db_adapter]}'"
         end

data/lib/gemstash/gem_fetcher.rb

@@ -31,7 +31,7 @@ module Gemstash
     end
 
     def filter_headers(headers)
-      headers.inject({}) do|properties, (key, value)|
+      headers.inject({}) do |properties, (key, value)|
         properties[key.downcase] = value if @valid_headers.include?(key.downcase)
         properties
       end

data/lib/gemstash/gem_pusher.rb

@@ -16,12 +16,17 @@ module Gemstash
     class YankedVersionError < ExistingVersionError
     end
 
-    def initialize(auth_key, content)
-      @auth_key = auth_key
+    def self.serve(app)
+      gem = app.request.body.read
+      new(app.auth, gem).serve
+    end
+
+    def initialize(auth, content)
+      @auth = auth
       @content = content
     end
 
-    def push
+    def serve
       check_auth
       store_gem
       store_gemspec
@@ -44,7 +49,7 @@ module Gemstash
     end
 
     def check_auth
-      Gemstash::Authorization.check(@auth_key, "push")
+      @auth.check("push")
     end
 
     def store_gem
@@ -64,15 +69,8 @@ module Gemstash
       gemstash_env.db.transaction do
         gem_id = Gemstash::DB::Rubygem.find_or_insert(spec)
         existing = Gemstash::DB::Version.find_by_spec(gem_id, spec)
-
-        if existing
-          if existing.indexed
-            raise ExistingVersionError, "Cannot push to an existing version!"
-          else
-            raise YankedVersionError, "Cannot push to a yanked version!"
-          end
-        end
-
+        raise ExistingVersionError, "Cannot push to an existing version!" if existing && existing.indexed
+        raise YankedVersionError, "Cannot push to a yanked version!" if existing && !existing.indexed
         version_id = Gemstash::DB::Version.insert_by_spec(gem_id, spec)
         Gemstash::DB::Dependency.insert_by_spec(version_id, spec)
       end
@@ -90,14 +88,14 @@ module Gemstash
     module LegacyRubyGemsSupport
       def self.included(base)
         base.class_eval do
-          alias_method :push_without_cleanup, :push
-          remove_method :push
+          alias_method :serve_without_cleanup, :serve
+          remove_method :serve
           remove_method :gem
         end
       end
 
-      def push
-        push_without_cleanup
+      def serve
+        serve_without_cleanup
       ensure
         cleanup
       end