gemstash 1.0.0.pre.1-java → 2.6.0-java

data/lib/gemstash/cli/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
 module Gemstash
@@ -21,25 +23,31 @@ module Gemstash
       end
 
       def check_rubygems_version
-        @cli.say(@cli.set_color("Rubygems version is too old, " \
-                                 "please update rubygems by running: " \
-                                 "gem update --system", :red)) unless
-        Gem::Requirement.new(">= 2.4").satisfied_by?(Gem::Version.new(Gem::VERSION))
+        unless Gem::Requirement.new(">= 2.4").satisfied_by?(Gem::Version.new(Gem::VERSION))
+          @cli.say(@cli.set_color("Rubygems version is too old, " \
+                                   "please update rubygems by running: " \
+                                   "gem update --system", :red))
+        end
       end
 
       def store_config
         config = Gemstash::Configuration.new(file: @cli.options[:config_file])
         gemstash_env.config = config
+      rescue Gemstash::Configuration::MissingFileError => e
+        raise Gemstash::CLI::Error.new(@cli, e.message)
       end
 
       def check_gemstash_version
         version = Gem::Version.new(Gemstash::Storage.metadata[:gemstash_version])
         return if Gem::Requirement.new("<= #{Gemstash::VERSION}").satisfied_by?(Gem::Version.new(version))
-        raise Gemstash::CLI::Error.new(@cli, "Gemstash version is too old")
+
+        raise Gemstash::CLI::Error.new(@cli, "Gemstash version #{Gemstash::VERSION} does not support version " \
+                                             "#{version}.\nIt appears you may have downgraded Gemstash, please " \
+                                             "install version #{version} or later.")
       end
 
       def pidfile_args
-        ["--pidfile", gemstash_env.base_file("puma.pid")]
+        ["--pidfile", gemstash_env.pidfile]
       end
     end
   end

data/lib/gemstash/cli/setup.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "fileutils"
 require "yaml"
@@ -22,6 +24,8 @@ module Gemstash
         ask_storage
         ask_cache
         ask_database
+        ask_protected_fetch
+        ask_timeout
         check_cache
         check_storage
         check_database
@@ -42,9 +46,27 @@ module Gemstash
 
       def say_current_config(option, label)
         return if gemstash_env.config.default?(option)
+
         @cli.say "#{label}: #{gemstash_env.config[option]}"
       end
 
+      def ask_with_default(prompt, options, default)
+        raise "The options must all be lower case" if options.any? {|x| x.downcase != x }
+
+        result = nil
+        displayed_options = options.map {|x| x == default ? x.upcase : x }
+        prompt = "#{prompt} [#{displayed_options.join(", ")}]"
+
+        until result
+          result = @cli.ask prompt
+          result = result.downcase
+          result = default if result.empty?
+          result = nil unless options.include?(result)
+        end
+
+        result
+      end
+
       def ask_storage
         say_current_config(:base_path, "Current base path")
         path = @cli.ask "Where should files go? [~/.gemstash]", path: true
@@ -54,18 +76,9 @@ module Gemstash
 
       def ask_cache
         say_current_config(:cache_type, "Current cache")
-        options = %w(memory memcached)
-        cache = nil
-
-        until cache
-          cache = @cli.ask "Cache with what? [MEMORY, memcached]"
-          cache = cache.downcase
-          cache = "memory" if cache.empty?
-          cache = nil unless options.include?(cache)
-        end
-
-        @config[:cache_type] = cache
-        ask_memcached_details if cache == "memcached"
+        @config[:cache_type] = ask_with_default("Cache with what?", %w[memory memcached redis], "memory")
+        ask_memcached_details if @config[:cache_type] == "memcached"
+        ask_redis_details if @config[:cache_type] == "redis"
       end
 
       def ask_memcached_details
@@ -75,29 +88,26 @@ module Gemstash
         @config[:memcached_servers] = servers
       end
 
+      def ask_redis_details
+        say_current_config(:redis_servers, "Current Redis servers")
+        servers = @cli.ask "What is the comma-separated list of Redis servers? [localhost:6379]"
+        servers = "localhost:6379" if servers.empty?
+        @config[:redis_servers] = servers
+      end
+
       def ask_database
         say_current_config(:db_adapter, "Current database adapter")
-        options = %w(sqlite3 postgres)
-        database = nil
-
-        until database
-          database = @cli.ask "What database adapter? [SQLITE3, postgres]"
-          database = database.downcase
-          database = "sqlite3" if database.empty?
-          database = nil unless options.include?(database)
-        end
-
-        @config[:db_adapter] = database
-        ask_postgres_details if database == "postgres"
+        @config[:db_adapter] = ask_with_default("What database adapter?", %w[sqlite3 postgres mysql mysql2], "sqlite3")
+        ask_database_details(@config[:db_adapter]) unless @config[:db_adapter] == "sqlite3"
       end
 
-      def ask_postgres_details
+      def ask_database_details(database)
         say_current_config(:db_url, "Current database url")
 
-        if RUBY_PLATFORM == "java"
-          default_value = "jdbc:postgres:///gemstash"
+        default_value = if RUBY_PLATFORM == "java"
+          "jdbc:#{database}:///gemstash"
         else
-          default_value = "postgres:///gemstash"
+          "#{database}:///gemstash"
         end
 
         url = @cli.ask "Where is the database? [#{default_value}]"
@@ -105,20 +115,27 @@ module Gemstash
         @config[:db_url] = url
       end
 
+      def ask_protected_fetch
+        say_current_config(:protected_fetch, "Protected Fetch enabled")
+
+        value = @cli.yes? "Use Protected Fetch for Private Gems? [y/N]"
+        value = Gemstash::Configuration::DEFAULTS[:protected_fetch] if value.is_a?(String) && value.empty?
+        @config[:protected_fetch] = value
+      end
+
+      def ask_timeout
+        say_current_config(:fetch_timeout, "Fetch timeout")
+        timeout = @cli.ask "How many seconds to wait when fetching a gem? [20]"
+        timeout = Gemstash::Configuration::DEFAULTS[:fetch_timeout] if timeout.to_i < 1
+        @config[:fetch_timeout] = timeout.to_i
+      end
+
       def check_cache
-        @cli.say "Checking that the cache is available"
-        with_new_config { gemstash_env.cache_client.alive! }
-      rescue => e
-        say_error "Cache error", e
-        raise Gemstash::CLI::Error.new(@cli, "The cache is not available")
+        try("cache") { gemstash_env.cache_client.alive! }
       end
 
       def check_database
-        @cli.say "Checking that the database is available"
-        with_new_config { gemstash_env.db.test_connection }
-      rescue => e
-        say_error "Database error", e
-        raise Gemstash::CLI::Error.new(@cli, "The database is not available")
+        try("database") { gemstash_env.db.test_connection }
       end
 
       def check_storage
@@ -130,8 +147,10 @@ module Gemstash
             # we don't want to store metadata just yet
             metadata_file = gemstash_env.base_file("metadata.yml")
             break unless File.exist?(metadata_file)
+
             version = Gem::Version.new(YAML.load_file(metadata_file)[:gemstash_version])
             break if Gem::Requirement.new("<= #{Gemstash::VERSION}").satisfied_by?(Gem::Version.new(version))
+
             raise Gemstash::CLI::Error.new(@cli, "The base path already exists with a newer version of Gemstash")
           else
             @cli.say "Creating the file storage path '#{dir}'"
@@ -143,7 +162,7 @@ module Gemstash
       def store_config
         config_dir = File.dirname(config_file)
         FileUtils.mkpath(config_dir) unless Dir.exist?(config_dir)
-        File.write(config_file, YAML.dump(@config))
+        gemstash_env.atomic_write(config_file) {|f| f.write(YAML.dump(@config)) }
       end
 
       def save_metadata
@@ -155,6 +174,7 @@ module Gemstash
 
       def say_error(title, error)
         return unless @cli.options[:debug]
+
         @cli.say @cli.set_color("#{title}: #{error}", :red)
 
         error.backtrace.each do |line|
@@ -168,6 +188,14 @@ module Gemstash
       ensure
         gemstash_env.reset
       end
+
+      def try(thing, &block)
+        @cli.say "Checking that the #{thing} is available"
+        with_new_config(&block)
+      rescue StandardError => e
+        say_error "Error checking #{thing}", e
+        raise Gemstash::CLI::Error.new(@cli, "The #{thing} is not available")
+      end
     end
   end
 end

data/lib/gemstash/cli/start.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "puma/cli"
 
@@ -10,6 +12,7 @@ module Gemstash
         prepare
         setup_logging
         store_daemonized
+        @cli.say("Starting gemstash!", :green)
         Puma::CLI.new(args, Gemstash::Logging::StreamLogger.puma_events).run
       end
 
@@ -17,7 +20,8 @@ module Gemstash
 
       def setup_logging
         return unless daemonize?
-        Gemstash::Logging.setup_logger(gemstash_env.base_file("server.log"))
+
+        Gemstash::Logging.setup_logger(gemstash_env.log_file)
       end
 
       def store_daemonized
@@ -29,7 +33,7 @@ module Gemstash
       end
 
       def puma_config
-        File.expand_path("../../puma.rb", __FILE__)
+        File.expand_path("../puma.rb", __dir__)
       end
 
       def args

data/lib/gemstash/cli/status.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "puma/control_cli"
 
@@ -14,7 +16,7 @@ module Gemstash
     private
 
       def args
-        pidfile_args + %w(status)
+        pidfile_args + %w[status]
       end
     end
   end

data/lib/gemstash/cli/stop.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "puma/control_cli"
 
@@ -9,12 +11,13 @@ module Gemstash
       def run
         prepare
         Puma::ControlCLI.new(args).run
+        @cli.say("Gemstash stopped successfully!", :green)
       end
 
     private
 
       def args
-        pidfile_args + %w(stop)
+        pidfile_args + %w[stop]
       end
     end
   end

data/lib/gemstash/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "thor"
 require "thor/error"
@@ -23,13 +25,40 @@ module Gemstash
       true
     end
 
-    desc "authorize [PERMISSIONS...]", "Add authorizations to push/yank/unyank private gems"
+    def self.start(args = ARGV)
+      help_flags = %w[-h --help]
+
+      if args.any? {|a| help_flags.include?(a) }
+        super(%w[help] + args.reject {|a| help_flags.include?(a) })
+      else
+        super
+      end
+    end
+
+    def help(command = nil)
+      command ||= "readme"
+      page = manpage(command)
+
+      if page && which("man")
+        exec "man", page
+      elsif page
+        puts File.read("#{page}.txt")
+      else
+        super
+      end
+    end
+
+    desc "authorize [PERMISSIONS...]", "Add authorizations to push/yank private gems"
     method_option :remove, :type => :boolean, :default => false, :desc =>
       "Remove an authorization key"
+    method_option :list, :type => :boolean, :default => false, :desc =>
+      "List existing authorization keys"
     method_option :config_file, :type => :string, :desc =>
       "Config file to save to"
     method_option :key, :type => :string, :desc =>
       "Authorization key to create/update/delete (optional unless deleting)"
+    method_option :name, :type => :string, :desc =>
+      "Name of the key (optional)"
     def authorize(*args)
       Gemstash::CLI::Authorize.new(self, *args).run
     end
@@ -67,5 +96,34 @@ module Gemstash
     def stop
       Gemstash::CLI::Stop.new(self).run
     end
+
+    desc "version", "Prints gemstash version information"
+    def version
+      say "Gemstash version #{Gemstash::VERSION}"
+    end
+    map %w[-v --version] => :version
+
+  private
+
+    def manpage(command)
+      page = File.expand_path("../man/gemstash-#{command}", __FILE__)
+      return page if File.file?(page)
+
+      1.upto(8) do |section|
+        page = File.expand_path("../man/gemstash-#{command}.#{section}", __FILE__)
+        return page if File.file?(page)
+      end
+
+      nil
+    end
+
+    def which(executable)
+      ENV["PATH"].split(File::PATH_SEPARATOR).each do |path|
+        exe_path = File.join(path, executable)
+        return exe_path if File.file?(exe_path) && File.executable?(exe_path)
+      end
+
+      nil
+    end
   end
 end

data/lib/gemstash/config.ru

@@ -1,13 +1,14 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "puma/commonlogger"
 
 use Rack::Deflater
 use Gemstash::Logging::RackMiddleware
 
-if Gemstash::Env.daemonized?
-  use Puma::CommonLogger, Gemstash::Logging::StreamLogger.for_stdout
-end
+use Puma::CommonLogger, Gemstash::Logging::StreamLogger.for_stdout if Gemstash::Env.daemonized?
 
 use Gemstash::Env::RackMiddleware, Gemstash::Env.current
 use Gemstash::GemSource::RackMiddleware
+use Gemstash::Health::RackMiddleware
 run Gemstash::Web.new(gemstash_env: Gemstash::Env.current)

data/lib/gemstash/configuration.rb

@@ -1,28 +1,50 @@
+# frozen_string_literal: true
+
 require "yaml"
+require "erb"
 
 module Gemstash
-  #:nodoc:
+  # :nodoc:
   class Configuration
     DEFAULTS = {
-      :cache_type => "memory",
-      :base_path => File.expand_path("~/.gemstash"),
-      :db_adapter => "sqlite3",
-      :bind => "tcp://0.0.0.0:9292",
-      :rubygems_url => "https://www.rubygems.org"
+      cache_type: "memory",
+      base_path: File.expand_path("~/.gemstash"),
+      db_adapter: "sqlite3",
+      bind: "tcp://0.0.0.0:9292",
+      rubygems_url: "https://rubygems.org",
+      ignore_gemfile_source: false,
+      protected_fetch: false,
+      fetch_timeout: 20,
+      # Actual default for db_connection_options is dynamic based on the adapter
+      db_connection_options: {},
+      puma_threads: 16,
+      puma_workers: 1,
+      cache_expiration: 30 * 60,
+      cache_max_size: 500
     }.freeze
 
     DEFAULT_FILE = File.expand_path("~/.gemstash/config.yml").freeze
 
+    # This error is thrown when a config file is explicitly specified that
+    # doesn't exist.
+    class MissingFileError < StandardError
+      def initialize(file)
+        super("Missing config file: #{file}")
+      end
+    end
+
     def initialize(file: nil, config: nil)
       if config
         @config = DEFAULTS.merge(config).freeze
         return
       end
 
-      file ||= DEFAULT_FILE
+      raise MissingFileError, file if file && !File.exist?(file)
+
+      file ||= default_file
 
       if File.exist?(file)
-        @config = YAML.load_file(file)
+        @config = parse_config(file)
         @config = DEFAULTS.merge(@config)
         @config.freeze
       else
@@ -37,5 +59,36 @@ module Gemstash
     def [](key)
       @config[key]
     end
+
+    # @return [Hash] Sequel connection configuration hash
+    def database_connection_config
+      case self[:db_adapter]
+      when "sqlite3"
+        { max_connections: 1 }.merge(self[:db_connection_options])
+      when "postgres", "mysql", "mysql2"
+        { max_connections: (self[:puma_workers] * self[:puma_threads]) + 1 }.merge(self[:db_connection_options])
+      else
+        raise "Unsupported DB adapter: '#{self[:db_adapter]}'"
+      end
+    end
+
+  private
+
+    def default_file
+      # Support the config file being specified via environment variable
+      gemstash_config = ENV["GEMSTASH_CONFIG"]
+      return gemstash_config if gemstash_config
+
+      # If no environment variable is used, fall back to the normal defaults
+      File.exist?("#{DEFAULT_FILE}.erb") ? "#{DEFAULT_FILE}.erb" : DEFAULT_FILE
+    end
+
+    def parse_config(file)
+      if file.end_with?(".erb")
+        YAML.load(ERB.new(File.read(file)).result) || {}
+      else
+        YAML.load_file(file) || {}
+      end
+    end
   end
 end

data/lib/gemstash/db/authorization.rb

@@ -1,17 +1,19 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
 module Gemstash
   module DB
     # Sequel model for authorizations table.
     class Authorization < Sequel::Model
-      def self.insert_or_update(auth_key, permissions)
+      def self.insert_or_update(auth_key, permissions, name = nil)
         db.transaction do
           record = self[auth_key: auth_key]
 
           if record
-            record.update(permissions: permissions)
+            record.update(permissions: permissions, name: name)
           else
-            create(auth_key: auth_key, permissions: permissions)
+            create(auth_key: auth_key, permissions: permissions, name: name)
           end
         end
       end

data/lib/gemstash/db/cached_rubygem.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "gemstash"
+
+module Gemstash
+  module DB
+    # Sequel model for cached_rubygems table.
+    class CachedRubygem < Sequel::Model
+      def self.store(upstream, gem_name, resource_type)
+        db.transaction do
+          upstream_id = Gemstash::DB::Upstream.find_or_insert(upstream)
+          record = self[upstream_id: upstream_id, name: gem_name.name, resource_type: resource_type.to_s]
+          return record.id if record
+
+          new(upstream_id: upstream_id, name: gem_name.name, resource_type: resource_type.to_s).tap(&:save).id
+        end
+      end
+    end
+  end
+end

data/lib/gemstash/db/dependency.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
 module Gemstash

data/lib/gemstash/db/rubygem.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
 module Gemstash
@@ -7,6 +9,7 @@ module Gemstash
       def self.find_or_insert(spec)
         record = self[name: spec.name]
         return record.id if record
+
         new(name: spec.name).tap(&:save).id
       end
     end

data/lib/gemstash/db/upstream.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gemstash
+  module DB
+    # Sequel model for upstreams table.
+    class Upstream < Sequel::Model
+      def self.find_or_insert(upstream)
+        record = self[uri: upstream.to_s]
+        return record.id if record
+
+        new(uri: upstream.to_s, host_id: upstream.host_id).tap(&:save).id
+      end
+    end
+  end
+end

data/lib/gemstash/db/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
 module Gemstash
@@ -19,8 +21,28 @@ module Gemstash
         [rubygem.name, Gem::Version.new(number), platform]
       end
 
-      def self.for_spec_collection(prerelease: false)
-        where(indexed: true, prerelease: prerelease).association_join(:rubygem)
+      def self.slug(params)
+        version = params[:version]
+        platform = params[:platform]
+
+        if platform.to_s.empty?
+          version
+        else
+          "#{version}-#{platform}"
+        end
+      end
+
+      def self.for_spec_collection(prerelease: false, latest: false)
+        versions = where(indexed: true, prerelease: prerelease).association_join(:rubygem)
+        latest ? select_latest(versions) : versions
+      end
+
+      def self.select_latest(versions)
+        versions.
+          all.
+          group_by {|version| [version.rubygem_id, version.platform] }.
+          values.
+          map {|gem_versions| gem_versions.max_by {|version| Gem::Version.new(version.number) } }
       end
 
       def self.find_by_spec(gem_id, spec)
@@ -32,6 +54,7 @@ module Gemstash
       def self.find_by_full_name(full_name)
         result = self[full_name: full_name]
         return result if result
+
         # Try again with the default platform, in case it is implied
         self[full_name: "#{full_name}-ruby"]
       end

data/lib/gemstash/db.rb

@@ -1,15 +1,20 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
 module Gemstash
   # Module containing the DB models.
   module DB
     raise "Gemstash::DB cannot be loaded until the Gemstash::Env is available" unless Gemstash::Env.available?
+
     Sequel::Model.db = Gemstash::Env.current.db
     Sequel::Model.raise_on_save_failure = true
     Sequel::Model.plugin :timestamps, update_on_create: true
     autoload :Authorization, "gemstash/db/authorization"
+    autoload :CachedRubygem, "gemstash/db/cached_rubygem"
     autoload :Dependency,    "gemstash/db/dependency"
     autoload :Rubygem,       "gemstash/db/rubygem"
+    autoload :Upstream,      "gemstash/db/upstream"
     autoload :Version,       "gemstash/db/version"
   end
 end

data/lib/gemstash/dependencies.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require "cgi"
 require "set"
 
 module Gemstash
-  #:nodoc:
+  # :nodoc:
   class Dependencies
     def self.for_private
       new(scope: "private", db_model: Gemstash::DB::Dependency)
@@ -22,7 +24,7 @@ module Gemstash
       Fetcher.new(gems, @scope, @http_client, @db_model).fetch
     end
 
-    #:nodoc:
+    # :nodoc:
     class Fetcher
       include Gemstash::Env::Helper
       include Gemstash::Logging
@@ -59,6 +61,7 @@ module Gemstash
       def fetch_from_database
         return if done?
         return unless @db_model
+
         log.info "Querying dependencies: #{@gems.to_a.join(", ")}"
 
         @db_model.fetch(@gems) do |gem, value|
@@ -71,6 +74,7 @@ module Gemstash
       def fetch_from_web
         return if done?
         return unless @http_client
+
         log.info "Fetching dependencies: #{@gems.to_a.join(", ")}"
         gems_param = @gems.map {|gem| CGI.escape(gem) }.join(",")
         fetched = @http_client.get("api/v1/dependencies?gems=#{gems_param}")