gemstash 1.0.0.pre.1-java

data/exe/gemstash

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require "gemstash/cli"
+Gemstash::CLI.start

data/gemstash.gemspec

@@ -0,0 +1,47 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "gemstash/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "gemstash"
+  spec.version       = Gemstash::VERSION
+  spec.authors       = ["Andre Arko"]
+  spec.email         = ["andre@arko.net"]
+  spec.platform      = "java" if RUBY_PLATFORM == "java"
+
+  spec.summary       = "A place to stash gems you'll need"
+  spec.description   = "Gemstash acts as a local RubyGems server, caching \
+copies of gems from RubyGems.org automatically, and eventually letting \
+you push your own private gems as well."
+  spec.homepage      = "https://github.com/bundler/gemstash"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject {|f|
+    f.match(%r{^(test|spec|features)/})
+  }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) {|f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "dalli", "~> 2.7"
+  spec.add_runtime_dependency "lru_redux", "~> 1.1"
+  spec.add_runtime_dependency "puma", "~> 2.14"
+  spec.add_runtime_dependency "sequel", "~> 4.26"
+  spec.add_runtime_dependency "sinatra", "~> 1.4"
+  spec.add_runtime_dependency "thor", "~> 0.19"
+  spec.add_runtime_dependency "faraday", "~> 0.9"
+  spec.add_runtime_dependency "faraday_middleware", "~> 0.10"
+
+  if RUBY_PLATFORM == "java"
+    spec.add_runtime_dependency "jdbc-sqlite3", "~> 3.8"
+  else
+    spec.add_runtime_dependency "sqlite3", "~> 1.3"
+  end
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rack-test", "~> 0.6"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.3"
+  spec.add_development_dependency "rubocop", "~> 0.34"
+end

data/lib/gemstash.rb

@@ -0,0 +1,26 @@
+#:nodoc:
+module Gemstash
+  autoload :Authorization,       "gemstash/authorization"
+  autoload :DB,                  "gemstash/db"
+  autoload :Cache,               "gemstash/cache"
+  autoload :CLI,                 "gemstash/cli"
+  autoload :Configuration,       "gemstash/configuration"
+  autoload :Dependencies,        "gemstash/dependencies"
+  autoload :Env,                 "gemstash/env"
+  autoload :GemFetcher,          "gemstash/gem_fetcher"
+  autoload :GemPusher,           "gemstash/gem_pusher"
+  autoload :GemSource,           "gemstash/gem_source"
+  autoload :GemUnyanker,         "gemstash/gem_unyanker"
+  autoload :GemYanker,           "gemstash/gem_yanker"
+  autoload :HTTPClient,          "gemstash/http_client"
+  autoload :Logging,             "gemstash/logging"
+  autoload :LruReduxClient,      "gemstash/cache"
+  autoload :NotAuthorizedError,  "gemstash/authorization"
+  autoload :RackEnvRewriter,     "gemstash/rack_env_rewriter"
+  autoload :SpecsBuilder,        "gemstash/specs_builder"
+  autoload :Storage,             "gemstash/storage"
+  autoload :Upstream,            "gemstash/upstream"
+  autoload :Web,                 "gemstash/web"
+  autoload :WebError,            "gemstash/http_client"
+  autoload :VERSION,             "gemstash/version"
+end

data/lib/gemstash/authorization.rb

@@ -0,0 +1,87 @@
+require "set"
+
+module Gemstash
+  # An action was not authorized and should cause the server to send a 401.
+  class NotAuthorizedError < StandardError
+  end
+
+  # Authorization mechanism to manipulate private gems.
+  class Authorization
+    extend Gemstash::Env::Helper
+    extend Gemstash::Logging
+    VALID_PERMISSIONS = %w(push yank unyank).freeze
+
+    def self.authorize(auth_key, permissions)
+      raise "Authorization key is required!" if auth_key.to_s.strip.empty?
+      raise "Permissions are required!" if permissions.to_s.empty?
+
+      unless permissions == "all"
+        permissions.each do |permission|
+          unless VALID_PERMISSIONS.include?(permission)
+            raise "Invalid permission '#{permission}'"
+          end
+        end
+
+        permissions = permissions.join(",")
+      end
+
+      Gemstash::DB::Authorization.insert_or_update(auth_key, permissions)
+      gemstash_env.cache.invalidate_authorization(auth_key)
+      log.info "Authorization '#{auth_key}' updated with access to '#{permissions}'"
+    end
+
+    def self.remove(auth_key)
+      record = Gemstash::DB::Authorization[auth_key: auth_key]
+      return unless record
+      record.destroy
+      gemstash_env.cache.invalidate_authorization(auth_key)
+      log.info "Authorization '#{auth_key}' with access to '#{record.permissions}' removed"
+    end
+
+    def self.check(auth_key, permission)
+      raise NotAuthorizedError, "Authorization key required" if auth_key.to_s.strip.empty?
+      auth = self[auth_key]
+      raise NotAuthorizedError, "Authorization key is invalid" unless auth
+      raise NotAuthorizedError, "Authorization key doesn't have #{permission} access" unless auth.can?(permission)
+    end
+
+    def self.[](auth_key)
+      cached_auth = gemstash_env.cache.authorization(auth_key)
+      return cached_auth if cached_auth
+      record = Gemstash::DB::Authorization[auth_key: auth_key]
+
+      if record
+        auth = new(record)
+        gemstash_env.cache.set_authorization(record.auth_key, auth)
+        auth
+      end
+    end
+
+    def initialize(record)
+      @auth_key = record.auth_key
+      @all = record.permissions == "all"
+      @permissions = Set.new(record.permissions.split(","))
+    end
+
+    def can?(permission)
+      raise "Invalid permission '#{permission}'" unless VALID_PERMISSIONS.include?(permission)
+      all? || @permissions.include?(permission)
+    end
+
+    def all?
+      @all
+    end
+
+    def push?
+      can?("push")
+    end
+
+    def yank?
+      can?("yank")
+    end
+
+    def unyank?
+      can?("unyank")
+    end
+  end
+end

data/lib/gemstash/cache.rb

@@ -0,0 +1,79 @@
+require "lru_redux"
+require "forwardable"
+
+module Gemstash
+  # Cache object which knows about what things are cached and what keys to use
+  # for them. Under the hood is either a Memcached client via the dalli gem, or
+  # an in memory client via the lru_redux gem.
+  class Cache
+    EXPIRY = 30 * 60
+    extend Forwardable
+    def_delegators :@client, :flush
+
+    def initialize(client)
+      @client = client
+    end
+
+    def authorization(auth_key)
+      @client.get("auths/#{auth_key}")
+    end
+
+    def set_authorization(auth_key, value)
+      @client.set("auths/#{auth_key}", value, EXPIRY)
+    end
+
+    def invalidate_authorization(auth_key)
+      @client.delete("auths/#{auth_key}")
+    end
+
+    def dependencies(scope, gems)
+      key_prefix = "deps/v1/#{scope}/"
+      keys = gems.map {|g| "#{key_prefix}#{g}" }
+
+      @client.get_multi(keys) do |key, value|
+        yield(key.sub(key_prefix, ""), value)
+      end
+    end
+
+    def set_dependency(scope, gem, value)
+      @client.set("deps/v1/#{scope}/#{gem}", value, EXPIRY)
+    end
+
+    def invalidate_gem(scope, gem)
+      @client.delete("deps/v1/#{scope}/#{gem}")
+      Gemstash::SpecsBuilder.invalidate_stored if scope == "private"
+    end
+  end
+
+  # Wrapper around the lru_redux gem to behave like a dalli Memcached client.
+  class LruReduxClient
+    MAX_SIZE = 500
+    EXPIRY = Gemstash::Cache::EXPIRY
+    extend Forwardable
+    def_delegators :@cache, :delete
+    def_delegator :@cache, :[], :get
+    def_delegator :@cache, :clear, :flush
+
+    def initialize
+      @cache = LruRedux::TTL::ThreadSafeCache.new MAX_SIZE, EXPIRY
+    end
+
+    def alive!
+      true
+    end
+
+    def get_multi(keys)
+      keys.each do |key|
+        found = true
+        # Atomic fetch... don't rely on nil meaning missing
+        value = @cache.fetch(key) { found = false }
+        next unless found
+        yield(key, value)
+      end
+    end
+
+    def set(key, value, expiry)
+      @cache[key] = value
+    end
+  end
+end

data/lib/gemstash/cli.rb

@@ -0,0 +1,71 @@
+require "gemstash"
+require "thor"
+require "thor/error"
+
+module Gemstash
+  # Base Command Line Interface class.
+  class CLI < Thor
+    autoload :Authorize, "gemstash/cli/authorize"
+    autoload :Base,      "gemstash/cli/base"
+    autoload :Setup,     "gemstash/cli/setup"
+    autoload :Start,     "gemstash/cli/start"
+    autoload :Status,    "gemstash/cli/status"
+    autoload :Stop,      "gemstash/cli/stop"
+
+    # Thor::Error for the CLI, which colors the message red.
+    class Error < Thor::Error
+      def initialize(cli, message)
+        super(cli.set_color(message, :red))
+      end
+    end
+
+    def self.exit_on_failure?
+      true
+    end
+
+    desc "authorize [PERMISSIONS...]", "Add authorizations to push/yank/unyank private gems"
+    method_option :remove, :type => :boolean, :default => false, :desc =>
+      "Remove an authorization key"
+    method_option :config_file, :type => :string, :desc =>
+      "Config file to save to"
+    method_option :key, :type => :string, :desc =>
+      "Authorization key to create/update/delete (optional unless deleting)"
+    def authorize(*args)
+      Gemstash::CLI::Authorize.new(self, *args).run
+    end
+
+    desc "setup", "Checks for dependencies and does initial setup"
+    method_option :redo, :type => :boolean, :default => false, :desc =>
+      "Redo configuration"
+    method_option :debug, :type => :boolean, :default => false, :desc =>
+      "Show detailed errors"
+    method_option :config_file, :type => :string, :desc =>
+      "Config file to save to"
+    def setup
+      Gemstash::CLI::Setup.new(self).run
+    end
+
+    desc "start", "Starts your gemstash server"
+    method_option :daemonize, :type => :boolean, :default => true, :desc =>
+      "Daemonize the server"
+    method_option :config_file, :type => :string, :desc =>
+      "Config file to load when starting"
+    def start
+      Gemstash::CLI::Start.new(self).run
+    end
+
+    desc "status", "Check the status of your gemstash server"
+    method_option :config_file, :type => :string, :desc =>
+      "Config file to load when checking the status"
+    def status
+      Gemstash::CLI::Status.new(self).run
+    end
+
+    desc "stop", "Stops your gemstash server"
+    method_option :config_file, :type => :string, :desc =>
+      "Config file to load when stopping"
+    def stop
+      Gemstash::CLI::Stop.new(self).run
+    end
+  end
+end

data/lib/gemstash/cli/authorize.rb

@@ -0,0 +1,69 @@
+require "gemstash"
+require "securerandom"
+
+module Gemstash
+  class CLI
+    # This implements the command line authorize task to authorize users:
+    #  $ gemstash authorize authorized-key
+    class Authorize < Gemstash::CLI::Base
+      def run
+        prepare
+        setup_logging
+
+        if @cli.options[:remove]
+          remove_authorization
+        else
+          save_authorization
+        end
+      end
+
+    private
+
+      def setup_logging
+        Gemstash::Logging.setup_logger(gemstash_env.base_file("server.log"))
+      end
+
+      def remove_authorization
+        raise Gemstash::CLI::Error.new(@cli, "To remove individual permissions, you do not need --remove
+Instead just authorize with the new set of permissions") unless @args.empty?
+        Gemstash::Authorization.remove(auth_key(false))
+      end
+
+      def save_authorization
+        if @args.include?("all")
+          raise Gemstash::CLI::Error.new(@cli, "Don't specify permissions to authorize for all")
+        end
+
+        @args.each do |arg|
+          unless Gemstash::Authorization::VALID_PERMISSIONS.include?(arg)
+            valid = Gemstash::Authorization::VALID_PERMISSIONS.join(", ")
+            raise Gemstash::CLI::Error.new(@cli, "Invalid permission '#{arg}'\nValid permissions include: #{valid}")
+          end
+        end
+
+        Gemstash::Authorization.authorize(auth_key, permissions)
+      end
+
+      def auth_key(allow_generate = true)
+        if @cli.options[:key]
+          @cli.options[:key]
+        elsif allow_generate
+          key = SecureRandom.hex(16)
+          key = SecureRandom.hex(16) while Gemstash::Authorization[key]
+          @cli.say "Your new key is: #{key}"
+          key
+        else
+          raise Gemstash::CLI::Error.new(@cli, "The --key option is required to remove an authorization key")
+        end
+      end
+
+      def permissions
+        if @args.empty?
+          "all"
+        else
+          @args
+        end
+      end
+    end
+  end
+end

data/lib/gemstash/cli/base.rb

@@ -0,0 +1,46 @@
+require "gemstash"
+
+module Gemstash
+  class CLI
+    # Base class for common functionality for CLI tasks.
+    class Base
+      include Gemstash::Env::Helper
+
+      def initialize(cli, *args)
+        Gemstash::Env.current = Gemstash::Env.new
+        @cli = cli
+        @args = args
+      end
+
+    private
+
+      def prepare
+        check_rubygems_version
+        store_config
+        check_gemstash_version
+      end
+
+      def check_rubygems_version
+        @cli.say(@cli.set_color("Rubygems version is too old, " \
+                                 "please update rubygems by running: " \
+                                 "gem update --system", :red)) unless
+        Gem::Requirement.new(">= 2.4").satisfied_by?(Gem::Version.new(Gem::VERSION))
+      end
+
+      def store_config
+        config = Gemstash::Configuration.new(file: @cli.options[:config_file])
+        gemstash_env.config = config
+      end
+
+      def check_gemstash_version
+        version = Gem::Version.new(Gemstash::Storage.metadata[:gemstash_version])
+        return if Gem::Requirement.new("<= #{Gemstash::VERSION}").satisfied_by?(Gem::Version.new(version))
+        raise Gemstash::CLI::Error.new(@cli, "Gemstash version is too old")
+      end
+
+      def pidfile_args
+        ["--pidfile", gemstash_env.base_file("puma.pid")]
+      end
+    end
+  end
+end

data/lib/gemstash/cli/setup.rb

@@ -0,0 +1,173 @@
+require "gemstash"
+require "fileutils"
+require "yaml"
+
+module Gemstash
+  class CLI
+    # This implements the command line setup task:
+    #  $ gemstash setup
+    class Setup < Gemstash::CLI::Base
+      def initialize(cli)
+        super
+        @config = {}
+      end
+
+      def run
+        if setup? && !@cli.options[:redo]
+          @cli.say @cli.set_color("Everything is already setup!", :green)
+          return
+        end
+
+        check_rubygems_version
+        ask_storage
+        ask_cache
+        ask_database
+        check_cache
+        check_storage
+        check_database
+        store_config
+        save_metadata
+        @cli.say @cli.set_color("You are all setup!", :green)
+      end
+
+    private
+
+      def config_file
+        @cli.options[:config_file] || Gemstash::Configuration::DEFAULT_FILE
+      end
+
+      def setup?
+        File.exist?(config_file)
+      end
+
+      def say_current_config(option, label)
+        return if gemstash_env.config.default?(option)
+        @cli.say "#{label}: #{gemstash_env.config[option]}"
+      end
+
+      def ask_storage
+        say_current_config(:base_path, "Current base path")
+        path = @cli.ask "Where should files go? [~/.gemstash]", path: true
+        path = Gemstash::Configuration::DEFAULTS[:base_path] if path.empty?
+        @config[:base_path] = File.expand_path(path)
+      end
+
+      def ask_cache
+        say_current_config(:cache_type, "Current cache")
+        options = %w(memory memcached)
+        cache = nil
+
+        until cache
+          cache = @cli.ask "Cache with what? [MEMORY, memcached]"
+          cache = cache.downcase
+          cache = "memory" if cache.empty?
+          cache = nil unless options.include?(cache)
+        end
+
+        @config[:cache_type] = cache
+        ask_memcached_details if cache == "memcached"
+      end
+
+      def ask_memcached_details
+        say_current_config(:memcached_servers, "Current Memcached servers")
+        servers = @cli.ask "What is the comma separated Memcached servers? [localhost:11211]"
+        servers = "localhost:11211" if servers.empty?
+        @config[:memcached_servers] = servers
+      end
+
+      def ask_database
+        say_current_config(:db_adapter, "Current database adapter")
+        options = %w(sqlite3 postgres)
+        database = nil
+
+        until database
+          database = @cli.ask "What database adapter? [SQLITE3, postgres]"
+          database = database.downcase
+          database = "sqlite3" if database.empty?
+          database = nil unless options.include?(database)
+        end
+
+        @config[:db_adapter] = database
+        ask_postgres_details if database == "postgres"
+      end
+
+      def ask_postgres_details
+        say_current_config(:db_url, "Current database url")
+
+        if RUBY_PLATFORM == "java"
+          default_value = "jdbc:postgres:///gemstash"
+        else
+          default_value = "postgres:///gemstash"
+        end
+
+        url = @cli.ask "Where is the database? [#{default_value}]"
+        url = default_value if url.empty?
+        @config[:db_url] = url
+      end
+
+      def check_cache
+        @cli.say "Checking that the cache is available"
+        with_new_config { gemstash_env.cache_client.alive! }
+      rescue => e
+        say_error "Cache error", e
+        raise Gemstash::CLI::Error.new(@cli, "The cache is not available")
+      end
+
+      def check_database
+        @cli.say "Checking that the database is available"
+        with_new_config { gemstash_env.db.test_connection }
+      rescue => e
+        say_error "Database error", e
+        raise Gemstash::CLI::Error.new(@cli, "The database is not available")
+      end
+
+      def check_storage
+        with_new_config do
+          dir = gemstash_env.config[:base_path]
+
+          if Dir.exist?(dir)
+            # Do metadata check without using Gemstash::Storage.metadata because
+            # we don't want to store metadata just yet
+            metadata_file = gemstash_env.base_file("metadata.yml")
+            break unless File.exist?(metadata_file)
+            version = Gem::Version.new(YAML.load_file(metadata_file)[:gemstash_version])
+            break if Gem::Requirement.new("<= #{Gemstash::VERSION}").satisfied_by?(Gem::Version.new(version))
+            raise Gemstash::CLI::Error.new(@cli, "The base path already exists with a newer version of Gemstash")
+          else
+            @cli.say "Creating the file storage path '#{dir}'"
+            FileUtils.mkpath(dir)
+          end
+        end
+      end
+
+      def store_config
+        config_dir = File.dirname(config_file)
+        FileUtils.mkpath(config_dir) unless Dir.exist?(config_dir)
+        File.write(config_file, YAML.dump(@config))
+      end
+
+      def save_metadata
+        with_new_config do
+          # Touch metadata to ensure it gets written
+          Gemstash::Storage.metadata
+        end
+      end
+
+      def say_error(title, error)
+        return unless @cli.options[:debug]
+        @cli.say @cli.set_color("#{title}: #{error}", :red)
+
+        error.backtrace.each do |line|
+          @cli.say @cli.set_color("  #{line}", :red)
+        end
+      end
+
+      def with_new_config
+        gemstash_env.config = Gemstash::Configuration.new(config: @config)
+        yield
+      ensure
+        gemstash_env.reset
+      end
+    end
+  end
+end