gemstash 1.0.0.pre.1

data/lib/gemstash/gem_unyanker.rb

@@ -0,0 +1,61 @@
+require "gemstash"
+
+module Gemstash
+  # Class that supports unyanking a gem from the private repository of gems.
+  class GemUnyanker
+    include Gemstash::Env::Helper
+
+    # This error is thrown when unyanking a non-existing gem name.
+    class UnknownGemError < StandardError
+    end
+
+    # This error is thrown when unyanking a non-existing gem version.
+    class UnknownVersionError < StandardError
+    end
+
+    # This error is thrown when unyanking a non-yanked gem version.
+    class NotYankedVersionError < StandardError
+    end
+
+    def initialize(auth_key, gem_name, slug)
+      @auth_key = auth_key
+      @gem_name = gem_name
+      @slug = slug
+    end
+
+    def unyank
+      check_auth
+      update_database
+      invalidate_cache
+    end
+
+  private
+
+    def storage
+      @storage ||= Gemstash::Storage.for("private").for("gems")
+    end
+
+    def full_name
+      @full_name ||= "#{@gem_name}-#{@slug}"
+    end
+
+    def check_auth
+      Gemstash::Authorization.check(@auth_key, "unyank")
+    end
+
+    def update_database
+      gemstash_env.db.transaction do
+        raise UnknownGemError, "Cannot unyank an unknown gem!" unless Gemstash::DB::Rubygem[name: @gem_name]
+        version = Gemstash::DB::Version.find_by_full_name(full_name)
+        raise UnknownVersionError, "Cannot unyank an unknown version!" unless version
+        raise NotYankedVersionError, "Cannot unyank a non-yanked version!" if version.indexed
+        version.reindex
+        storage.resource(version.storage_id).update_properties(indexed: true)
+      end
+    end
+
+    def invalidate_cache
+      gemstash_env.cache.invalidate_gem("private", @gem_name)
+    end
+  end
+end

data/lib/gemstash/gem_yanker.rb

@@ -0,0 +1,61 @@
+require "gemstash"
+
+module Gemstash
+  # Class that supports yanking a gem from the private repository of gems.
+  class GemYanker
+    include Gemstash::Env::Helper
+
+    # This error is thrown when yanking a non-existing gem name.
+    class UnknownGemError < StandardError
+    end
+
+    # This error is thrown when yanking a non-existing gem version.
+    class UnknownVersionError < StandardError
+    end
+
+    # This error is thrown when yanking an already yanked gem version.
+    class YankedVersionError < StandardError
+    end
+
+    def initialize(auth_key, gem_name, slug)
+      @auth_key = auth_key
+      @gem_name = gem_name
+      @slug = slug
+    end
+
+    def yank
+      check_auth
+      update_database
+      invalidate_cache
+    end
+
+  private
+
+    def storage
+      @storage ||= Gemstash::Storage.for("private").for("gems")
+    end
+
+    def full_name
+      @full_name ||= "#{@gem_name}-#{@slug}"
+    end
+
+    def check_auth
+      Gemstash::Authorization.check(@auth_key, "yank")
+    end
+
+    def update_database
+      gemstash_env.db.transaction do
+        raise UnknownGemError, "Cannot yank an unknown gem!" unless Gemstash::DB::Rubygem[name: @gem_name]
+        version = Gemstash::DB::Version.find_by_full_name(full_name)
+        raise UnknownVersionError, "Cannot yank an unknown version!" unless version
+        raise YankedVersionError, "Cannot yank an already yanked version!" unless version.indexed
+        version.deindex
+        storage.resource(version.storage_id).update_properties(indexed: false)
+      end
+    end
+
+    def invalidate_cache
+      gemstash_env.cache.invalidate_gem("private", @gem_name)
+    end
+  end
+end

data/lib/gemstash/http_client.rb

@@ -0,0 +1,77 @@
+require "gemstash"
+require "faraday"
+require "faraday_middleware"
+
+module Gemstash
+  #:nodoc:
+  class WebError < StandardError
+    attr_reader :code
+
+    def initialize(message, code)
+      @code = code
+      super(message)
+    end
+  end
+
+  #:nodoc:
+  class ConnectionError < WebError
+    def initialize(message)
+      super(message, 502) # Bad Gateway
+    end
+  end
+
+  #:nodoc:
+  class HTTPClient
+    include Gemstash::Logging
+
+    DEFAULT_USER_AGENT = "Gemstash/#{Gemstash::VERSION}"
+
+    def self.for(upstream)
+      client = Faraday.new(upstream.to_s) do |config|
+        config.use FaradayMiddleware::FollowRedirects
+        config.adapter :net_http
+      end
+      user_agent = "#{upstream.user_agent} " unless upstream.user_agent.to_s.empty?
+      user_agent = user_agent.to_s + DEFAULT_USER_AGENT
+
+      new(client, user_agent: user_agent)
+    end
+
+    def initialize(client = nil, user_agent: nil)
+      @client = client
+      @user_agent = user_agent || DEFAULT_USER_AGENT
+    end
+
+    def get(path)
+      response = with_retries do
+        @client.get(path) do |req|
+          req.headers["User-Agent"] = @user_agent
+          req.options.open_timeout = 2
+        end
+      end
+
+      raise Gemstash::WebError.new(response.body, response.status) unless response.success?
+
+      if block_given?
+        yield(response.body, response.headers)
+      else
+        response.body
+      end
+    end
+
+  private
+
+    def with_retries(times: 3, &block)
+      loop do
+        times -= 1
+        begin
+          return block.call
+        rescue Faraday::ConnectionFailed => e
+          log_error("Connection failure", e)
+          raise(ConnectionError, e.message) unless times > 0
+          log.info "retrying... #{times} more times"
+        end
+      end
+    end
+  end
+end

data/lib/gemstash/logging.rb

@@ -0,0 +1,93 @@
+require "logger"
+require "puma/events"
+
+module Gemstash
+  #:nodoc:
+  module Logging
+    LEVELS = {
+      debug: Logger::DEBUG,
+      info: Logger::INFO,
+      warn: Logger::WARN,
+      error: Logger::ERROR,
+      fatal: Logger::FATAL
+    }
+
+    def log
+      Gemstash::Logging.logger
+    end
+
+    def log_error(message, error, level: :error)
+      log.add(LEVELS[level]) do
+        "#{message} - #{error.message} (#{error.class})\n  #{error.backtrace.join("\n  ")}"
+      end
+    end
+
+    def self.setup_logger(logfile)
+      @logger = Logger.new(logfile, 2, 10_485_760)
+      @logger.level = Logger::INFO
+      @logger.datetime_format = "%d/%b/%Y:%H:%M:%S %z"
+      @logger.formatter = proc do |severity, datetime, _progname, msg|
+        if msg.end_with?("\n")
+          "[#{datetime}] - #{severity} - #{msg}"
+        else
+          "[#{datetime}] - #{severity} - #{msg}\n"
+        end
+      end
+      @logger
+    end
+
+    def self.logger
+      @logger ||= setup_logger($stdout)
+    end
+
+    def self.reset
+      @logger.close if @logger
+      @logger = nil
+    end
+
+    # Rack middleware to set the Rack logger to the Gemstash logger.
+    class RackMiddleware
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        env["rack.logger"] = Gemstash::Logging.logger
+        @app.call(env)
+      end
+    end
+
+    # Logger that looks like a stream, for Puma and Rack to log to.
+    class StreamLogger
+      def self.puma_events
+        Puma::Events.new(for_stdout, for_stderr)
+      end
+
+      def self.for_stdout
+        new(Logger::INFO)
+      end
+
+      def self.for_stderr
+        new(Logger::ERROR)
+      end
+
+      def initialize(level)
+        @level = level
+      end
+
+      def flush
+      end
+
+      def sync=(_value)
+      end
+
+      def write(message)
+        Gemstash::Logging.logger.add(@level, message)
+      end
+
+      def puts(message)
+        Gemstash::Logging.logger.add(@level, message)
+      end
+    end
+  end
+end

data/lib/gemstash/migrations/01_gem_dependencies.rb

@@ -0,0 +1,41 @@
+Sequel.migration do
+  change do
+    create_table :rubygems do
+      primary_key :id
+      String :name, :size => 255, :null => false
+      DateTime :created_at, :null => false
+      DateTime :updated_at, :null => false
+      index [:name], :unique => true
+    end
+
+    create_table :versions do
+      primary_key :id
+      Integer :rubygem_id, :null => false
+      String :storage_id, :size => 255, :null => false
+      String :number, :size => 255, :null => false
+      String :platform, :size => 255, :null => false
+      String :full_name, :size => 255, :null => false
+      TrueClass :indexed, :default => true, :null => false
+      TrueClass :prerelease, :null => false
+      DateTime :created_at, :null => false
+      DateTime :updated_at, :null => false
+      index [:rubygem_id, :number, :platform], :unique => true
+      index [:indexed]
+      index [:indexed, :prerelease]
+      index [:number]
+      index [:full_name], :unique => true
+      index [:storage_id], :unique => true
+    end
+
+    create_table :dependencies do
+      primary_key :id
+      Integer :version_id, :null => false
+      String :rubygem_name, :size => 255, :null => false
+      String :requirements, :size => 255, :null => false
+      DateTime :created_at, :null => false
+      DateTime :updated_at, :null => false
+      index [:version_id]
+      index [:rubygem_name]
+    end
+  end
+end

data/lib/gemstash/migrations/02_authorizations.rb

@@ -0,0 +1,12 @@
+Sequel.migration do
+  change do
+    create_table :authorizations do
+      primary_key :id
+      String :auth_key, :size => 2056, :null => false
+      String :permissions, :size => 255, :null => false
+      DateTime :created_at, :null => false
+      DateTime :updated_at, :null => false
+      index [:auth_key], :unique => true
+    end
+  end
+end

data/lib/gemstash/puma.rb

@@ -0,0 +1,6 @@
+require "gemstash"
+
+threads 0, 16
+bind "#{Gemstash::Env.current.config[:bind]}"
+workers 1
+rackup Gemstash::Env.current.rackup

data/lib/gemstash/rack_env_rewriter.rb

@@ -0,0 +1,66 @@
+require "gemstash"
+require "forwardable"
+
+module Gemstash
+  # Detects patterns in the Rack env variables related to URI and rewrites
+  # them, extracting parameters.
+  class RackEnvRewriter
+    attr_reader :regexp
+
+    def initialize(regexp)
+      @regexp = regexp
+    end
+
+    def for(rack_env)
+      Context.new(self, rack_env)
+    end
+
+    # Context containing the logic and the actual Rack environment.
+    class Context
+      include Gemstash::Logging
+      extend Forwardable
+      def_delegators :@rewriter, :regexp
+
+      def initialize(rewriter, rack_env)
+        @rewriter = rewriter
+        @rack_env = rack_env
+      end
+
+      def matches?
+        matches_request_uri? && matches_path_info?
+      end
+
+      def rewrite
+        check_match
+        log_start = "Rewriting '#{@rack_env["REQUEST_URI"]}'"
+        @rack_env["REQUEST_URI"][@request_uri_match.begin(0)...@request_uri_match.end(0)] = ""
+        @rack_env["PATH_INFO"][@path_info_match.begin(0)...@path_info_match.end(0)] = ""
+        log.info "#{log_start} to '#{@rack_env["REQUEST_URI"]}'"
+      end
+
+      def captures
+        @params ||= begin
+          check_match
+          @path_info_match.names.inject({}) do |result, name|
+            result[name] = @path_info_match[name]
+            result
+          end
+        end
+      end
+
+    private
+
+      def matches_request_uri?
+        @request_uri_match ||= @rack_env["REQUEST_URI"].match(regexp)
+      end
+
+      def matches_path_info?
+        @path_info_match ||= @rack_env["PATH_INFO"].match(regexp)
+      end
+
+      def check_match
+        raise "Rack env did not match!" unless @request_uri_match && @path_info_match
+      end
+    end
+  end
+end

data/lib/gemstash/specs_builder.rb

@@ -0,0 +1,93 @@
+require "gemstash"
+require "stringio"
+require "zlib"
+
+module Gemstash
+  # Builds a Marshal'ed and GZipped array of arrays containing specs as:
+  # [name, Gem::Version, platform]
+  class SpecsBuilder
+    attr_reader :result
+
+    # Used for the /private/specs.4.8.gz endpoint. Fetches non-prerelease,
+    # indexed private gems.
+    def self.all
+      new.build
+    end
+
+    # Used for the /private/prerelease_specs.4.8.gz endpoint. Fetches
+    # prerelease, indexed private gems.
+    def self.prerelease
+      new(prerelease: true).build
+    end
+
+    def self.invalidate_stored
+      storage = Gemstash::Storage.for("private").for("specs_collection")
+      storage.resource("specs.4.8.gz").delete(:specs)
+      storage.resource("prerelease_specs.4.8.gz").delete(:specs)
+    end
+
+    def initialize(prerelease: false)
+      @prerelease = prerelease
+    end
+
+    def build
+      fetch_from_storage
+      return result if result
+      fetch_versions
+      marshal
+      gzip
+      store_result
+      result
+    end
+
+  private
+
+    def storage
+      @storage ||= Gemstash::Storage.for("private").for("specs_collection")
+    end
+
+    def fetch_resource
+      if @prerelease
+        storage.resource("prerelease_specs.4.8.gz")
+      else
+        storage.resource("specs.4.8.gz")
+      end
+    end
+
+    def fetch_from_storage
+      specs = fetch_resource
+      return unless specs.exist?(:specs)
+      @result = specs.load(:specs).content(:specs)
+    rescue
+      # On the off-chance of a race condition between specs.exist? and specs.load
+      @result = nil
+    end
+
+    def fetch_versions
+      @versions = Gemstash::DB::Version.for_spec_collection(prerelease: @prerelease).map(&:to_spec)
+    end
+
+    def marshal
+      @marshal ||= Marshal.dump(@versions)
+    end
+
+    def gzip
+      @result ||= begin
+        output = StringIO.new
+        gz = Zlib::GzipWriter.new(output)
+
+        begin
+          gz.write(@marshal)
+        ensure
+          gz.close
+        end
+
+        output.string
+      end
+    end
+
+    def store_result
+      fetch_resource.save(specs: @result)
+    end
+  end
+end