gemstash 1.0.0.pre.1-java → 2.5.0-java

data/lib/gemstash/migrations/03_cached_gems.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table :upstreams do
+      primary_key :id
+      String :uri, size: 191, null: false
+      String :host_id, size: 191, null: false
+      DateTime :created_at, null: false
+      DateTime :updated_at, null: false
+      index [:uri], unique: true
+      index [:host_id], unique: true
+    end
+
+    create_table :cached_rubygems do
+      primary_key :id
+      Integer :upstream_id, null: false
+      String :name, size: 191, null: false
+      String :resource_type, size: 191, null: false
+      DateTime :created_at, null: false
+      DateTime :updated_at, null: false
+      index %i[upstream_id resource_type name], unique: true
+      index [:name]
+    end
+  end
+end

data/lib/gemstash/migrations/04_health_tests.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table :health_tests do
+      primary_key :id
+      String :string
+    end
+  end
+end

data/lib/gemstash/migrations/05_authorization_names.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table :authorizations do
+      add_column :name, String, :size => 191
+      add_index [:name], :unique => true
+    end
+  end
+end

data/lib/gemstash/puma.rb

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 require "gemstash"
 
-threads 0, 16
-bind "#{Gemstash::Env.current.config[:bind]}"
-workers 1
+threads 0, Gemstash::Env.current.config[:puma_threads].to_i
+bind Gemstash::Env.current.config[:bind].to_s
+workers Gemstash::Env.current.config[:puma_workers].to_i unless RUBY_PLATFORM == "java"
 rackup Gemstash::Env.current.rackup

data/lib/gemstash/rack_env_rewriter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "forwardable"
 
@@ -33,8 +35,15 @@ module Gemstash
       def rewrite
         check_match
         log_start = "Rewriting '#{@rack_env["REQUEST_URI"]}'"
-        @rack_env["REQUEST_URI"][@request_uri_match.begin(0)...@request_uri_match.end(0)] = ""
-        @rack_env["PATH_INFO"][@path_info_match.begin(0)...@path_info_match.end(0)] = ""
+
+        new_request_uri = @rack_env["REQUEST_URI"].dup
+        new_request_uri[@request_uri_match.begin(0)...@request_uri_match.end(0)] = ""
+
+        new_path_info = @rack_env["PATH_INFO"].dup
+        new_path_info[@path_info_match.begin(0)...@path_info_match.end(0)] = ""
+
+        @rack_env["REQUEST_URI"] = new_request_uri
+        @rack_env["PATH_INFO"]   = new_path_info
         log.info "#{log_start} to '#{@rack_env["REQUEST_URI"]}'"
       end
 

data/lib/gemstash/specs_builder.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "stringio"
 require "zlib"
@@ -6,33 +8,34 @@ module Gemstash
   # Builds a Marshal'ed and GZipped array of arrays containing specs as:
   # [name, Gem::Version, platform]
   class SpecsBuilder
+    include Gemstash::Env::Helper
     attr_reader :result
 
-    # Used for the /private/specs.4.8.gz endpoint. Fetches non-prerelease,
-    # indexed private gems.
-    def self.all
-      new.build
-    end
-
-    # Used for the /private/prerelease_specs.4.8.gz endpoint. Fetches
-    # prerelease, indexed private gems.
-    def self.prerelease
-      new(prerelease: true).build
+    def self.serve(app)
+      prerelease = app.params.fetch(:prerelease, false)
+      latest = app.params.fetch(:latest, false)
+      app.content_type "application/octet-stream"
+      new(app.auth, prerelease: prerelease, latest: latest).serve
     end
 
     def self.invalidate_stored
       storage = Gemstash::Storage.for("private").for("specs_collection")
       storage.resource("specs.4.8.gz").delete(:specs)
+      storage.resource("latest_specs.4.8.gz").delete(:specs)
       storage.resource("prerelease_specs.4.8.gz").delete(:specs)
     end
 
-    def initialize(prerelease: false)
+    def initialize(auth, prerelease: false, latest: false)
+      @auth = auth
       @prerelease = prerelease
+      @latest = latest
     end
 
-    def build
+    def serve
+      check_auth if gemstash_env.config[:protected_fetch]
       fetch_from_storage
       return result if result
+
       fetch_versions
       marshal
       gzip
@@ -47,7 +50,9 @@ module Gemstash
     end
 
     def fetch_resource
-      if @prerelease
+      if @latest
+        storage.resource("latest_specs.4.8.gz")
+      elsif @prerelease
         storage.resource("prerelease_specs.4.8.gz")
       else
         storage.resource("specs.4.8.gz")
@@ -57,14 +62,15 @@ module Gemstash
     def fetch_from_storage
       specs = fetch_resource
       return unless specs.exist?(:specs)
+
       @result = specs.load(:specs).content(:specs)
-    rescue
+    rescue StandardError
       # On the off-chance of a race condition between specs.exist? and specs.load
       @result = nil
     end
 
     def fetch_versions
-      @versions = Gemstash::DB::Version.for_spec_collection(prerelease: @prerelease).map(&:to_spec)
+      @versions = Gemstash::DB::Version.for_spec_collection(prerelease: @prerelease, latest: @latest).map(&:to_spec)
     end
 
     def marshal
@@ -89,5 +95,9 @@ module Gemstash
     def store_result
       fetch_resource.save(specs: @result)
     end
+
+    def check_auth
+      @auth.check("fetch")
+    end
   end
 end

data/lib/gemstash/storage.rb

@@ -1,68 +1,114 @@
+# frozen_string_literal: true
+
 require "gemstash"
 require "digest"
-require "pathname"
 require "fileutils"
+require "pathname"
 require "yaml"
 
 module Gemstash
-  #:nodoc:
+  # The entry point into the storage engine for storing cached gems, specs, and
+  # private gems.
   class Storage
     extend Gemstash::Env::Helper
     VERSION = 1
 
-    # If the storage engine detects something that was stored with a newer
-    # version of the storage engine, this error will be thrown.
+    # If the storage engine detects the base cache directory was originally
+    # initialized with a newer version, this error is thrown.
     class VersionTooNew < StandardError
+      def initialize(folder, version)
+        super("Gemstash storage version #{Gemstash::Storage::VERSION} does " \
+              "not support version #{version} found at #{folder}")
+      end
     end
 
+    # This object should not be constructed directly, but instead via
+    # {for} and {#for}.
     def initialize(folder, root: true)
-      check_engine if root
       @folder = folder
+      check_storage_version if root
       FileUtils.mkpath(@folder) unless Dir.exist?(@folder)
     end
 
+    # Fetch the resource with the given +id+ within this storage.
+    #
+    # @param id [String] the id of the resource to fetch
+    # @return [Gemstash::Resource] a new resource instance from the +id+
     def resource(id)
       Resource.new(@folder, id)
     end
 
+    # Fetch a nested entry from this instance in the storage engine.
+    #
+    # @param child [String] the name of the nested entry to load
+    # @return [Gemstash::Storage] a new storage instance for the +child+
     def for(child)
       Storage.new(File.join(@folder, child), root: false)
     end
 
+    # Fetch a base entry in the storage engine.
+    #
+    # @param name [String] the name of the entry to load
+    # @return [Gemstash::Storage] a new storage instance for the +name+
     def self.for(name)
       new(gemstash_env.base_file(name))
     end
 
+    # Read the global metadata for Gemstash and the storage engine. If the
+    # metadata hasn't been stored yet, it will be created.
+    #
+    # @return [Hash] the metadata about Gemstash and the storage engine
     def self.metadata
       file = gemstash_env.base_file("metadata.yml")
 
       unless File.exist?(file)
-        File.write(file, { storage_version: Gemstash::Storage::VERSION,
-                           gemstash_version: Gemstash::VERSION }.to_yaml)
+        gemstash_env.atomic_write(file) do |f|
+          f.write({ storage_version: Gemstash::Storage::VERSION,
+                    gemstash_version: Gemstash::VERSION }.to_yaml)
+        end
       end
 
-      YAML.load_file(file)
+      YAML.safe_load_file(file, permitted_classes: [Symbol])
     end
 
   private
 
-    def check_engine
+    def check_storage_version
       version = Gemstash::Storage.metadata[:storage_version]
       return if version <= Gemstash::Storage::VERSION
-      raise Gemstash::Storage::VersionTooNew, "Storage engine is out of date: #{version}"
+
+      raise Gemstash::Storage::VersionTooNew.new(@folder, version)
     end
 
     def path_valid?(path)
       return false if path.nil?
       return false unless File.writable?(path)
+
       true
     end
   end
 
-  #:nodoc:
+  # A resource within the storage engine. The resource may have 1 or more files
+  # associated with it along with a metadata Hash that is stored in a YAML file.
   class Resource
+    include Gemstash::Env::Helper
     include Gemstash::Logging
     attr_reader :name, :folder
+
+    VERSION = 1
+
+    # If the storage engine detects a resource was originally saved from a newer
+    # version, this error is thrown.
+    class VersionTooNew < StandardError
+      def initialize(name, folder, version)
+        super("Gemstash resource version #{Gemstash::Resource::VERSION} does " \
+              "not support version #{version} for resource #{name.inspect} " \
+              "found at #{folder}")
+      end
+    end
+
+    # This object should not be constructed directly, but instead via
+    # {Gemstash::Storage#resource}.
     def initialize(folder, name)
       @base_path = folder
       @name = name
@@ -76,8 +122,16 @@ module Gemstash
       digest = Digest::MD5.hexdigest(@name)
       child_folder = "#{safe_name}-#{digest}"
       @folder = File.join(@base_path, *trie_parents, child_folder)
+      @properties = nil
     end
 
+    # When +key+ is nil, this will test if this resource exists with any
+    # content. If a +key+ is provided, this will test that the resource exists
+    # with at least the given +key+ file. The +key+ corresponds to the +content+
+    # key provided to {#save}.
+    #
+    # @param key [Symbol, nil] the key of the content to check existence
+    # @return [Boolean] true if the indicated content exists
     def exist?(key = nil)
       if key
         File.exist?(properties_filename) && File.exist?(content_filename(key))
@@ -86,6 +140,25 @@ module Gemstash
       end
     end
 
+    # Save one or more files for this resource given by the +content+ hash.
+    # Metadata properties about the file(s) may be provided in the optional
+    # +properties+ parameter. The keys in the content hash correspond to the
+    # file name for this resource, while the values will be the content stored
+    # for that key.
+    #
+    # Separate calls to save for the same resource will replace existing files,
+    # and add new ones. Properties on additional calls will be merged with
+    # existing properties. Nested hashes in properties will also be merged.
+    #
+    # Examples:
+    #
+    #   Gemstash::Storage.for("foo").resource("bar").save(baz: "qux")
+    #   Gemstash::Storage.for("foo").resource("bar").save(baz: "one", qux: "two")
+    #   Gemstash::Storage.for("foo").resource("bar").save({ baz: "qux" }, meta: true)
+    #
+    # @param content [Hash{Symbol => String}] files to save, *must not be nil*
+    # @param properties [Hash, nil] metadata properties related to this resource
+    # @return [Gemstash::Resource] self for chaining purposes
     def save(content, properties = nil)
       content.each do |key, value|
         save_content(key, value)
@@ -95,61 +168,129 @@ module Gemstash
       self
     end
 
+    # Fetch the content for the given +key+. This will load and cache the
+    # properties and the content of the +key+. The +key+ corresponds to the
+    # +content+ key provided to {#save}.
+    #
+    # @param key [Symbol] the key of the content to load
+    # @return [String] the content stored in the +key+
     def content(key)
+      @content ||= {}
+      load(key) unless @content.include?(key)
       @content[key]
     end
 
+    # Fetch the metadata properties for this resource. The properties will be
+    # cached for future calls.
+    #
+    # @return [Hash] the metadata properties for this resource
     def properties
+      load_properties
       @properties || {}
     end
 
+    # Update the metadata properties of this resource. The +props+ will be
+    # merged with any existing properties. Nested hashes in the properties will
+    # also be merged.
+    #
+    # @param props [Hash] the properties to add
+    # @return [Gemstash::Resource] self for chaining purposes
     def update_properties(props)
-      load_properties
+      load_properties(force: true)
+
+      deep_merge = proc do |_, old_value, new_value|
+        if old_value.is_a?(Hash) && new_value.is_a?(Hash)
+          old_value.merge(new_value, &deep_merge)
+        else
+          new_value
+        end
+      end
+
+      props = properties.merge(props || {}, &deep_merge)
       save_properties(properties.merge(props || {}))
       self
     end
 
-    def load(key)
-      raise "Resource #{@name} has no content to load" unless exist?(key)
-      load_properties
-      @content ||= {}
-      @content[key] = read_file(content_filename(key))
-      self
+    # Check if the metadata properties includes the +keys+. The +keys+ represent
+    # a nested path in the properties to check.
+    #
+    # Examples:
+    #
+    #   resource = Gemstash::Storage.for("x").resource("y")
+    #   resource.save({ file: "content" }, foo: "one", bar: { baz: "qux" })
+    #   resource.has_property?(:foo)       # true
+    #   resource.has_property?(:bar, :baz) # true
+    #   resource.has_property?(:missing)   # false
+    #   resource.has_property?(:foo, :bar) # false
+    #
+    # @param keys [Array<Object>] one or more keys pointing to a property
+    # @return [Boolean] whether the nested keys points to a valid property
+    def property?(*keys)
+      keys.inject(node: properties, result: true) do |memo, key|
+        if memo[:result]
+          memo[:result] = memo[:node].is_a?(Hash) && memo[:node].include?(key)
+          memo[:node] = memo[:node][key] if memo[:result]
+        end
+
+        memo
+      end[:result]
     end
 
+    # Delete the content for the given +key+. If the +key+ is the last one for
+    # this resource, the metadata properties will be deleted as well. The +key+
+    # corresponds to the +content+ key provided to {#save}.
+    #
+    # The resource will be reset afterwards, clearing any cached content or
+    # properties.
+    #
+    # Does nothing if the key doesn't {#exist?}.
+    #
+    # @param key [Symbol] the key of the content to delete
+    # @return [Gemstash::Resource] self for chaining purposes
     def delete(key)
       return self unless exist?(key)
 
       begin
         File.delete(content_filename(key))
-      rescue => e
+      rescue StandardError => e
         log_error "Failed to delete stored content at #{content_filename(key)}", e, level: :warn
       end
 
       begin
         File.delete(properties_filename) unless content?
-      rescue => e
+      rescue StandardError => e
         log_error "Failed to delete stored properties at #{properties_filename}", e, level: :warn
       end
 
-      return self
+      self
     ensure
       reset
     end
 
   private
 
-    def load_properties
+    def load(key)
+      raise "Resource #{@name} has no #{key.inspect} content to load" unless exist?(key)
+
+      load_properties # Ensures storage version is checked
+      @content ||= {}
+      @content[key] = read_file(content_filename(key))
+    end
+
+    def load_properties(force: false)
+      return if @properties && !force
       return unless File.exist?(properties_filename)
-      @properties = YAML.load_file(properties_filename)
-      check_version
+
+      @properties = YAML.safe_load_file(properties_filename, permitted_classes: [Symbol]) || {}
+      check_resource_version
     end
 
-    def check_version
-      version = @properties[:gemstash_storage_version]
-      return if version <= Gemstash::Storage::VERSION
+    def check_resource_version
+      version = @properties[:gemstash_resource_version]
+      return if version <= Gemstash::Resource::VERSION
+
       reset
-      raise Gemstash::Storage::VersionTooNew, "Resource was stored with a newer storage: #{version}"
+      raise Gemstash::Resource::VersionTooNew.new(name, folder, version)
     end
 
     def reset
@@ -159,8 +300,9 @@ module Gemstash
 
     def content?
       return false unless Dir.exist?(@folder)
-      entries = Dir.entries(@folder).reject {|file| file =~ /\A\.\.?\z/ }
-      !entries.empty? && entries != %w(properties.yaml)
+
+      entries = Dir.entries(@folder).reject {|file| file =~ /\A\.\.?\z/ || file == "properties.yaml" }
+      !entries.empty?
     end
 
     def sanitize(name)
@@ -175,7 +317,7 @@ module Gemstash
 
     def save_properties(props)
       props ||= {}
-      props = { gemstash_storage_version: Gemstash::Storage::VERSION }.merge(props)
+      props = { gemstash_resource_version: Gemstash::Resource::VERSION }.merge(props)
       store(properties_filename, props.to_yaml)
       @properties = props
     end
@@ -187,7 +329,7 @@ module Gemstash
 
     def save_file(filename)
       content = yield
-      File.open(filename, "wb") {|f| f.write(content) }
+      gemstash_env.atomic_write(filename) {|f| f.write(content) }
     end
 
     def read_file(filename)
@@ -197,6 +339,7 @@ module Gemstash
     def content_filename(key)
       name = sanitize(key.to_s)
       raise "Invalid content key #{key.inspect}" if name.empty?
+
       File.join(@folder, name)
     end
 

data/lib/gemstash/upstream.rb

@@ -1,18 +1,24 @@
+# frozen_string_literal: true
+
 require "digest"
+require "forwardable"
+require "uri"
 
 module Gemstash
-  #:nodoc:
+  # :nodoc:
   class Upstream
     extend Forwardable
 
-    attr_reader :user_agent
+    attr_reader :user_agent, :uri
 
-    def_delegators :@uri, :scheme, :host, :user, :password, :to_s
+    def_delegators :@uri, :scheme, :host, :to_s
 
     def initialize(upstream, user_agent: nil)
-      @uri = URI(URI.decode(upstream.to_s))
+      url = CGI.unescape(upstream.to_s)
+      url = "https://#{url}" unless %r{^https?://}.match?(url)
+      @uri = URI(url)
       @user_agent = user_agent
-      raise "URL '#{@uri}' is not valid!" unless @uri.to_s =~ URI.regexp
+      raise "URL '#{@uri}' is not valid!" unless @uri.to_s&.match?(URI::DEFAULT_PARSER.make_regexp)
     end
 
     def url(path = nil, params = nil)
@@ -20,7 +26,7 @@ module Gemstash
 
       unless path.to_s.empty?
         base = "#{base}/" unless base.end_with?("/")
-        path = path[1..-1] if path.to_s.start_with?("/")
+        path = path[1..] if path.to_s.start_with?("/")
       end
 
       params = "?#{params}" if !params.nil? && !params.empty?
@@ -28,7 +34,7 @@ module Gemstash
     end
 
     def auth?
-      !user.to_s.empty? && !password.to_s.empty?
+      !user.to_s.empty? || !password.to_s.empty?
     end
 
     # Utilized as the parent directory for cached gems
@@ -36,13 +42,42 @@ module Gemstash
       @host_id ||= "#{host}_#{hash}"
     end
 
+    def user
+      env_auth_user || @uri.user
+    end
+
+    def password
+      env_auth_pass || @uri.password
+    end
+
   private
 
     def hash
       Digest::MD5.hexdigest(to_s)
     end
 
-    #:nodoc:
+    def env_auth_user
+      return unless env_auth
+
+      env_auth.split(":", 2).first
+    end
+
+    def env_auth_pass
+      return unless env_auth
+      return unless env_auth.include?(":")
+
+      env_auth.split(":", 2).last
+    end
+
+    def env_auth
+      @env_auth ||= ENV["GEMSTASH_#{host_for_env}"]
+    end
+
+    def host_for_env
+      host.upcase.gsub(".", "__").gsub("-", "___")
+    end
+
+    # :nodoc:
     class GemName
       def initialize(upstream, gem_name)
         @upstream = upstream

data/lib/gemstash/version.rb

@@ -1,4 +1,6 @@
-#:nodoc:
+# frozen_string_literal: true
+
+# :nodoc:
 module Gemstash
-  VERSION = "1.0.0.pre.1"
+  VERSION = "2.5.0"
 end

data/lib/gemstash/web.rb

@@ -1,13 +1,17 @@
+# frozen_string_literal: true
+
 require "sinatra/base"
 require "json"
 require "gemstash"
 
 module Gemstash
-  #:nodoc:
+  # :nodoc:
   class Web < Sinatra::Base
-    def initialize(gemstash_env: nil, http_client_builder: nil)
-      @gemstash_env = gemstash_env || Gemstash::Env.new
-      @http_client_builder = http_client_builder || Gemstash::HTTPClient
+    ruby2_keywords def initialize(options = {})
+      raise ArgumentError unless options.is_a?(Hash)
+
+      @gemstash_env = options[:gemstash_env] || Gemstash::Env.new
+      @http_client_builder = options[:http_client_builder] || Gemstash::HTTPClient
       Gemstash::Env.current = @gemstash_env
       super()
     end
@@ -26,6 +30,11 @@ module Gemstash
       body JSON.dump("error" => "Not found", "code" => 404)
     end
 
+    error GemPusher::ExistingVersionError do
+      status 422
+      body JSON.dump("error" => "Version already exists", "code" => 422)
+    end
+
     get "/" do
       @gem_source.serve_root
     end
@@ -46,10 +55,6 @@ module Gemstash
       @gem_source.serve_yank
     end
 
-    put "/api/v1/gems/unyank" do
-      @gem_source.serve_unyank
-    end
-
     post "/api/v1/add_spec.json" do
       @gem_source.serve_add_spec_json
     end