gemstash 1.0.0.pre.1

data/lib/gemstash/storage.rb

@@ -0,0 +1,207 @@
+require "gemstash"
+require "digest"
+require "pathname"
+require "fileutils"
+require "yaml"
+
+module Gemstash
+  #:nodoc:
+  class Storage
+    extend Gemstash::Env::Helper
+    VERSION = 1
+
+    # If the storage engine detects something that was stored with a newer
+    # version of the storage engine, this error will be thrown.
+    class VersionTooNew < StandardError
+    end
+
+    def initialize(folder, root: true)
+      check_engine if root
+      @folder = folder
+      FileUtils.mkpath(@folder) unless Dir.exist?(@folder)
+    end
+
+    def resource(id)
+      Resource.new(@folder, id)
+    end
+
+    def for(child)
+      Storage.new(File.join(@folder, child), root: false)
+    end
+
+    def self.for(name)
+      new(gemstash_env.base_file(name))
+    end
+
+    def self.metadata
+      file = gemstash_env.base_file("metadata.yml")
+
+      unless File.exist?(file)
+        File.write(file, { storage_version: Gemstash::Storage::VERSION,
+                           gemstash_version: Gemstash::VERSION }.to_yaml)
+      end
+
+      YAML.load_file(file)
+    end
+
+  private
+
+    def check_engine
+      version = Gemstash::Storage.metadata[:storage_version]
+      return if version <= Gemstash::Storage::VERSION
+      raise Gemstash::Storage::VersionTooNew, "Storage engine is out of date: #{version}"
+    end
+
+    def path_valid?(path)
+      return false if path.nil?
+      return false unless File.writable?(path)
+      true
+    end
+  end
+
+  #:nodoc:
+  class Resource
+    include Gemstash::Logging
+    attr_reader :name, :folder
+    def initialize(folder, name)
+      @base_path = folder
+      @name = name
+      # Avoid odd characters in paths, in case of issues with the file system
+      safe_name = sanitize(@name)
+      # Use a trie structure to avoid file system limits causing too many files in 1 folder
+      # Downcase to avoid issues with case insensitive file systems
+      trie_parents = safe_name[0...3].downcase.split("")
+      # The digest is included in case the name differs only by case
+      # Some file systems are case insensitive, so such collisions will be a problem
+      digest = Digest::MD5.hexdigest(@name)
+      child_folder = "#{safe_name}-#{digest}"
+      @folder = File.join(@base_path, *trie_parents, child_folder)
+    end
+
+    def exist?(key = nil)
+      if key
+        File.exist?(properties_filename) && File.exist?(content_filename(key))
+      else
+        File.exist?(properties_filename) && content?
+      end
+    end
+
+    def save(content, properties = nil)
+      content.each do |key, value|
+        save_content(key, value)
+      end
+
+      update_properties(properties)
+      self
+    end
+
+    def content(key)
+      @content[key]
+    end
+
+    def properties
+      @properties || {}
+    end
+
+    def update_properties(props)
+      load_properties
+      save_properties(properties.merge(props || {}))
+      self
+    end
+
+    def load(key)
+      raise "Resource #{@name} has no content to load" unless exist?(key)
+      load_properties
+      @content ||= {}
+      @content[key] = read_file(content_filename(key))
+      self
+    end
+
+    def delete(key)
+      return self unless exist?(key)
+
+      begin
+        File.delete(content_filename(key))
+      rescue => e
+        log_error "Failed to delete stored content at #{content_filename(key)}", e, level: :warn
+      end
+
+      begin
+        File.delete(properties_filename) unless content?
+      rescue => e
+        log_error "Failed to delete stored properties at #{properties_filename}", e, level: :warn
+      end
+
+      return self
+    ensure
+      reset
+    end
+
+  private
+
+    def load_properties
+      return unless File.exist?(properties_filename)
+      @properties = YAML.load_file(properties_filename)
+      check_version
+    end
+
+    def check_version
+      version = @properties[:gemstash_storage_version]
+      return if version <= Gemstash::Storage::VERSION
+      reset
+      raise Gemstash::Storage::VersionTooNew, "Resource was stored with a newer storage: #{version}"
+    end
+
+    def reset
+      @content = nil
+      @properties = nil
+    end
+
+    def content?
+      return false unless Dir.exist?(@folder)
+      entries = Dir.entries(@folder).reject {|file| file =~ /\A\.\.?\z/ }
+      !entries.empty? && entries != %w(properties.yaml)
+    end
+
+    def sanitize(name)
+      name.gsub(/[^a-zA-Z0-9_]/, "_")
+    end
+
+    def save_content(key, content)
+      store(content_filename(key), content)
+      @content ||= {}
+      @content[key] = content
+    end
+
+    def save_properties(props)
+      props ||= {}
+      props = { gemstash_storage_version: Gemstash::Storage::VERSION }.merge(props)
+      store(properties_filename, props.to_yaml)
+      @properties = props
+    end
+
+    def store(filename, content)
+      FileUtils.mkpath(@folder) unless Dir.exist?(@folder)
+      save_file(filename) { content }
+    end
+
+    def save_file(filename)
+      content = yield
+      File.open(filename, "wb") {|f| f.write(content) }
+    end
+
+    def read_file(filename)
+      File.open(filename, "rb", &:read)
+    end
+
+    def content_filename(key)
+      name = sanitize(key.to_s)
+      raise "Invalid content key #{key.inspect}" if name.empty?
+      File.join(@folder, name)
+    end
+
+    def properties_filename
+      File.join(@folder, "properties.yaml")
+    end
+  end
+end

data/lib/gemstash/upstream.rb

@@ -0,0 +1,65 @@
+require "digest"
+
+module Gemstash
+  #:nodoc:
+  class Upstream
+    extend Forwardable
+
+    attr_reader :user_agent
+
+    def_delegators :@uri, :scheme, :host, :user, :password, :to_s
+
+    def initialize(upstream, user_agent: nil)
+      @uri = URI(URI.decode(upstream.to_s))
+      @user_agent = user_agent
+      raise "URL '#{@uri}' is not valid!" unless @uri.to_s =~ URI.regexp
+    end
+
+    def url(path = nil, params = nil)
+      base = to_s
+
+      unless path.to_s.empty?
+        base = "#{base}/" unless base.end_with?("/")
+        path = path[1..-1] if path.to_s.start_with?("/")
+      end
+
+      params = "?#{params}" if !params.nil? && !params.empty?
+      "#{base}#{path}#{params}"
+    end
+
+    def auth?
+      !user.to_s.empty? && !password.to_s.empty?
+    end
+
+    # Utilized as the parent directory for cached gems
+    def host_id
+      @host_id ||= "#{host}_#{hash}"
+    end
+
+  private
+
+    def hash
+      Digest::MD5.hexdigest(to_s)
+    end
+
+    #:nodoc:
+    class GemName
+      def initialize(upstream, gem_name)
+        @upstream = upstream
+        @id = gem_name
+      end
+
+      def to_s
+        name
+      end
+
+      def id
+        @id
+      end
+
+      def name
+        @name ||= @id.gsub(/\.gem(spec\.rz)?$/i, "")
+      end
+    end
+  end
+end

data/lib/gemstash/version.rb

@@ -0,0 +1,4 @@
+#:nodoc:
+module Gemstash
+  VERSION = "1.0.0.pre.1"
+end

data/lib/gemstash/web.rb

@@ -0,0 +1,97 @@
+require "sinatra/base"
+require "json"
+require "gemstash"
+
+module Gemstash
+  #:nodoc:
+  class Web < Sinatra::Base
+    def initialize(gemstash_env: nil, http_client_builder: nil)
+      @gemstash_env = gemstash_env || Gemstash::Env.new
+      @http_client_builder = http_client_builder || Gemstash::HTTPClient
+      Gemstash::Env.current = @gemstash_env
+      super()
+    end
+
+    before do
+      Gemstash::Env.current = @gemstash_env
+      @gem_source = env["gemstash.gem_source"].new(self)
+    end
+
+    def http_client_for(server_url)
+      @http_client_builder.for(server_url)
+    end
+
+    not_found do
+      status 404
+      body JSON.dump("error" => "Not found", "code" => 404)
+    end
+
+    get "/" do
+      @gem_source.serve_root
+    end
+
+    get "/api/v1/dependencies" do
+      @gem_source.serve_dependencies
+    end
+
+    get "/api/v1/dependencies.json" do
+      @gem_source.serve_dependencies_json
+    end
+
+    post "/api/v1/gems" do
+      @gem_source.serve_add_gem
+    end
+
+    delete "/api/v1/gems/yank" do
+      @gem_source.serve_yank
+    end
+
+    put "/api/v1/gems/unyank" do
+      @gem_source.serve_unyank
+    end
+
+    post "/api/v1/add_spec.json" do
+      @gem_source.serve_add_spec_json
+    end
+
+    post "/api/v1/remove_spec.json" do
+      @gem_source.serve_remove_spec_json
+    end
+
+    get "/names" do
+      @gem_source.serve_names
+    end
+
+    get "/versions" do
+      @gem_source.serve_versions
+    end
+
+    get "/info/:name" do
+      @gem_source.serve_info(params[:name])
+    end
+
+    get "/quick/Marshal.4.8/:id" do
+      @gem_source.serve_marshal(params[:id])
+    end
+
+    get "/fetch/actual/gem/:id" do
+      @gem_source.serve_actual_gem(params[:id])
+    end
+
+    get "/gems/:id" do
+      @gem_source.serve_gem(params[:id])
+    end
+
+    get "/latest_specs.4.8.gz" do
+      @gem_source.serve_latest_specs
+    end
+
+    get "/specs.4.8.gz" do
+      @gem_source.serve_specs
+    end
+
+    get "/prerelease_specs.4.8.gz" do
+      @gem_source.serve_prerelease_specs
+    end
+  end
+end

metadata

@@ -0,0 +1,306 @@
+--- !ruby/object:Gem::Specification
+name: gemstash
+version: !ruby/object:Gem::Version
+  version: 1.0.0.pre.1
+platform: ruby
+authors:
+- Andre Arko
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-11-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dalli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: lru_redux
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.26'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.26'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.34'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.34'
+description: Gemstash acts as a local RubyGems server, caching copies of gems from
+  RubyGems.org automatically, and eventually letting you push your own private gems
+  as well.
+email:
+- andre@arko.net
+executables:
+- gemstash
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop-bundler.yml"
+- ".rubocop-relax.yml"
+- ".rubocop.yml"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/gemstash
+- bin/setup
+- docs/config.md
+- docs/debug.md
+- docs/deploy.md
+- docs/mirror.md
+- docs/multiple_sources.md
+- docs/private_gems.md
+- docs/reference.md
+- exe/gemstash
+- gemstash.gemspec
+- gemstash.png
+- lib/gemstash.rb
+- lib/gemstash/authorization.rb
+- lib/gemstash/cache.rb
+- lib/gemstash/cli.rb
+- lib/gemstash/cli/authorize.rb
+- lib/gemstash/cli/base.rb
+- lib/gemstash/cli/setup.rb
+- lib/gemstash/cli/start.rb
+- lib/gemstash/cli/status.rb
+- lib/gemstash/cli/stop.rb
+- lib/gemstash/config.ru
+- lib/gemstash/configuration.rb
+- lib/gemstash/db.rb
+- lib/gemstash/db/authorization.rb
+- lib/gemstash/db/dependency.rb
+- lib/gemstash/db/rubygem.rb
+- lib/gemstash/db/version.rb
+- lib/gemstash/dependencies.rb
+- lib/gemstash/env.rb
+- lib/gemstash/gem_fetcher.rb
+- lib/gemstash/gem_pusher.rb
+- lib/gemstash/gem_source.rb
+- lib/gemstash/gem_source/dependency_caching.rb
+- lib/gemstash/gem_source/private_source.rb
+- lib/gemstash/gem_source/rack_middleware.rb
+- lib/gemstash/gem_source/upstream_source.rb
+- lib/gemstash/gem_unyanker.rb
+- lib/gemstash/gem_yanker.rb
+- lib/gemstash/http_client.rb
+- lib/gemstash/logging.rb
+- lib/gemstash/migrations/01_gem_dependencies.rb
+- lib/gemstash/migrations/02_authorizations.rb
+- lib/gemstash/puma.rb
+- lib/gemstash/rack_env_rewriter.rb
+- lib/gemstash/specs_builder.rb
+- lib/gemstash/storage.rb
+- lib/gemstash/upstream.rb
+- lib/gemstash/version.rb
+- lib/gemstash/web.rb
+homepage: https://github.com/bundler/gemstash
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: A place to stash gems you'll need
+test_files: []