geminabox 0.11.1 → 0.12.0

data/lib/geminabox/incoming_gem.rb

@@ -1,65 +1,70 @@
-class Geminabox::IncomingGem
-  def initialize(gem_data, root_path = Geminabox.settings.data)
-    unless gem_data.respond_to? :read
-      raise ArgumentError, "Expected an instance of IO"
-    end
+module Geminabox
 
-    digest = Digest::SHA1.new
-    if RbConfig::CONFIG["MAJOR"].to_i <= 1 and RbConfig::CONFIG["MINOR"].to_i <= 8
-      @tempfile = Tempfile.new("gem")
-    else
-      @tempfile = Tempfile.new("gem", :encoding => 'binary')
-    end
+  class IncomingGem
+    def initialize(gem_data, root_path = Geminabox.settings.data)
+      unless gem_data.respond_to? :read
+        raise ArgumentError, "Expected an instance of IO"
+      end
 
-    while data = gem_data.read(1024**2)
-      @tempfile.write data
-      digest << data
-    end
+      digest = Digest::SHA1.new
+      if RbConfig::CONFIG["MAJOR"].to_i <= 1 and RbConfig::CONFIG["MINOR"].to_i <= 8
+        @tempfile = Tempfile.new("gem")
+      else
+        @tempfile = Tempfile.new("gem", :encoding => 'binary')
+      end
 
-    @tempfile.close
-    @sha1 = digest.hexdigest
+      while data = gem_data.read(1024**2)
+        @tempfile.write data
+        digest << data
+      end
 
-    @root_path = root_path
-  end
+      @tempfile.close
+      @sha1 = digest.hexdigest
 
-  def gem_data
-    File.open(@tempfile.path, "rb")
-  end
+      @root_path = root_path
+    end
 
-  def valid?
-    spec && spec.name && spec.version
-  rescue Gem::Package::Error
-    false
-  end
+    def gem_data
+      File.open(@tempfile.path, "rb")
+    end
 
-  def spec
-    @spec ||= extract_spec
-  end
+    def valid?
+      spec && spec.name && spec.version
+    rescue Gem::Package::Error
+      false
+    end
+
+    def spec
+      @spec ||= extract_spec
+    end
 
-  def extract_spec
-    if Gem::Package.respond_to? :open
-      Gem::Package.open(gem_data, "r", nil) do |pkg|
-        return pkg.metadata
+    def extract_spec
+      if Gem::Package.respond_to? :open
+        Gem::Package.open(gem_data, "r", nil) do |pkg|
+          return pkg.metadata
+        end
+      else
+        Gem::Package.new(@tempfile.path).spec
       end
-    else
-      Gem::Package.new(@tempfile.path).spec
     end
-  end
 
-  def name
-    unless @name
+    def name
+      @name ||= get_name
+    end
+
+    def get_name
       filename = %W[#{spec.name} #{spec.version}]
       filename.push(spec.platform) if spec.platform && spec.platform != "ruby"
-      @name = filename.join("-") + ".gem"
+      filename.join("-") + ".gem"
     end
-    @name
-  end
 
-  def dest_filename
-    File.join(@root_path, "gems", name)
-  end
+    def dest_filename
+      File.join(@root_path, "gems", name)
+    end
 
-  def hexdigest
-    @sha1
+    def hexdigest
+      @sha1
+    end
   end
+
 end

data/lib/geminabox/proxy.rb

@@ -0,0 +1,13 @@
+
+module Geminabox
+  module Proxy
+    def self.proxy_path(file)
+      File.join File.dirname(__FILE__), 'proxy', file
+    end
+
+    autoload :Hostess,      proxy_path('hostess')
+    autoload :FileHandler,  proxy_path('file_handler')
+    autoload :Splicer,      proxy_path('splicer')
+    autoload :Copier,       proxy_path('copier')
+  end
+end

data/lib/geminabox/proxy/copier.rb

@@ -0,0 +1,29 @@
+
+
+module Geminabox
+  module Proxy
+    class Copier < FileHandler
+
+      def self.copy(file_name)
+        copier = new(file_name)
+        copier.get_file
+        copier
+      end
+
+      def get_file
+        return true if proxy_file_exists?
+        return copy_local if local_file_exists?
+        get_remote
+      end
+
+      def copy_local
+        FileUtils.cp local_path, proxy_path
+      end
+
+      def get_remote
+        File.open(proxy_path, 'w'){|f| f.write(remote_content)}
+      end
+
+    end
+  end
+end

data/lib/geminabox/proxy/file_handler.rb

@@ -0,0 +1,93 @@
+require 'httpclient'
+module Geminabox
+  module Proxy
+    class FileHandler
+
+      attr_reader :file_name
+
+      def initialize(file_name)
+        @file_name = file_name
+        ensure_destination_exists
+      end
+      
+      def local_path
+        File.expand_path(file_name, root_path)
+      end
+
+      def root_path
+        Geminabox.data
+      end
+
+      def local_file_exists?
+        file_exists? local_path
+      end
+      
+      def proxy_file_exists?
+        file_exists? proxy_path
+      end
+
+      def proxy_path
+        File.expand_path(file_name, proxy_folder_path)
+      end
+
+      def file_exists?(path)
+        File.exists? path
+      end
+
+      def proxy_folder_path
+        File.join(root_path, proxy_folder_name)
+      end
+
+      def proxy_folder_name
+        'proxy'
+      end
+
+      def remote_content
+        HTTPClient.get_content(remote_url).force_encoding(encoding)
+      end
+
+      def remote_url
+        "http://rubygems.org/#{file_name}"
+      end
+
+      def local_content
+        File.read(local_path).force_encoding(encoding)
+      end
+
+      private
+      def encoding
+        "UTF-8"
+      end
+
+      def ensure_destination_exists
+        create_local_folder unless local_folder_exists?
+        create_proxy_folder unless proxy_folder_exists?
+      end
+
+      def proxy_file_folder
+        File.dirname proxy_path
+      end
+
+      def proxy_folder_exists?
+        Dir.exists?(proxy_file_folder)
+      end
+
+      def create_proxy_folder
+        FileUtils.mkdir_p(proxy_file_folder)
+      end
+
+      def local_file_folder
+        File.dirname local_path
+      end
+
+      def local_folder_exists?
+        Dir.exists?(local_file_folder)
+      end
+
+      def create_local_folder
+        FileUtils.mkdir_p(local_file_folder)
+      end
+
+    end
+  end
+end

data/lib/geminabox/proxy/hostess.rb

@@ -0,0 +1,81 @@
+require 'sinatra/base'
+
+module Geminabox
+  module Proxy
+    class Hostess < Sinatra::Base
+      attr_accessor :file_handler
+      
+      def serve
+        if file_handler
+          send_file file_handler.proxy_path
+        else
+          send_file(File.expand_path(File.join(Server.data, *request.path_info)), :type => response['Content-Type'])
+        end
+      end
+
+      %w[specs.4.8.gz
+         latest_specs.4.8.gz
+         prerelease_specs.4.8.gz
+      ].each do |index|
+        get "/#{index}" do
+          splice_file index
+          content_type 'application/x-gzip'
+          serve
+        end
+      end
+
+      %w[quick/Marshal.4.8/*.gemspec.rz
+         yaml.Z
+         Marshal.4.8.Z
+      ].each do |deflated_index|
+        get "/#{deflated_index}" do
+          copy_file request.path_info[1..-1]
+          content_type('application/x-deflate')
+          serve
+        end
+      end
+
+      %w[yaml
+         Marshal.4.8
+         specs.4.8
+         latest_specs.4.8
+         prerelease_specs.4.8
+      ].each do |old_index|
+        get "/#{old_index}" do
+          splice_file old_index
+          serve
+        end
+      end
+
+      get "/gems/*.gem" do
+        get_from_rubygems_if_not_local
+        serve
+      end
+
+      private
+      def get_from_rubygems_if_not_local
+
+        file = File.expand_path(File.join(Server.data, *request.path_info))
+
+        unless File.exists?(file)
+          Net::HTTP.start("production.cf.rubygems.org") do |http|
+            path = File.join(*request.path_info)
+            response = http.get(path)
+            GemStore.create(IncomingGem.new(StringIO.new(response.body)))
+          end
+        end
+
+      end
+
+      def splice_file(file_name)
+        self.file_handler = Splicer.make(file_name)
+      end
+
+      def copy_file(file_name)
+        self.file_handler = Copier.copy(file_name)
+      end
+
+
+    end
+  end
+end

data/lib/geminabox/proxy/splicer.rb

@@ -0,0 +1,67 @@
+
+module Geminabox
+  module Proxy
+    class Splicer < FileHandler
+      
+      def self.make(file_name)
+        splicer = new(file_name)
+        splicer.create
+        splicer
+      end
+
+      def create
+        File.open(splice_path, 'w'){|f| f.write(new_content)}
+      end
+
+      def new_content
+        if local_file_exists?
+          merge_content
+        else
+          remote_content
+        end
+      end
+
+      def splice_path
+        proxy_path
+      end
+
+      def splice_folder_path
+        proxy_folder_path
+      end
+
+      def splice_file_exists?
+        file_exists? splice_path
+      end
+
+      def merge_content
+        if gzip?
+          merge_gziped_content
+        else
+          merge_text_content
+        end
+      end
+
+      def gzip?
+        /\.gz$/ =~ file_name
+      end
+
+      private
+      def merge_gziped_content
+        package(unpackage(local_content) | unpackage(remote_content))
+      end
+
+      def unpackage(content)
+        Marshal.load(Gem.gunzip(content))
+      end
+
+      def package(content)
+        Gem.gzip(Marshal.dump(content))
+      end
+
+      def merge_text_content
+        local_content.to_s + remote_content.to_s
+      end
+
+    end
+  end
+end

data/lib/geminabox/rubygems_dependency.rb

@@ -0,0 +1,27 @@
+require 'httpclient'
+require 'json'
+
+module Geminabox
+  module RubygemsDependency
+
+    class << self
+
+      def for(*gems)
+
+        url = [
+          rubygems_uri,
+          '?gems=',
+          gems.map(&:to_s).join(',')
+        ].join
+        body = HTTPClient.get_content(url)
+        JSON.parse(body)
+      end
+
+      def rubygems_uri
+        "https://bundler.rubygems.org/api/v1/dependencies.json"
+      end
+
+    end
+  end
+end
+

data/lib/geminabox/server.rb

@@ -0,0 +1,279 @@
+module Geminabox
+
+  class Server < Sinatra::Base
+    enable :static, :methodoverride
+
+    def self.delegate_to_geminabox(*delegate_methods)
+      delegate_methods.each{|m| set m, Geminabox.send(m)}
+    end
+
+    delegate_to_geminabox(
+      :public_folder,
+      :data,
+      :build_legacy,
+      :incremental_updates,
+      :views,
+      :allow_replace,
+      :gem_permissions,
+      :allow_delete,
+      :rubygems_proxy
+    )
+
+    if Server.rubygems_proxy
+      use Proxy::Hostess
+    else
+      use Hostess
+    end
+
+    class << self
+      def disallow_replace?
+        ! allow_replace
+      end
+
+      def allow_delete?
+        allow_delete
+      end
+
+      def fixup_bundler_rubygems!
+        return if @post_reset_hook_applied
+        Gem.post_reset{ Gem::Specification.all = nil } if defined? Bundler and Gem.respond_to? :post_reset
+        @post_reset_hook_applied = true
+      end
+
+      def reindex(force_rebuild = false)
+        fixup_bundler_rubygems!
+        force_rebuild = true unless incremental_updates
+        if force_rebuild
+          indexer.generate_index
+          dependency_cache.flush
+        else
+          begin
+            require 'geminabox/indexer'
+            updated_gemspecs = Geminabox::Indexer.updated_gemspecs(indexer)
+            Geminabox::Indexer.patch_rubygems_update_index_pre_1_8_25(indexer)
+            indexer.update_index
+            updated_gemspecs.each { |gem| dependency_cache.flush_key(gem.name) }
+          rescue => e
+            puts "#{e.class}:#{e.message}"
+            puts e.backtrace.join("\n")
+            reindex(:force_rebuild)
+          end
+        end
+      end
+
+      def indexer
+        Gem::Indexer.new(data, :build_legacy => build_legacy)
+      end
+
+      def dependency_cache
+        @dependency_cache ||= Geminabox::DiskCache.new(File.join(data, "_cache"))
+      end
+    end
+
+
+
+    before do
+      headers 'X-Powered-By' => "geminabox #{Geminabox::VERSION}"
+    end
+
+    get '/' do
+      @gems = load_gems
+      @index_gems = index_gems(@gems)
+      erb :index
+    end
+
+    get '/atom.xml' do
+      @gems = load_gems
+      erb :atom, :layout => false
+    end
+
+    get '/api/v1/dependencies' do
+      Marshal.dump(gem_list)
+    end
+
+    get '/api/v1/dependencies.json' do
+      gem_list.to_json
+    end
+
+    get '/upload' do
+      erb :upload
+    end
+
+    get '/reindex' do
+      self.class.reindex(:force_rebuild)
+      redirect url("/")
+    end
+
+    get '/gems/:gemname' do
+      gems = Hash[load_gems.by_name]
+      @gem = gems[params[:gemname]]
+      halt 404 unless @gem
+      erb :gem
+    end
+
+    delete '/gems/*.gem' do
+      unless self.class.allow_delete?
+        error_response(403, 'Gem deletion is disabled - see https://github.com/cwninja/geminabox/issues/115')
+      end
+      File.delete file_path if File.exists? file_path
+      self.class.reindex(:force_rebuild)
+      redirect url("/")
+    end
+
+    post '/upload' do
+      unless params[:file] && params[:file][:filename] && (tmpfile = params[:file][:tempfile])
+        @error = "No file selected"
+        halt [400, erb(:upload)]
+      end
+      handle_incoming_gem(Geminabox::IncomingGem.new(tmpfile))
+    end
+
+    post '/api/v1/gems' do
+      begin
+        handle_incoming_gem(Geminabox::IncomingGem.new(request.body))
+      rescue Object => o
+        File.open "/tmp/debug.txt", "a" do |io|
+          io.puts o, o.backtrace
+        end
+      end
+    end
+
+  private
+
+    def handle_incoming_gem(gem)
+      begin
+        GemStore.create(gem, params[:overwrite])
+      rescue GemStoreError => error
+        error_response error.code, error.reason
+      end
+
+      if api_request?
+        "Gem #{gem.name} received and indexed."
+      else
+        redirect url("/")
+      end
+    end
+
+    def api_request?
+      request.accept.first != "text/html"
+    end
+
+    def error_response(code, message)
+      halt [code, message] if api_request?
+      html = <<HTML
+<html>
+  <head><title>Error - #{code}</title></head>
+  <body>
+    <h1>Error - #{code}</h1>
+    <p>#{message}</p>
+  </body>
+</html>
+HTML
+      halt [code, html]
+    end
+
+    def file_path
+      File.expand_path(File.join(settings.data, *request.path_info))
+    end
+
+    def dependency_cache
+      self.class.dependency_cache
+    end
+
+    def all_gems
+      all_gems_with_duplicates.inject(:|)
+    end
+
+    def all_gems_with_duplicates
+      specs_files_paths.map do |specs_file_path|
+        if File.exists?(specs_file_path)
+          Marshal.load(Gem.gunzip(Gem.read_binary(specs_file_path)))
+        else
+          []
+        end
+      end
+    end
+
+    def specs_file_types
+      [:specs, :prerelease_specs]
+    end
+
+    def specs_files_paths
+      specs_file_types.map do |specs_file_type|
+        File.join(settings.data, spec_file_name(specs_file_type))
+      end
+    end
+
+    def spec_file_name(specs_file_type)
+      [specs_file_type, Gem.marshal_version, 'gz'].join('.')
+    end
+
+    def load_gems
+      @loaded_gems ||= Geminabox::GemVersionCollection.new(all_gems)
+    end
+
+    def index_gems(gems)
+      Set.new(gems.map{|gem| gem.name[0..0].downcase})
+    end
+
+    def gem_list
+      settings.rubygems_proxy ? combined_gem_list : local_gem_list
+    end
+
+    def query_gems
+      params[:gems].to_s.split(',')
+    end
+
+    def local_gem_list
+      query_gems.map{|query_gem| gem_dependencies(query_gem) }.flatten(1)
+    end
+
+    def remote_gem_list
+      RubygemsDependency.for(*query_gems)
+    end
+
+    def combined_gem_list
+      GemListMerge.from(local_gem_list, remote_gem_list)
+    end
+
+    helpers do
+      def spec_for(gem_name, version)
+        spec_file = File.join(settings.data, "quick", "Marshal.#{Gem.marshal_version}", "#{gem_name}-#{version}.gemspec.rz")
+        Marshal.load(Gem.inflate(File.read(spec_file))) if File.exists? spec_file
+      end
+
+      # Return a list of versions of gem 'gem_name' with the dependencies of each version.
+      def gem_dependencies(gem_name)
+        dependency_cache.marshal_cache(gem_name) do
+          load_gems.
+            select { |gem| gem_name == gem.name }.
+            map    { |gem| [gem, spec_for(gem.name, gem.number)] }.
+            reject { |(_, spec)| spec.nil? }.
+            map do |(gem, spec)|
+              {
+                :name => gem.name,
+                :number => gem.number.version,
+                :platform => gem.platform,
+                :dependencies => runtime_dependencies(spec)
+              }
+            end
+        end
+      end
+
+
+
+      def runtime_dependencies(spec)
+        spec.
+          dependencies.
+          select { |dep| dep.type == :runtime }.
+          map    { |dep| name_and_requirements_for(dep) }
+      end
+
+      def name_and_requirements_for(dep)
+        name = dep.name.kind_of?(Array) ? dep.name.first : dep.name
+        [name, dep.requirement.to_s]
+      end
+    end
+  end
+
+end