r5_geminabox 0.0.1

data/lib/geminabox/proxy/file_handler.rb

@@ -0,0 +1,95 @@
+
+module Geminabox
+  module Proxy
+    class FileHandler
+
+      attr_reader :file_name
+
+      def initialize(file_name)
+        @file_name = file_name
+        ensure_destination_exists
+      end
+      
+      def local_path
+        File.expand_path(file_name, root_path)
+      end
+
+      def root_path
+        Geminabox.data
+      end
+
+      def local_file_exists?
+        file_exists? local_path
+      end
+      
+      def proxy_file_exists?
+        file_exists? proxy_path
+      end
+
+      def proxy_path
+        File.expand_path(file_name, proxy_folder_path)
+      end
+
+      def file_exists?(path)
+        File.exists? path
+      end
+
+      def proxy_folder_path
+        File.join(root_path, proxy_folder_name)
+      end
+
+      def proxy_folder_name
+        'proxy'
+      end
+
+      def remote_content
+        Geminabox.http_adapter.get_content(remote_url).force_encoding(encoding)
+      rescue
+        raise GemStoreError.new(500, "Unable to get content from #{remote_url}")
+      end
+
+      def remote_url
+        "http://rubygems.org/#{file_name}"
+      end
+
+      def local_content
+        File.read(local_path).force_encoding(encoding)
+      end
+
+      private
+      def encoding
+        "UTF-8"
+      end
+
+      def ensure_destination_exists
+        create_local_folder unless local_folder_exists?
+        create_proxy_folder unless proxy_folder_exists?
+      end
+
+      def proxy_file_folder
+        File.dirname proxy_path
+      end
+
+      def proxy_folder_exists?
+        Dir.exists?(proxy_file_folder)
+      end
+
+      def create_proxy_folder
+        FileUtils.mkdir_p(proxy_file_folder)
+      end
+
+      def local_file_folder
+        File.dirname local_path
+      end
+
+      def local_folder_exists?
+        Dir.exists?(local_file_folder)
+      end
+
+      def create_local_folder
+        FileUtils.mkdir_p(local_file_folder)
+      end
+
+    end
+  end
+end

data/lib/geminabox/proxy/hostess.rb

@@ -0,0 +1,81 @@
+require 'sinatra/base'
+require 'net/http'
+
+module Geminabox
+  module Proxy
+    class Hostess < Sinatra::Base
+      attr_accessor :file_handler
+      
+      def serve
+        if file_handler
+          send_file file_handler.proxy_path
+        else
+          send_file(File.expand_path(File.join(Server.data, *request.path_info)), :type => response['Content-Type'])
+        end
+      end
+
+      %w[specs.4.8.gz
+         latest_specs.4.8.gz
+         prerelease_specs.4.8.gz
+      ].each do |index|
+        get "/#{index}" do
+          splice_file index
+          content_type 'application/x-gzip'
+          serve
+        end
+      end
+
+      %w[quick/Marshal.4.8/*.gemspec.rz
+         yaml.Z
+         Marshal.4.8.Z
+      ].each do |deflated_index|
+        get "/#{deflated_index}" do
+          copy_file request.path_info[1..-1]
+          content_type('application/x-deflate')
+          serve
+        end
+      end
+
+      %w[yaml
+         Marshal.4.8
+         specs.4.8
+         latest_specs.4.8
+         prerelease_specs.4.8
+      ].each do |old_index|
+        get "/#{old_index}" do
+          splice_file old_index
+          serve
+        end
+      end
+
+      get "/gems/*.gem" do
+        get_from_rubygems_if_not_local
+        serve
+      end
+
+      private
+      def get_from_rubygems_if_not_local
+
+        file = File.expand_path(File.join(Server.data, *request.path_info))
+
+        unless File.exists?(file)
+          ruby_gems_url = 'http://production.cf.rubygems.org'
+          path = File.join(ruby_gems_url, *request.path_info)
+          content = Geminabox.http_adapter.get_content(path)
+          GemStore.create(IncomingGem.new(StringIO.new(content)))
+        end
+
+      end
+
+      def splice_file(file_name)
+        self.file_handler = Splicer.make(file_name)
+      end
+
+      def copy_file(file_name)
+        self.file_handler = Copier.copy(file_name)
+      end
+
+
+    end
+  end
+end

data/lib/geminabox/proxy/splicer.rb

@@ -0,0 +1,67 @@
+
+module Geminabox
+  module Proxy
+    class Splicer < FileHandler
+      
+      def self.make(file_name)
+        splicer = new(file_name)
+        splicer.create
+        splicer
+      end
+
+      def create
+        File.open(splice_path, 'w'){|f| f.write(new_content)}
+      end
+
+      def new_content
+        if local_file_exists?
+          merge_content
+        else
+          remote_content
+        end
+      end
+
+      def splice_path
+        proxy_path
+      end
+
+      def splice_folder_path
+        proxy_folder_path
+      end
+
+      def splice_file_exists?
+        file_exists? splice_path
+      end
+
+      def merge_content
+        if gzip?
+          merge_gziped_content
+        else
+          merge_text_content
+        end
+      end
+
+      def gzip?
+        /\.gz$/ =~ file_name
+      end
+
+      private
+      def merge_gziped_content
+        package(unpackage(local_content) | unpackage(remote_content))
+      end
+
+      def unpackage(content)
+        Marshal.load(Gem.gunzip(content))
+      end
+
+      def package(content)
+        Gem.gzip(Marshal.dump(content))
+      end
+
+      def merge_text_content
+        local_content.to_s + remote_content.to_s
+      end
+
+    end
+  end
+end

data/lib/geminabox/rubygems_dependency.rb

@@ -0,0 +1,29 @@
+require 'json'
+
+module Geminabox
+  module RubygemsDependency
+
+    class << self
+
+      def for(*gems)
+
+        url = [
+          rubygems_uri,
+          '?gems=',
+          gems.map(&:to_s).join(',')
+        ].join
+        body = Geminabox.http_adapter.get_content(url)
+        JSON.parse(body)
+      rescue Exception => e
+        return [] if Geminabox.allow_remote_failure
+        raise e
+      end
+
+      def rubygems_uri
+        "https://bundler.rubygems.org/api/v1/dependencies.json"
+      end
+
+    end
+  end
+end
+

data/lib/geminabox/server.rb

@@ -0,0 +1,290 @@
+module Geminabox
+
+  class Server < Sinatra::Base
+    enable :static, :methodoverride
+
+    def self.delegate_to_geminabox(*delegate_methods)
+      delegate_methods.each{|m| set m, Geminabox.send(m)}
+    end
+
+    delegate_to_geminabox(
+      :public_folder,
+      :data,
+      :build_legacy,
+      :incremental_updates,
+      :views,
+      :allow_replace,
+      :gem_permissions,
+      :allow_delete,
+      :rubygems_proxy
+    )
+
+    if Server.rubygems_proxy
+      use Proxy::Hostess
+    else
+      use Hostess
+    end
+
+    class << self
+      def disallow_replace?
+        ! allow_replace
+      end
+
+      def allow_delete?
+        allow_delete
+      end
+
+      def fixup_bundler_rubygems!
+        return if @post_reset_hook_applied
+        Gem.post_reset{ Gem::Specification.all = nil } if defined? Bundler and Gem.respond_to? :post_reset
+        @post_reset_hook_applied = true
+      end
+
+      def reindex(force_rebuild = false)
+        fixup_bundler_rubygems!
+        force_rebuild = true unless incremental_updates
+        if force_rebuild
+          indexer.generate_index
+          dependency_cache.flush
+        else
+          begin
+            require 'geminabox/indexer'
+            updated_gemspecs = Geminabox::Indexer.updated_gemspecs(indexer)
+            Geminabox::Indexer.patch_rubygems_update_index_pre_1_8_25(indexer)
+            indexer.update_index
+            updated_gemspecs.each { |gem| dependency_cache.flush_key(gem.name) }
+          rescue => e
+            puts "#{e.class}:#{e.message}"
+            puts e.backtrace.join("\n")
+            reindex(:force_rebuild)
+          end
+        end
+      end
+
+      def indexer
+        Gem::Indexer.new(data, :build_legacy => build_legacy)
+      end
+
+      def dependency_cache
+        @dependency_cache ||= Geminabox::DiskCache.new(File.join(data, "_cache"))
+      end
+    end
+
+
+
+    before do
+      headers 'X-Powered-By' => "geminabox #{Geminabox::VERSION}"
+    end
+
+    get '/' do
+      @gems = load_gems
+      @index_gems = index_gems(@gems)
+      erb :index
+    end
+
+    get '/atom.xml' do
+      @gems = load_gems
+      erb :atom, :layout => false
+    end
+
+    get '/api/v1/dependencies' do
+      query_gems.any? ? Marshal.dump(gem_list) : 200
+    end
+
+    get '/api/v1/dependencies.json' do
+      query_gems.any? ? gem_list.to_json : {}
+    end
+
+    get '/upload' do
+      erb :upload
+    end
+
+    get '/reindex' do
+      self.class.reindex(:force_rebuild)
+      redirect url("/")
+    end
+
+    get '/gems/:gemname' do
+      gems = Hash[load_gems.by_name]
+      @gem = gems[params[:gemname]]
+      halt 404 unless @gem
+      erb :gem
+    end
+
+    delete '/gems/*.gem' do
+      unless self.class.allow_delete?
+        error_response(403, 'Gem deletion is disabled - see https://github.com/cwninja/geminabox/issues/115')
+      end
+      File.delete file_path if File.exists? file_path
+      self.class.reindex(:force_rebuild)
+      redirect url("/")
+    end
+
+    post '/upload' do
+      unless params[:file] && params[:file][:filename] && (tmpfile = params[:file][:tempfile])
+        @error = "No file selected"
+        halt [400, erb(:upload)]
+      end
+      handle_incoming_gem(Geminabox::IncomingGem.new(tmpfile))
+    end
+
+    post '/api/v1/gems' do
+      begin
+        handle_incoming_gem(Geminabox::IncomingGem.new(request.body))
+      rescue Object => o
+        File.open "/tmp/debug.txt", "a" do |io|
+          io.puts o, o.backtrace
+        end
+      end
+    end
+
+  private
+
+    def handle_incoming_gem(gem)
+      begin
+        GemStore.create(gem, params[:overwrite])
+      rescue GemStoreError => error
+        error_response error.code, error.reason
+      end
+
+      if api_request?
+        "Gem #{gem.name} received and indexed."
+      else
+        redirect url("/")
+      end
+      sleep 10
+      redirect url("/")
+    end
+
+    def api_request?
+      request.accept.first.to_s != "text/html"
+    end
+
+    def error_response(code, message)
+      halt [code, message] if api_request?
+      html = <<HTML
+<html>
+  <head><title>Error - #{code}</title></head>
+  <body>
+    <h1>Error - #{code}</h1>
+    <p>#{message}</p>
+  </body>
+</html>
+HTML
+      halt [code, html]
+    end
+
+    def file_path
+      File.expand_path(File.join(settings.data, *request.path_info))
+    end
+
+    def dependency_cache
+      self.class.dependency_cache
+    end
+
+    def all_gems
+      all_gems_with_duplicates.inject(:|)
+    end
+
+    def all_gems_with_duplicates
+      specs_files_paths.map do |specs_file_path|
+        if File.exists?(specs_file_path)
+          Marshal.load(Gem.gunzip(Gem.read_binary(specs_file_path)))
+        else
+          []
+        end
+      end
+    end
+
+    def specs_file_types
+      [:specs, :prerelease_specs]
+    end
+
+    def specs_files_paths
+      specs_file_types.map do |specs_file_type|
+        File.join(settings.data, spec_file_name(specs_file_type))
+      end
+    end
+
+    def spec_file_name(specs_file_type)
+      [specs_file_type, Gem.marshal_version, 'gz'].join('.')
+    end
+
+    def load_gems
+      @loaded_gems ||= Geminabox::GemVersionCollection.new(all_gems)
+    end
+
+    def index_gems(gems)
+      Set.new(gems.map{|gem| gem.name[0..0].downcase})
+    end
+
+    def gem_list
+      settings.rubygems_proxy ? combined_gem_list : local_gem_list
+    end
+
+    def query_gems
+      params[:gems].to_s.split(',')
+    end
+
+    def local_gem_list
+      query_gems.map{|query_gem| gem_dependencies(query_gem) }.flatten(1)
+    end
+
+    def remote_gem_list
+      RubygemsDependency.for(*query_gems)
+    end
+
+    def combined_gem_list
+      GemListMerge.from(local_gem_list, remote_gem_list)
+    end
+
+    helpers do
+      def spec_for(gem_name, version, platform = default_platform)
+        filename = [gem_name, version]
+        filename.push(platform) if platform != default_platform
+        spec_file = File.join(settings.data, "quick", "Marshal.#{Gem.marshal_version}", "#{filename.join("-")}.gemspec.rz")
+        File::open(spec_file, 'r') do |unzipped_spec_file|
+          unzipped_spec_file.binmode
+          Marshal.load(Gem.inflate(unzipped_spec_file.read))
+        end if File.exists? spec_file
+      end
+
+      def default_platform
+        'ruby'
+      end
+
+      # Return a list of versions of gem 'gem_name' with the dependencies of each version.
+      def gem_dependencies(gem_name)
+        dependency_cache.marshal_cache(gem_name) do
+          load_gems.
+            select { |gem| gem_name == gem.name }.
+            map    { |gem| [gem, spec_for(gem.name, gem.number, gem.platform)] }.
+            reject { |(_, spec)| spec.nil? }.
+            map do |(gem, spec)|
+              {
+                :name => gem.name,
+                :number => gem.number.version,
+                :platform => gem.platform,
+                :dependencies => runtime_dependencies(spec)
+              }
+            end
+        end
+      end
+
+
+
+      def runtime_dependencies(spec)
+        spec.
+          dependencies.
+          select { |dep| dep.type == :runtime }.
+          map    { |dep| name_and_requirements_for(dep) }
+      end
+
+      def name_and_requirements_for(dep)
+        name = dep.name.kind_of?(Array) ? dep.name.first : dep.name
+        [name, dep.requirement.to_s]
+      end
+    end
+  end
+
+end