riak-client 1.0.0.beta → 1.0.0

data/lib/riak/client/beefcake_protobuffs_backend.rb

@@ -3,7 +3,6 @@ require 'riak/json'
 require 'riak/client'
 require 'riak/failed_request'
 require 'riak/client/protobuffs_backend'
-require 'riak/client/pump'
 
 module Riak
   class Client
@@ -91,11 +90,10 @@ module Riak
         req = RpbListKeysReq.new(:bucket => maybe_encode(bucket))
         write_protobuff(:ListKeysReq, req)
         keys = []
-        pump = Pump.new(block) if block_given?
         while msg = decode_response
           break if msg.done
-          if pump
-            pump.pump msg.keys
+          if block_given?
+            yield msg.keys
           else
             keys += msg.keys
           end
@@ -107,13 +105,10 @@ module Riak
         req = RpbMapRedReq.new(:request => mr.to_json, :content_type => "application/json")
         write_protobuff(:MapRedReq, req)
         results = []
-        pump = Pump.new(lambda do |msg|
-          block.call msg.phase, JSON.parse(msg.response)
-        end) if block_given?
         while msg = decode_response
           break if msg.done
-          if pump
-            pump.pump msg
+          if block_given?
+            yield msg.phase, JSON.parse(msg.response)
           else
             results[msg.phase] ||= []
             results[msg.phase] += JSON.parse(msg.response)
@@ -176,7 +171,8 @@ module Riak
         end
       rescue SystemCallError, SocketError => e
         reset_socket
-        raise Riak::ProtobuffsFailedRequest.new(:server_error, e.message)
+        raise
+        #raise Riak::ProtobuffsFailedRequest.new(:server_error, e.message)
       end
     end
   end

data/lib/riak/client/decaying.rb

@@ -0,0 +1,28 @@
+class Riak::Client::Decaying
+  # A float which decays exponentially with time.
+
+  attr_accessor :e
+  attr_accessor :p
+  # @param[:p] The initial value
+  # @param[:e] Exponent base
+  # @param[:r] Timescale
+  def initialize(opts = {})
+    @p = opts[:p] || 0
+    @e = opts[:e] || Math::E
+    @r = opts[:r] || Math.log(0.5) / 10
+    @t0 = Time.now
+  end
+
+  # Add d to current value.
+  def <<(d)
+    @p = value + d
+  end
+
+  # Return current value
+  def value
+    now = Time.now
+    dt = now - @t0
+    @t0 = now
+    @p = @p * (@e ** (@r * dt))
+  end
+end

data/lib/riak/client/excon_backend.rb

@@ -1,8 +1,6 @@
-
 require 'riak/failed_request'
 require 'riak/client/http_backend'
 require 'riak/client/http_backend/request_headers'
-require 'riak/client/pump'
 
 module Riak
   class Client
@@ -13,15 +11,36 @@ module Riak
       def self.configured?
         begin
           require 'excon'
-          Excon::VERSION >= "0.5.7"
+          Client::NETWORK_ERRORS << Excon::Errors::SocketError
+          Client::NETWORK_ERRORS.uniq!
+          Excon::VERSION >= "0.5.7" && patch_excon
         rescue LoadError
           false
         end
       end
 
+      # Adjusts Excon's connection collection to allow multiple
+      # connections to the same host from the same Thread. Instead we
+      # use the Riak::Client::Pool to segregate connections.
+      # @note This can be changed when Excon has a proper pool of its own.
+      def self.patch_excon
+        unless defined? @@patched
+          ::Excon::Connection.class_eval do
+            def sockets
+              @sockets ||= {}
+            end
+          end
+        end
+        @@patched = true
+      end
+
+      def teardown
+        connection.reset
+      end
+
       private
       def perform(method, uri, headers, expect, data=nil, &block)
-        configure_ssl if @client.ssl_enabled?
+        configure_ssl if @node.ssl_enabled?
 
         params = {
           :method => method.to_s.upcase,
@@ -32,11 +51,8 @@ module Riak
         params[:body] = data if [:put,:post].include?(method)
         params[:idempotent] = (method != :post)
 
-        if block_given?
-          pump = Pump.new(block)
-          # Later versions of Excon pass multiple arguments to the block
-          block = lambda {|*args| pump.pump(args.first) }
-        end
+        # Later versions of Excon pass multiple arguments to the block
+        block = lambda {|*args| yield args.first } if block_given?
 
         response = connection.request(params, &block)
         response_headers.initialize_http_header(response.headers)
@@ -57,8 +73,8 @@ module Riak
       end
 
       def configure_ssl
-        Excon.ssl_verify_peer = @client.ssl_options[:verify_mode].to_s === "peer"
-        Excon.ssl_ca_path     = @client.ssl_options[:ca_path] if @client.ssl_options[:ca_path]
+        Excon.ssl_verify_peer = @node.ssl_options[:verify_mode].to_s === "peer"
+        Excon.ssl_ca_path     = @node.ssl_options[:ca_path] if @node.ssl_options[:ca_path]
       end
     end
   end

data/lib/riak/client/http_backend.rb

@@ -29,11 +29,17 @@ module Riak
       # The Riak::Client that uses this backend
       attr_reader :client
 
+      # The Riak::Client::Node that uses this backend
+      attr_reader :node
+
       # Create an HTTPBackend for the Riak::Client.
-      # @param [Client] client the client
-      def initialize(client)
+      # @param [Client] The client
+      # @param [Node] The node we're connecting to.
+      def initialize(client, node)
         raise ArgumentError, t("client_type", :client => client) unless Client === client
+        raise ArgumentError, t("node_type", :node => node) unless Node === node
         @client = client
+        @node = node
       end
 
       # Pings the server
@@ -246,6 +252,69 @@ module Riak
       def update_search_index(index, updates)
         post(200, solr_update_path(index), updates, {'Content-Type' => 'text/xml'})
       end
+
+      # (Luwak) Fetches a file from the Luwak large-file interface.
+      # @param [String] filename the name of the file
+      # @yield [chunk] A block which will receive individual chunks of
+      #   the file as they are streamed
+      # @yieldparam [String] chunk a block of the file
+      # @return [IO, nil] the file (also having content_type and
+      #   original_filename accessors). The file will need to be
+      #   reopened to be read
+      def get_file(filename, &block)
+        if block_given?
+          get(200, luwak_path(filename), &block)
+          nil
+        else
+          tmpfile = LuwakFile.new(escape(filename))
+          begin
+            response = get(200, luwak_path(filename)) do |chunk|
+              tmpfile.write chunk
+            end
+            tmpfile.content_type = response[:headers]['content-type'].first
+            tmpfile
+          ensure
+            tmpfile.close
+          end
+        end
+      end
+
+      # (Luwak) Detects whether a file exists in the Luwak large-file
+      # interface.
+      # @param [String] filename the name of the file
+      # @return [true,false] whether the file exists
+      def file_exists?(filename)
+        result = head([200,404], luwak_path(filename))
+        result[:code] == 200
+      end
+
+      # (Luwak) Deletes a file from the Luwak large-file interface.
+      # @param [String] filename the name of the file
+      def delete_file(filename)
+        delete([204,404], luwak_path(filename))
+      end
+
+      # (Luwak) Uploads a file to the Luwak large-file interface.
+      # @overload store_file(filename, content_type, data)
+      #   Stores the file at the given key/filename
+      #   @param [String] filename the key/filename for the object
+      #   @param [String] content_type the MIME Content-Type for the data
+      #   @param [IO, String] data the contents of the file
+      # @overload store_file(content_type, data)
+      #   Stores the file with a server-determined key/filename
+      #   @param [String] content_type the MIME Content-Type for the data
+      #   @param [String, #read] data the contents of the file
+      # @return [String] the key/filename where the object was stored
+      def store_file(*args)
+        data, content_type, filename = args.reverse
+        if filename
+          put(204, luwak_path(filename), data, {"Content-Type" => content_type})
+          filename
+        else
+          response = post(201, luwak_path(nil), data, {"Content-Type" => content_type})
+          response[:headers]["location"].first.split("/").last
+        end
+      end
     end
   end
 end

data/lib/riak/client/http_backend/configuration.rb

@@ -132,6 +132,16 @@ module Riak
           end
         end
 
+        # @return [URI] a URL path for the Luwak interface
+        def luwak_path(key)
+          raise t('luwak_unsupported') unless luwak_wm_file
+          if key
+            path(luwak_wm_file, escape(key))
+          else
+            path(luwak_wm_file)
+          end
+        end
+
         private
         def server_config
           @server_config ||= {}.tap do |hash|
@@ -152,15 +162,15 @@ module Riak
         end
 
         def riak_kv_wm_raw
-          server_config[:riak_kv_wm_raw] || client.http_paths[:prefix]
+          server_config[:riak_kv_wm_raw] || node.http_paths[:prefix]
         end
 
         def riak_kv_wm_link_walker
-          server_config[:riak_kv_wm_link_walker] || client.http_paths[:prefix]
+          server_config[:riak_kv_wm_link_walker] || node.http_paths[:prefix]
         end
 
         def riak_kv_wm_mapred
-          server_config[:riak_kv_wm_mapred] || client.http_paths[:mapred]
+          server_config[:riak_kv_wm_mapred] || node.http_paths[:mapred]
         end
 
         def riak_kv_wm_ping
@@ -178,6 +188,10 @@ module Riak
         def riak_solr_indexer_wm
           server_config[:riak_solr_indexer_wm]
         end
+
+        def luwak_wm_file
+          server_config[:luwak_wm_file]
+        end
       end
     end
   end

data/lib/riak/client/http_backend/transport_methods.rb

@@ -140,13 +140,13 @@ module Riak
         end
 
         def basic_auth_header
-          @client.basic_auth ? {"Authorization" => "Basic #{Base64::encode64(@client.basic_auth)}"} : {}
+          @node.basic_auth ? {"Authorization" => "Basic #{Base64::encode64(@node.basic_auth)}"} : {}
         end
 
         # @return [URI] The calculated root URI for the Riak HTTP endpoint
         def root_uri
-          protocol = client.ssl_enabled? ? "https" : "http"
-          URI.parse("#{protocol}://#{client.host}:#{client.http_port}")
+          protocol = node.ssl_enabled? ? "https" : "http"
+          URI.parse("#{protocol}://#{node.host}:#{node.http_port}")
         end
 
         # Calculates an absolute URI from a relative path specification

data/lib/riak/client/net_http_backend.rb

@@ -27,11 +27,15 @@ module Riak
         @read_timeout ||= 4096
       end
 
+      # Net::HTTP doesn't use persistent connections, so there's no
+      # work to do here.
+      def teardown; end
+
       private
       def perform(method, uri, headers, expect, data=nil) #:nodoc:
         http = Net::HTTP.new(uri.host, uri.port)
         http.read_timeout = self.class.read_timeout
-        configure_ssl(http) if @client.ssl_enabled?
+        configure_ssl(http) if @node.ssl_enabled?
 
         request = Net::HTTP.const_get(method.to_s.capitalize).new(uri.request_uri, headers)
         case data
@@ -51,28 +55,28 @@ module Riak
 
         {}.tap do |result|
           http.request(request) do |response|
-            if valid_response?(expect, response.code)
-              result.merge!({:headers => response.to_hash, :code => response.code.to_i})
-              response.read_body {|chunk| yield chunk } if block_given?
-              if return_body?(method, response.code, block_given?)
-                result[:body] = response.body
-              end
-            else
+            unless valid_response?(expect, response.code)
               raise Riak::HTTPFailedRequest.new(method, expect, response.code.to_i, response.to_hash, response.body)
             end
+
+            result.merge!({:headers => response.to_hash, :code => response.code.to_i})
+            response.read_body {|chunk| yield chunk } if block_given?
+            if return_body?(method, response.code, block_given?)
+              result[:body] = response.body
+            end
           end
         end
       end
 
       def configure_ssl(http)
         http.use_ssl = true
-        http.verify_mode = OpenSSL::SSL.const_get("VERIFY_#{@client.ssl_options[:verify_mode].upcase}")
-        if @client.ssl_options[:pem]
-          http.cert = OpenSSL::X509::Certificate.new(@client.ssl_options[:pem])
-          http.key  = OpenSSL::PKey::RSA.new(@client.ssl_options[:pem], @client.ssl_options[:pem_password])
+        http.verify_mode = OpenSSL::SSL.const_get("VERIFY_#{@node.ssl_options[:verify_mode].upcase}")
+        if @node.ssl_options[:pem]
+          http.cert = OpenSSL::X509::Certificate.new(@node.ssl_options[:pem])
+          http.key  = OpenSSL::PKey::RSA.new(@node.ssl_options[:pem], @node.ssl_options[:pem_password])
         end
-        http.ca_file = @client.ssl_options[:ca_file] if @client.ssl_options[:ca_file]
-        http.ca_path = @client.ssl_options[:ca_path] if @client.ssl_options[:ca_path]
+        http.ca_file = @node.ssl_options[:ca_file] if @node.ssl_options[:ca_file]
+        http.ca_path = @node.ssl_options[:ca_path] if @node.ssl_options[:ca_path]
       end
     end
   end

data/lib/riak/client/node.rb

@@ -0,0 +1,111 @@
+module Riak
+  class Client
+    class Node
+      # Represents a single riak node in a cluster.
+
+      include Util::Translation
+      include Util::Escape
+
+      VALID_OPTIONS = [:host, :http_port, :pb_port, :http_paths, :prefix,
+        :mapred, :luwak, :solr, :port, :basic_auth, :ssl_options, :ssl]
+
+      # For a score which halves in 10 seconds, choose
+      # ln(1/2)/10
+      ERRORS_DECAY_RATE = Math.log(0.5)/10
+
+      # What IP address or hostname does this node listen on?
+      attr_accessor :host
+      # Which port does the HTTP interface listen on?
+      attr_accessor :http_port
+      # Which port does the protocol buffers interface listen on?
+      attr_accessor :pb_port
+      # A hash of HTTP paths used on this node.
+      attr_accessor :http_paths
+      # A "user:password" string.
+      attr_reader :basic_auth
+      attr_accessor :ssl_options
+      # A Decaying rate of errors.
+      attr_reader :error_rate
+
+      def initialize(client, opts = {})
+        @client = client
+        @ssl = opts[:ssl]
+        @ssl_options = opts[:ssl_options]
+        @host = opts[:host] || "127.0.0.1"
+        @http_port = opts[:http_port] || opts[:port] || 8098
+        @pb_port = opts[:pb_port] || 8087
+        @http_paths = {
+          :prefix => opts[:prefix] || "/riak/",
+          :mapred => opts[:mapred] || "/mapred",
+          :luwak =>  opts[:luwak]  || "/luwak",
+          :solr =>   opts[:solr]   || "/solr" # Unused?
+        }.merge(opts[:http_paths] || {})
+        self.basic_auth = opts[:basic_auth]
+
+        @error_rate = Decaying.new
+      end
+
+      def ==(o)
+        o.kind_of? Node and
+          @host == o.host and
+          @http_port == o.http_port and
+          @pb_port == o.pb_port
+      end
+
+      # Sets the HTTP Basic Authentication credentials.
+      # @param [String] value an auth string in the form "user:password"
+      def basic_auth=(value)
+        case value
+        when nil
+          @basic_auth = nil
+        when String
+          raise ArgumentError, t("invalid_basic_auth") unless value.to_s.split(':').length === 2
+          @basic_auth = value
+        end
+      end
+
+      # Can this node be used for HTTP requests?
+      def http?
+        # TODO: Need to sort out capabilities
+        true
+      end
+
+      # Can this node be used for protocol buffers requests?
+      def protobuffs?
+        # TODO: Need to sort out capabilities
+        true
+      end
+
+      # Enables or disables SSL on this node to be utilized by the HTTP
+      # Backends
+      def ssl=(value)
+        @ssl_options = Hash === value ? value : {}
+        value ? ssl_enable : ssl_disable
+      end
+
+      # Checks if SSL is enabled for HTTP
+      def ssl_enabled?
+        @client.protocol == 'https' || @ssl_options.present?
+      end
+
+      def ssl_enable
+        @client.protocol = 'https'
+        @ssl_options[:pem] = File.read(@ssl_options[:pem_file]) if @ssl_options[:pem_file]
+        @ssl_options[:verify_mode] ||= "peer" if @ssl_options.stringify_keys.any? {|k,v| %w[pem ca_file ca_path].include?(k)}
+        @ssl_options[:verify_mode] ||= "none"
+        raise ArgumentError.new(t('invalid_ssl_verify_mode', :invalid => @ssl_options[:verify_mode])) unless %w[none peer].include?(@ssl_options[:verify_mode])
+
+        @ssl_options
+      end
+
+      def ssl_disable
+        @client.protocol = 'http'
+        @ssl_options  = nil
+      end
+
+      def inspect
+        "<#Node #{@host}:#{@http_port}:#{@pb_port}>"
+      end
+    end
+  end
+end