riak-client 1.0.5 → 1.1.0

data/lib/riak/client/feature_detection.rb

@@ -0,0 +1,88 @@
+module Riak
+  class Client
+    # Methods that can be used to determine whether certain features
+    # are supported by the Riak node to which the client backend is
+    # connected.
+    #
+    # Backends must implement the "get_server_version" method,
+    # returning a string representing the Riak node's version. This is
+    # implemented on HTTP using the stats resource, and on Protocol
+    # Buffers using the RpbGetServerInfoReq message.
+    module FeatureDetection
+      # Constants representing Riak versions
+      VERSION = {
+        1 => Gem::Version.new("1.0.0"),
+        1.1 => Gem::Version.new("1.1.0"),
+        1.2 => Gem::Version.new("1.2.0")
+      }.freeze
+
+      # @return [String] the version of the Riak node
+      # @abstract
+      def get_server_version
+        raise NotImplementedError
+      end
+
+      # @return [Gem::Version] the version of the Riak node to which
+      #   this backend is connected
+      def server_version
+        @server_version ||= Gem::Version.new(get_server_version)
+      end
+
+      # @return [true,false] whether MapReduce requests can be submitted without
+      #   phases.
+      def mapred_phaseless?
+        at_least? VERSION[1.1]
+      end
+
+      # @return [true,false] whether secondary index queries are
+      #  supported over Protocol Buffers
+      def pb_indexes?
+        at_least? VERSION[1.2]
+      end
+
+      # @return [true,false] whether search queries are supported over
+      #   Protocol Buffers
+      def pb_search?
+        at_least? VERSION[1.2]
+      end
+
+      # @return [true,false] whether conditional fetch/store semantics
+      #   are supported over Protocol Buffers
+      def pb_conditionals?
+        at_least? VERSION[1]
+      end
+
+      # @return [true,false] whether additional quorums and FSM
+      #   controls are available, e.g. primary quorums, basic_quorum,
+      #   notfound_ok
+      def quorum_controls?
+        at_least? VERSION[1]
+      end
+
+      # @return [true,false] whether "not found" responses might
+      #   include vclocks
+      def tombstone_vclocks?
+        at_least? VERSION[1]
+      end
+
+      # @return [true,false] whether partial-fetches (vclock and
+      #   metadata only) are supported over Protocol Buffers
+      def pb_head?
+        at_least? VERSION[1]
+      end
+
+      protected
+      # @return [true,false] whether the server version is the same or
+      #  newer than the requested version
+      def at_least?(version)
+        server_version >= version
+      end
+
+      # Backends should call this when their connection is interrupted
+      # or reset so as to facilitate rolling upgrades
+      def reset_server_version
+        @server_version = nil
+      end
+    end
+  end
+end

data/lib/riak/client/http_backend.rb

@@ -10,6 +10,7 @@ require 'riak/client/http_backend/transport_methods'
 require 'riak/client/http_backend/object_methods'
 require 'riak/client/http_backend/configuration'
 require 'riak/client/http_backend/key_streamer'
+require 'riak/client/feature_detection'
 
 module Riak
   class Client
@@ -21,6 +22,7 @@ module Riak
     class HTTPBackend
       include Util::Escape
       include Util::Translation
+      include FeatureDetection
 
       include TransportMethods
       include ObjectMethods
@@ -162,6 +164,7 @@ module Riak
       # @return [Array<Object>] the list of results, if no block was
       #        given
       def mapred(mr)
+        raise MapReduceError.new(t("empty_map_reduce_query")) if mr.query.empty? && !mapred_phaseless?
         if block_given?
           parser = Riak::Util::Multipart::StreamParser.new do |response|
             result = JSON.parse(response[:body])
@@ -170,12 +173,13 @@ module Riak
           post(200, mapred_path({:chunked => true}), mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"}, &parser)
           nil
         else
-          response = post(200, mapred_path, mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"})
-          begin
-            JSON.parse(response[:body])
-          rescue
-            response
+          results = MapReduce::Results.new(mr)
+          parser = Riak::Util::Multipart::StreamParser.new do |response|
+            result = JSON.parse(response[:body])
+            results.add result['phase'], result['data']
           end
+          post(200, mapred_path({:chunked => true}), mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"}, &parser)
+          results.report
         end
       end
 
@@ -238,7 +242,7 @@ module Riak
       def search(index, query, options={})
         response = get(200, solr_select_path(index, query, options.stringify_keys))
         if response[:headers]['content-type'].include?("application/json")
-          JSON.parse(response[:body])
+          normalize_search_response JSON.parse(response[:body])
         else
           response[:body]
         end
@@ -315,6 +319,23 @@ module Riak
           response[:headers]["location"].first.split("/").last
         end
       end
+
+      private
+      def normalize_search_response(json)
+        {}.tap do |result|
+          if json['response']
+            result['num_found'] = json['response']['numFound']
+            result['max_score'] = json['response']['maxScore'].to_f
+            result['docs'] = json['response']['docs'].map do |d|
+              if d['fields']
+                d['fields'].merge('id' => d['id'])
+              else
+                d
+              end
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/riak/client/http_backend/configuration.rb

@@ -157,6 +157,19 @@ module Riak
           !riak_kv_wm_buckets.nil?
         end
 
+        def get_server_version
+          begin
+            # Attempt to grab the server version from the stats resource
+            stats = send(:stats)
+            stats['riak_kv_version']
+          rescue FailedRequest
+            # If stats is disabled, we can't assume the Riak version
+            # is >= 1.1. However, we can assume the new URL scheme is
+            # at least version 1.0.
+            new_scheme? ? "1.0.0" : "0.14.0"
+          end
+        end
+
         def riak_kv_wm_buckets
           server_config[:riak_kv_wm_buckets]
         end

data/lib/riak/client/http_backend/object_methods.rb

@@ -52,28 +52,15 @@ module Riak
         # @param [Hash] response a response from {Riak::Client::HTTPBackend}
         def load_object(robject, response)
           extract_header(robject, response, "location", :key) {|v| URI.unescape(v.match(%r{.*/(.*?)(\?.*)?$})[1]) }
-          extract_header(robject, response, "content-type", :content_type)
           extract_header(robject, response, "x-riak-vclock", :vclock)
-          extract_header(robject, response, "link", :links) {|v| Set.new(Link.parse(v)) }
-          extract_header(robject, response, "etag", :etag)
-          extract_header(robject, response, "last-modified", :last_modified) {|v| Time.httpdate(v) }
-          robject.meta = response[:headers].inject({}) do |h,(k,v)|
-            if k =~ /x-riak-meta-(.*)/i
-              h[$1] = v
-            end
-            h
-          end
-          robject.indexes = response[:headers].inject(Hash.new {|h,k| h[k] = Set.new }) do |h,(k,v)|
-            if k =~ /x-riak-index-((?:.*)_(?:int|bin))$/i
-              key = $1
-              h[key].merge Array(v).map {|vals| vals.split(/,\s*/).map {|i| key =~ /int$/ ? i.to_i : i } }.flatten
-            end
-            h
+          case response[:code] && response[:code].to_i
+          when 304
+            # Resulted from a reload, don't modify anything
+          when 300
+            robject.siblings = extract_siblings(robject, response[:body], response[:headers]['content-type'].first)
+          else
+            robject.siblings = [ load_content(response, RContent.new(robject)) ]
           end
-          robject.conflict = (response[:code] && response[:code].to_i == 300 && robject.content_type =~ /multipart\/mixed/)
-          robject.siblings = robject.conflict? ? extract_siblings(robject, response[:body]) : nil
-          robject.raw_data = response[:body] if response[:body].present? && !robject.conflict?
-
           robject.conflict? ? robject.attempt_conflict_resolution : robject
         end
 
@@ -92,13 +79,34 @@ module Riak
           end
         end
 
-        def extract_siblings(robject, data)
-          Util::Multipart.parse(data, Util::Multipart.extract_boundary(robject.content_type)).map do |part|
-            RObject.new(robject.bucket, robject.key) do |sibling|
-              load_object(sibling, part)
-              sibling.vclock = robject.vclock
+        def extract_siblings(robject, data, content_type)
+          Util::Multipart.parse(data, Util::Multipart.extract_boundary(content_type)).map do |part|
+            RContent.new(robject) do |sibling|
+              load_content(part, sibling)
+            end
+          end
+        end
+
+        def load_content(response, rcontent)
+          extract_header(rcontent, response, "link", :links) {|v| Set.new(Link.parse(v)) }
+          extract_header(rcontent, response, "etag", :etag)
+          extract_header(rcontent, response, "last-modified", :last_modified) {|v| Time.httpdate(v) }
+          extract_header(rcontent, response, "content-type", :content_type)
+          rcontent.meta = response[:headers].inject({}) do |h,(k,v)|
+            if k =~ /x-riak-meta-(.*)/i
+              h[$1] = v
+            end
+            h
+          end
+          rcontent.indexes = response[:headers].inject(Hash.new {|h,k| h[k] = Set.new }) do |h,(k,v)|
+            if k =~ /x-riak-index-((?:.*)_(?:int|bin))$/i
+              key = $1
+              h[key].merge Array(v).map {|vals| vals.split(/,\s*/).map {|i| key =~ /int$/ ? i.to_i : i } }.flatten
             end
+            h
           end
+          rcontent.raw_data = response[:body] if response[:body].present?
+          rcontent
         end
       end
     end

data/lib/riak/client/node.rb

@@ -79,7 +79,12 @@ module Riak
       # Enables or disables SSL on this node to be utilized by the HTTP
       # Backends
       def ssl=(value)
-        @ssl_options = Hash === value ? value : {}
+        case value
+        when TrueClass
+          @ssl_options ||= {}
+        when Hash
+          (@ssl_options ||= {}).merge!(value)
+        end
         value ? ssl_enable : ssl_disable
       end
 
@@ -89,7 +94,7 @@ module Riak
       end
 
       def inspect
-        "<#Node #{@host}:#{@http_port}:#{@pb_port}>"
+        "#<Node #{@host}:#{@http_port}:#{@pb_port}>"
       end
 
       protected

data/lib/riak/client/protobuffs_backend.rb

@@ -3,12 +3,14 @@ require 'socket'
 require 'base64'
 require 'digest/sha1'
 require 'riak/util/translation'
+require 'riak/client/feature_detection'
 
 module Riak
   class Client
     class ProtobuffsBackend
       include Util::Translation
       include Util::Escape
+      include FeatureDetection
 
       # Message Codes Enum
       MESSAGE_CODES = %W[
@@ -37,6 +39,10 @@ module Riak
           SetBucketResp
           MapRedReq
           MapRedResp
+          IndexReq
+          IndexResp
+          SearchQueryReq
+          SearchQueryResp
        ].map {|s| s.intern }.freeze
 
       def self.simple(method, code)
@@ -65,9 +71,32 @@ module Riak
       #   range query to perform
       # @return [Array<String>] a list of keys matching the query
       def get_index(bucket, index, query)
-        mapred(Riak::MapReduce.new(client).
-               index(bucket, index, query).
-               reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)).map {|p| p.last }
+        mr = Riak::MapReduce.new(client).index(bucket, index, query)
+        unless mapred_phaseless?
+          mr.reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)
+        end
+        mapred(mr).map {|p| p.last }
+      end
+
+      # Performs search query via emulation through MapReduce. This
+      # has more limited capabilites than native queries. Essentially,
+      # only the 'id' field of matched documents will ever be
+      # returned, the 'fl' and other options have no effect.
+      # @param [String] index the index to query
+      # @param [String] query the Lucene-style search query
+      # @param [Hash] options ignored in MapReduce emulation
+      # @return [Hash] the search results
+      def search(index, query, options={})
+        mr = Riak::MapReduce.new(client).search(index || 'search', query)
+        unless mapred_phaseless?
+          mr.reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)
+        end
+        docs = mapred(mr).map {|d| {'id' => d[1] } }
+        # Since we don't get this information back from the MapReduce,
+        # we have to fake the max_score and num_found.
+        { 'docs' => docs,
+          'num_found' => docs.size,
+          'max_score' => 0.0 }
       end
 
       # Gracefully shuts down this connection.
@@ -76,6 +105,10 @@ module Riak
       end
 
       private
+      def get_server_version
+        server_info[:server_version]
+      end
+
       # Implemented by subclasses
       def decode_response
         raise NotImplementedError
@@ -94,6 +127,7 @@ module Riak
       end
 
       def reset_socket
+        reset_server_version
         @socket.close if @socket && !@socket.closed?
         @socket = nil
       end
@@ -106,6 +140,23 @@ module Riak
         "default" => UINTMAX - 4
       }.freeze
 
+      def prune_unsupported_options(req,options={})
+        unless quorum_controls?
+          [:notfound_ok, :basic_quorum, :pr, :pw].each {|k| options.delete k }
+        end
+        unless pb_head?
+          [:head, :return_head].each {|k| options.delete k }
+        end
+        unless tombstone_vclocks?
+          options.delete :deletedvclock
+          options.delete :vclock if req == :DelReq
+        end
+        unless pb_conditionals?
+          [:if_not_modified, :if_none_match, :if_modified].each {|k| options.delete k }
+        end
+        options
+      end
+
       def normalize_quorums(options={})
         options.dup.tap do |o|
           [:r, :pr, :w, :pw, :dw, :rw].each do |k|

data/lib/riak/client/search.rb

@@ -20,8 +20,8 @@ module Riak
     def search(*args)
       options = args.extract_options!
       index, query = args[-2], args[-1]  # Allows nil index, while keeping it as firstargument
-      http do |h|
-        h.search(index, query, options)
+      backend do |b|
+        b.search(index, query, options)
       end
     end
     alias :select :search

data/lib/riak/conflict.rb

@@ -0,0 +1,13 @@
+require 'riak/util/translation'
+
+module Riak
+  # Raised when an object that is in conflict (i.e. has siblings) is
+  # stored or manipulated as if it had a single value.
+  class Conflict < StandardError
+    include Util::Translation
+
+    def initialize(robject)
+      super t('object_in_conflict', :robject => robject.inspect)
+    end
+  end
+end

data/lib/riak/locale/en.yml

@@ -36,8 +36,10 @@ en:
     module_function_pair_required: "function must have two elements when an array"
     not_found: "The requested object was not found."
     no_pipes: "Could not find or open pipes for Riak console in %{path}."
+    object_in_conflict: "The object is in conflict (has siblings) and cannot be treated singly or saved: %{robject}"
     port_invalid: "port must be an integer between 0 and 65535"
     protobuffs_failed_request: "Expected success from Riak but received %{code}. %{body}"
+    protobuffs_configuration: "The %{backend} Protobuffs backend cannot be used. Please check its requirements."
     request_body_type: "Request body must be a String or respond to :read."
     search_unsupported: "Riak server does not support search."
     search_docs_require_id: "Search index documents must include the 'id' field."

data/lib/riak/map_reduce.rb

@@ -9,6 +9,7 @@ require 'riak/failed_request'
 require 'riak/map_reduce_error'
 require 'riak/map_reduce/phase'
 require 'riak/map_reduce/filter_builder'
+require 'riak/map_reduce/results'
 
 module Riak
   # Class for invoking map-reduce jobs using the HTTP interface.
@@ -213,7 +214,6 @@ module Riak
     #   @yieldparam [Array] data a list of results from the phase
     #   @return [nil] nothing
     def run(&block)
-      raise MapReduceError.new(t("empty_map_reduce_query")) if @query.empty?
       @client.mapred(self, &block)
     rescue FailedRequest => fr
       if fr.server_error? && fr.is_json?

data/lib/riak/map_reduce/filter_builder.rb

@@ -46,8 +46,8 @@ module Riak
       #    FilterBuilder.new do
       #      string_to_int
       #      AND do
-      #        seq { greater_than_eq 50 }
-      #        seq { neq 100 }
+      #        greater_than_eq 50
+      #        neq 100
       #      end
       #    end
       LOGICAL_OPERATIONS = %w{and or not}