riak-client 1.2.0 → 1.4.0

data/lib/riak/client/beefcake_protobuffs_backend.rb

@@ -77,11 +77,41 @@ module Riak
         decode_response
       end
 
+      def get_counter(bucket, key, options={})
+        bucket = bucket.name if bucket.is_a? Bucket 
+
+        options = normalize_quorums(options)
+        options[:bucket] = bucket
+        options[:key] = key
+        
+        request = RpbCounterGetReq.new options
+        write_protobuff :CounterGetReq, request
+        
+        decode_response
+      end
+
+      def post_counter(bucket, key, amount, options={})
+        bucket = bucket.name if bucket.is_a? Bucket
+
+        options = normalize_quorums(options)
+        options[:bucket] = bucket
+        options[:key] = key
+        # TODO: raise if ammount doesn't fit in sint64
+        options[:amount] = amount
+
+        request = RpbCounterUpdateReq.new options
+        write_protobuff :CounterUpdateReq, request
+
+        decode_response
+      end
+
       def get_bucket_props(bucket)
         bucket = bucket.name if Bucket === bucket
         req = RpbGetBucketReq.new(:bucket => maybe_encode(bucket))
         write_protobuff(:GetBucketReq, req)
-        decode_response
+        resp = normalize_quorums decode_response
+        normalized = normalize_hooks resp
+        normalized.stringify_keys
       end
 
       def set_bucket_props(bucket, props)
@@ -92,9 +122,16 @@ module Riak
         decode_response
       end
 
-      def list_keys(bucket, &block)
+      def reset_bucket_props(bucket)
+        bucket = bucket.name if Bucket === bucket
+        req = RpbResetBucketReq.new(:bucket => maybe_encode(bucket))
+        write_protobuff(:ResetBucketReq)
+        decode_response
+      end
+
+      def list_keys(bucket, options={}, &block)
         bucket = bucket.name if Bucket === bucket
-        req = RpbListKeysReq.new(:bucket => maybe_encode(bucket))
+        req = RpbListKeysReq.new(options.merge(:bucket => maybe_encode(bucket)))
         write_protobuff(:ListKeysReq, req)
         keys = []
         while msg = decode_response
@@ -108,6 +145,21 @@ module Riak
         block_given? || keys
       end
 
+      # override the simple list_buckets
+      def list_buckets(options={}, &blk)
+        if block_given? 
+          return streaming_list_buckets options, &blk
+        end
+        
+        raise t("streaming_bucket_list_without_block") if options[:stream]
+        
+        request = RpbListBucketsReq.new options
+
+        write_protobuff :ListBucketsReq, request
+
+        decode_response
+      end
+
       def mapred(mr, &block)
         raise MapReduceError.new(t("empty_map_reduce_query")) if mr.query.empty? && !mapred_phaseless?
         req = RpbMapRedReq.new(:request => mr.to_json, :content_type => "application/json")
@@ -124,7 +176,7 @@ module Riak
         block_given? || results.report
       end
 
-      def get_index(bucket, index, query)
+      def get_index(bucket, index, query, query_options={}, &block)
         return super unless pb_indexes?
         bucket = bucket.name if Bucket === bucket
         if Range === query
@@ -139,9 +191,14 @@ module Riak
             :key => query.to_s
           }
         end
-        req = RpbIndexReq.new(options.merge(:bucket => bucket, :index => index))
+
+        options.merge!(:bucket => bucket, :index => index)
+        options.merge!(query_options)
+        options[:stream] = block_given?
+
+        req = RpbIndexReq.new(options)
         write_protobuff(:IndexReq, req)
-        decode_response
+        decode_index_response(&block)
       end
 
       def search(index, query, options={})
@@ -166,12 +223,22 @@ module Riak
         msglen, msgcode = header.unpack("NC")
         if msglen == 1
           case MESSAGE_CODES[msgcode]
-          when :PingResp, :SetClientIdResp, :PutResp, :DelResp, :SetBucketResp
+          when :PingResp, 
+               :SetClientIdResp, 
+               :PutResp, 
+               :DelResp, 
+               :SetBucketResp, 
+               :ResetBucketResp
             true
-          when :ListBucketsResp, :ListKeysResp
+          when :ListBucketsResp, 
+               :ListKeysResp, 
+               :IndexResp
             []
           when :GetResp
             raise Riak::ProtobuffsFailedRequest.new(:not_found, t('not_found'))
+          when :CounterGetResp,
+               :CounterUpdateResp
+            0
           else
             false
           end
@@ -200,17 +267,25 @@ module Riak
             RpbListKeysResp.decode(message)
           when :GetBucketResp
             res = RpbGetBucketResp.decode(message)
-            {'n_val' => res.props.n_val, 'allow_mult' => res.props.allow_mult}
+            res.props.to_hash.stringify_keys
           when :MapRedResp
             RpbMapRedResp.decode(message)
           when :IndexResp
             res = RpbIndexResp.decode(message)
-            res.keys
+            IndexCollection.new_from_protobuf res
           when :SearchQueryResp
             res = RpbSearchQueryResp.decode(message)
             { 'docs' => res.docs.map {|d| decode_doc(d) },
               'max_score' => res.max_score,
               'num_found' => res.num_found }
+          when :CSBucketResp
+            res = RpbCSBucketResp.decode message
+          when :CounterUpdateResp
+            res = RpbCounterUpdateResp.decode message
+            res.value || nil
+          when :CounterGetResp
+            res = RpbCounterGetResp.decode message
+            res.value || 0
           end
         end
       rescue SystemCallError, SocketError => e
@@ -218,6 +293,46 @@ module Riak
         raise
       end
 
+      def streaming_list_buckets(options = {})
+        request = RpbListBucketsReq.new(options.merge(stream: true))
+        write_protobuff :ListBucketsReq, request
+        loop do
+          header = socket.read 5
+          raise SocketError, "Unexpected EOF on PBC socket" if header.nil?
+          len, code = header.unpack 'NC'
+          if MESSAGE_CODES[code] != :ListBucketsResp
+            raise SocketError, "Unexpected non-ListBucketsResp during streaming list buckets"
+          end
+
+          message = socket.read(len - 1)
+          section = RpbListBucketsResp.decode message
+          yield section.buckets
+
+          return if section.done
+        end
+      end
+
+      def decode_index_response
+        loop do
+          header = socket.read(5)
+          raise SocketError, "Unexpected EOF on PBC socket" if header.nil?
+          msglen, msgcode = header.unpack("NC")
+          code = MESSAGE_CODES[msgcode]
+          raise SocketError, "Expected IndexResp, got #{code}" unless code == :IndexResp
+
+          message = RpbIndexResp.decode socket.read msglen - 1
+
+          if !block_given?
+            return IndexCollection.new_from_protobuf(message)
+          end
+          
+          content = message.keys || message.results || []
+          yield content
+          
+          return if message.done
+        end
+      end
+
       def decode_doc(doc)
         Hash[doc.properties.map {|p| [ force_utf8(p.key), force_utf8(p.value) ] }]
       end
@@ -226,6 +341,24 @@ module Riak
         # Search returns strings that should always be valid UTF-8
         ObjectMethods::ENCODING ? str.force_encoding('UTF-8') : str
       end
+
+      def normalize_hooks(message)
+        message.dup.tap do |o|
+          %w{chash_keyfun linkfun}.each do |k|
+            o[k] = {'mod' => message[k].module, 'fun' => message[k].function}
+          end
+          %w{precommit postcommit}.each do |k|
+            orig = message[k]
+            o[k] = orig.map do |hook|
+              if hook.modfun
+                {'mod' => hook.modfun.module, 'fun' => hook.modfun.function}
+              else
+                hook.name
+              end
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/riak/client/feature_detection.rb

@@ -14,7 +14,8 @@ module Riak
         1 => Gem::Version.new("1.0.0"),
         1.1 => Gem::Version.new("1.1.0"),
         1.2 => Gem::Version.new("1.2.0"),
-        1.3 => Gem::Version.new("1.3.0")
+        1.3 => Gem::Version.new("1.3.0"),
+        1.4 => Gem::Version.new("1.4.0")
       }.freeze
 
       # @return [String] the version of the Riak node
@@ -78,6 +79,30 @@ module Riak
         at_least? VERSION[1.3]
       end
 
+      # @return [true,false] whether secondary indexes support
+      #   pagination
+      def index_pagination?
+        at_least? VERSION[1.4]
+      end
+
+      # @return [true,false] whether secondary indexes support
+      #   return_terms
+      def index_return_terms?
+        at_least? VERSION[1.4]
+      end
+
+      # @return [true,false] whether secondary indexes support
+      #   streaming
+      def index_streaming?
+        at_least? VERSION[1.4]
+      end
+
+      # @return [true,false] whether timeouts are accepted for
+      #   object CRUD, key listing, and bucket listing
+      def key_object_bucket_timeouts?
+        at_least? VERSION[1.4]
+      end
+
       protected
       # @return [true,false] whether the server version is the same or
       #  newer than the requested version

data/lib/riak/client/http_backend.rb

@@ -10,6 +10,7 @@ require 'riak/client/http_backend/transport_methods'
 require 'riak/client/http_backend/object_methods'
 require 'riak/client/http_backend/configuration'
 require 'riak/client/http_backend/key_streamer'
+require 'riak/client/http_backend/bucket_streamer'
 require 'riak/client/feature_detection'
 
 module Riak
@@ -115,6 +116,38 @@ module Riak
         delete([204, 404], object_path(bucket, key, options), headers)
       end
 
+      # Fetches a counter
+      # @param [Bucket, String] bucket the bucket where the counter exists
+      # @param [String] key the key for the counter
+      # @param [Hash] options unused
+      def get_counter(bucket, key, options={})
+        bucket = bucket.name if bucket.is_a? Bucket
+        response = get([200, 404], counter_path(bucket, key, options))
+        case response[:code]
+        when 200
+          return response[:body].to_i
+        when 404
+          return 0
+        end
+      end
+
+      # Updates a counter
+      # @param [Bucket, String] bucket the bucket where the counter exists
+      # @param [String] key the key for the counter
+      # @param [Integer] amount how much to increment the counter
+      # @param [Hash] options unused
+      def post_counter(bucket, key, amount, options={})
+        bucket = bucket.name if bucket.is_a? Bucket
+        response = post([200, 204], counter_path(bucket, key, options), amount.to_s)
+        case response[:code]
+        when 200
+          return response[:body].to_i
+        when 204
+          return 0 if options[:return_value]
+          return nil
+        end
+      end
+
       # Fetches bucket properties
       # @param [Bucket, String] bucket the bucket properties to fetch
       # @return [Hash] bucket properties
@@ -155,12 +188,14 @@ module Riak
       # @yield [Array<String>] a list of keys from the current
       #        streamed chunk
       # @return [Array<String>] the list of keys, if no block was given
-      def list_keys(bucket, &block)
+      def list_keys(bucket, options={}, &block)
         bucket = bucket.name if Bucket === bucket
         if block_given?
-          get(200, key_list_path(bucket, :keys => 'stream'), {}, &KeyStreamer.new(block))
+          stream_opts = options.merge keys: 'stream'
+          get(200, key_list_path(bucket, stream_opts), {}, &KeyStreamer.new(block))
         else
-          response = get(200, key_list_path(bucket))
+          list_opts = options.merge keys: true
+          response = get(200, key_list_path(bucket, list_opts))
           obj = JSON.parse(response[:body])
           obj && obj['keys'].map {|k| unescape(k) }
         end
@@ -168,7 +203,12 @@ module Riak
 
       # Lists known buckets
       # @return [Array<String>] the list of buckets
-      def list_buckets
+      def list_buckets(&block)
+        if block_given?
+          get(200, bucket_list_path(stream: true), &BucketStreamer.new(block))
+          return
+        end
+
         response = get(200, bucket_list_path)
         JSON.parse(response[:body])['buckets']
       end
@@ -234,19 +274,28 @@ module Riak
       # @param [String, Integer, Range] query the equality query or
       #   range query to perform
       # @return [Array<String>] a list of keys matching the query
-      def get_index(bucket, index, query)
+      def get_index(bucket, index, query, options={})
         bucket = bucket.name if Bucket === bucket
         path = case query
                when Range
                  raise ArgumentError, t('invalid_index_query', :value => query.inspect) unless String === query.begin || Integer === query.end
-                 index_range_path(bucket, index, query.begin, query.end)
+                 index_range_path(bucket, index, query.begin, query.end, options)
                when String, Integer
-                 index_eq_path(bucket, index, query)
+                 index_eq_path(bucket, index, query, options)
                else
                  raise ArgumentError, t('invalid_index_query', :value => query.inspect)
                end
-        response = get(200, path)
-        JSON.parse(response[:body])['keys']
+        if block_given?
+          parser = Riak::Util::Multipart::StreamParser.new do |response|
+            result = JSON.parse response[:body]
+
+            yield result['keys'] || result['results'] || []
+          end
+          get(200, path, &parser)
+        else
+          response = get(200, path)
+          Riak::IndexCollection.new_from_json response[:body]
+        end
       end
 
       # (Riak Search) Performs a search query

data/lib/riak/client/http_backend/bucket_streamer.rb

@@ -0,0 +1,15 @@
+require 'riak/client/http_backend/chunked_json_streamer'
+
+module Riak
+   class Client
+     class HTTPBackend
+       # @private
+       class BucketStreamer < ChunkedJsonStreamer
+         def get_values(obj)
+           obj['buckets']
+         end
+       end
+     end
+   end
+ end
+

data/lib/riak/client/http_backend/chunked_json_streamer.rb

@@ -0,0 +1,42 @@
+require 'riak/util/escape'
+require 'riak/json'
+
+module Riak
+  class Client
+    class HTTPBackend
+      class ChunkedJsonStreamer
+        include Util::Escape
+
+         def initialize(block)
+           @buffer = ""
+           @block = block
+         end
+
+         def accept(chunk)
+           @buffer << chunk
+           consume
+         end
+
+         def to_proc
+           method(:accept).to_proc
+         end
+
+         private
+         def consume
+           while @buffer =~ /\}\{/
+             stream($~.pre_match + '}')
+             @buffer = '{' + $~.post_match
+           end
+         end
+
+         def stream(str)
+           obj = JSON.parse(str) rescue nil
+           if obj && get_values(obj)
+             @block.call get_values(obj).map(&method(:maybe_unescape))
+           end
+         end
+       end
+     end
+   end
+ end
+

data/lib/riak/client/http_backend/configuration.rb

@@ -29,7 +29,10 @@ module Riak
 
         # @return [URI] a URL path for the "buckets list" resource
         def bucket_list_path(options={})
-          if new_scheme?
+          if options[:stream] && new_scheme?
+            options.delete :stream
+            path(riak_kv_wm_buckets, options.merge(buckets: 'stream'))
+          elsif new_scheme?
             path(riak_kv_wm_buckets, options.merge(:buckets => true))
           else
             path(riak_kv_wm_raw, options.merge(:buckets => true))
@@ -71,6 +74,19 @@ module Riak
           end
         end
 
+        # @return [URI] a URL path for the "counter" resource
+        # @param [String] bucket the bucket of the counter
+        # @param [String] key the key of the counter
+        def counter_path(bucket, key, options={})
+          path([
+                riak_kv_wm_buckets,
+                escape(bucket),
+                "counters",
+                escape(key),
+                options
+                ].compact)
+        end
+
         # @return [URI] a URL path for the "link-walking" resource
         # @param [String] bucket the bucket of the origin object
         # @param [String] key the key of the origin object