riak-client 1.2.0 → 1.4.0

data/lib/riak/client/http_backend/key_streamer.rb

@@ -1,40 +1,12 @@
-require 'riak/util/escape'
-require 'riak/json'
+require 'riak/client/http_backend/chunked_json_streamer'
 
 module Riak
    class Client
      class HTTPBackend
        # @private
-       class KeyStreamer
-         include Util::Escape
-
-         def initialize(block)
-           @buffer = ""
-           @block = block
-         end
-
-         def accept(chunk)
-           @buffer << chunk
-           consume
-         end
-
-         def to_proc
-           method(:accept).to_proc
-         end
-
-         private
-         def consume
-           while @buffer =~ /\}\{/
-             stream($~.pre_match + '}')
-             @buffer = '{' + $~.post_match
-           end
-         end
-
-         def stream(str)
-           obj = JSON.parse(str) rescue nil
-           if obj && obj['keys']
-             @block.call obj['keys'].map(&method(:maybe_unescape))
-           end
+       class KeyStreamer < ChunkedJsonStreamer
+         def get_values(obj)
+           obj['keys']
          end
        end
      end

data/lib/riak/client/protobuffs_backend.rb

@@ -4,6 +4,7 @@ require 'base64'
 require 'digest/sha1'
 require 'riak/util/translation'
 require 'riak/client/feature_detection'
+require 'riak/client/beefcake/message_codes'
 
 module Riak
   class Client
@@ -12,38 +13,7 @@ module Riak
       include Util::Escape
       include FeatureDetection
 
-      # Message Codes Enum
-      MESSAGE_CODES = %W[
-          ErrorResp
-          PingReq
-          PingResp
-          GetClientIdReq
-          GetClientIdResp
-          SetClientIdReq
-          SetClientIdResp
-          GetServerInfoReq
-          GetServerInfoResp
-          GetReq
-          GetResp
-          PutReq
-          PutResp
-          DelReq
-          DelResp
-          ListBucketsReq
-          ListBucketsResp
-          ListKeysReq
-          ListKeysResp
-          GetBucketReq
-          GetBucketResp
-          SetBucketReq
-          SetBucketResp
-          MapRedReq
-          MapRedResp
-          IndexReq
-          IndexResp
-          SearchQueryReq
-          SearchQueryResp
-       ].map {|s| s.intern }.freeze
+      MESSAGE_CODES = BeefcakeMessageCodes
 
       def self.simple(method, code)
         define_method method do
@@ -62,7 +32,6 @@ module Riak
       simple :ping,          :PingReq
       simple :get_client_id, :GetClientIdReq
       simple :server_info,   :GetServerInfoReq
-      simple :list_buckets,  :ListBucketsReq
 
       # Performs a secondary-index query via emulation through MapReduce.
       # @param [String, Bucket] bucket the bucket to query
@@ -144,6 +113,9 @@ module Riak
         unless quorum_controls?
           [:notfound_ok, :basic_quorum, :pr, :pw].each {|k| options.delete k }
         end
+        unless key_object_bucket_timeouts?
+          options.delete :timeout
+        end
         unless pb_head?
           [:head, :return_head].each {|k| options.delete k }
         end
@@ -160,7 +132,9 @@ module Riak
       def normalize_quorums(options={})
         options.dup.tap do |o|
           [:r, :pr, :w, :pw, :dw, :rw].each do |k|
-            o[k] = normalize_quorum_value(o[k]) if o[k]
+            next o[k] = normalize_quorum_value(o[k]) if o[k]
+            s = k.to_s
+            o[k] = o[s] = denormalize_quorum_value(o[s]) if o[s]
           end
         end
       end
@@ -168,6 +142,10 @@ module Riak
       def normalize_quorum_value(q)
         QUORUMS[q.to_s] || q.to_i
       end
+
+      def denormalize_quorum_value(q)
+        QUORUMS.invert[q] || q.to_i
+      end
     end
   end
 end

data/lib/riak/counter.rb

@@ -0,0 +1,101 @@
+require 'riak/failed_request'
+
+module Riak
+
+  # A distributed counter that supports incrementing by positive and negative
+  # integers.
+  class Counter
+    include Util::Translation
+    attr_accessor :bucket
+    attr_accessor :key
+    attr_accessor :client
+
+    # Create a Riak counter.
+    # @param [Bucket] bucket the {Riak::Bucket} for this counter
+    # @param [String] key the name of the counter
+    def initialize(bucket, key)
+      raise ArgumentError, t("bucket_type", bucket: bucket.inspect) unless bucket.is_a? Bucket
+      raise ArgumentError, t("string_type", string: key.inspect) unless key.is_a? String
+      @bucket, @key = bucket, key
+      @client = bucket.client
+
+      validate_bucket
+    end
+
+    # Retrieve the current value of the counter.
+    # @param [Hash] options 
+    # @option options [Fixnum,String] :r ("quorum") read quorum (numeric or
+    # symbolic)
+    def value(options={})
+      backend do |backend|
+        backend.get_counter bucket, key, options
+      end
+    end
+    alias :to_i :value
+
+    # Increment the counter and return its new value.
+    # @param amount [Integer] the amount to increment the counter by.
+    def increment_and_return(amount=1)
+      increment amount, return_value: true
+    end
+
+    # Decrement the counter and return its new value.
+    # @param amount [Integer] the amount to decrement the counter by. Negative 
+    # values increment the counter.
+    def decrement_and_return(amount=1)
+      increment_and_return -amount
+    end
+
+    # Increment the counter.
+    # @param amount [Integer] the amount to increment the counter by
+    # @param [Hash] options 
+    # @option options [Boolean] :return_value whether to return the new counter
+    # value. Default false.
+    # @option options [Fixnum,String] :r ("quorum") read quorum (numeric or
+    # symbolic)
+    # @option options [Fixnum] :w the "w" parameter (Write quorum)
+    # @option options [Fixnum] :dw the "dw" parameter (Durable-write quorum)
+    def increment(amount=1, options={})
+      validate_amount amount
+
+      backend do |backend|
+        backend.post_counter bucket, key, amount, options
+      end
+    end
+
+    # Decrement the counter.
+    # @param amount [Integer] the amount to decrement the counter by. Negative 
+    # values increment the counter.
+    # @param [Hash] options 
+    # @option options [Boolean] :return_value whether to return the new counter
+    # value. Default false.
+    # @option options [Fixnum,String] :r ("quorum") read quorum (numeric or
+    # symbolic)
+    # @option options [Fixnum] :w the "w" parameter (Write quorum)
+    # @option options [Fixnum] :dw the "dw" parameter (Durable-write quorum)
+    def decrement(amount=1, options={})
+      increment(-amount, options)
+    end
+
+    private
+    def validate_bucket
+      raise ArgumentError, t("counter.bucket_needs_allow_mult") unless bucket.allow_mult
+    end
+
+    def validate_amount(amount)
+      raise ArgumentError, t("counter.increment_by_integer") unless amount.is_a? Integer
+    end
+
+    def backend(&blk)
+      begin
+        return client.backend &blk
+      rescue Riak::FailedRequest => e
+        raise QuorumError.new e if e.message =~ /unsatisfied/
+        raise e
+      end
+    end
+
+    class QuorumError < Riak::FailedRequest
+    end
+  end
+end

data/lib/riak/index_collection.rb

@@ -0,0 +1,71 @@
+module Riak
+
+  # IndexCollection provides extra tools for managing index matches returned by
+  # a Secondary Index query. In Riak 1.4, these queries can be paginaged, and
+  # match keys up with the index values they matched against.
+  class IndexCollection < Array
+
+    # @return [String] The continuation used to retrieve the next page of a
+    # paginated query.
+    attr_accessor :continuation
+
+    # @return [Hash<Integer/String, String>] A hash of index keys (String or
+    # Integer, depending on whether the query was a binary or integer) to
+    # arrays of keys.
+    attr_accessor :with_terms
+
+    def self.new_from_json(json)
+      parsed = JSON.parse json
+      fresh = nil
+      if parsed['keys']
+        fresh = new parsed['keys'] 
+      elsif parsed['results']
+        fresh_terms = load_json_terms(parsed)
+        fresh = new fresh_terms.values.flatten
+        fresh.with_terms = fresh_terms
+      else
+        fresh = new []
+      end
+      fresh.continuation = parsed['continuation']
+
+      fresh
+    end
+
+    def self.new_from_protobuf(message)
+      fresh = nil
+      if message.keys
+        fresh = new message.keys
+      elsif message.results
+        fresh_terms = load_pb_terms(message)
+        fresh = new fresh_terms.values.flatten
+        fresh.with_terms = fresh_terms
+      else
+        fresh = new
+      end
+      fresh.continuation = message.continuation
+
+      fresh
+    end
+
+    private
+    def self.load_json_terms(parsed)
+      fresh_terms = Hash.new{Array.new}
+      parsed['results'].each do |r|
+        k = r.keys.first
+        v = r[k]
+        fresh_terms[k] += [v]
+      end
+
+      fresh_terms
+    end
+
+    def self.load_pb_terms(message)
+      fresh_terms = Hash.new{Array.new}
+      message.results.each do |r|
+        fresh_terms[r.key] += [r.value]
+      end
+
+      fresh_terms
+    end
+  end
+end

data/lib/riak/list_buckets.rb

@@ -0,0 +1,28 @@
+module Riak
+  class ListBuckets
+    def initialize(client, options, block)
+      @client = client
+      @block = block
+      @options = options
+      perform_request
+    end
+
+    def perform_request
+      @client.backend do |be|
+        be.list_buckets @options, &wrapped_block
+      end
+    end
+
+    private
+
+    def wrapped_block
+      proc do |bucket_names|
+        next if bucket_names.nil?
+        bucket_names.each do |bucket_name|
+          bucket = @client.bucket bucket_name
+          @block.call bucket
+        end
+      end
+    end
+  end
+end

data/lib/riak/locale/en.yml

@@ -2,9 +2,13 @@ en:
   riak:
     backwards_clock: "System clock moved backwards, ID generation will fail for %{delay} more milliseconds."
     bucket_link_conversion: "Can't convert a bucket link to a walk spec"
+    bucket_type: "invalid argument %{bucket} is not a Riak::Bucket"
     client_type: "invalid argument %{client} is not a Riak::Client"
     conflict_resolver_invalid: "The given resolver (%{resolver}) did not respond to :call"
     content_type_undefined: "content_type is not defined!"
+    counter:
+      bucket_needs_allow_mult: "Counters require allow_mult to be enabled on their bucket."
+      increment_by_integer: "Counters can only be incremented or decremented by integers."
     deprecated:
       port: "DEPRECATION: Riak::Client#port has been deprecated, use #http_port or #pb_port for the appropriate protocol.\n%{backtrace}"
       search: "DEPRECATION: Riak Search features are included in the main client, you no longer need to require 'riak/search'.\n%{backtrace}"
@@ -18,10 +22,17 @@ en:
     http_failed_request: "Expected %{expected} from Riak but received %{code}. %{body}"
     hostname_invalid: "host must be a valid hostname"
     protocol_invalid: "'%{invalid}' is not a valid protocol, valid values are %{valid}"
+    index:
+        no_next_page: "The returned search did not have a continuation available."
+        pagination_not_available: "The Riak server does not support secondary index pagination."
+        return_terms_not_available: "The Riak server does not support return_terms."
+        streaming_not_available: "The Riak server does not support streaming."
+        include_terms_is_wrong: "include_terms isn't a valid option; return_terms is."
     invalid_basic_auth: "basic auth must be set using 'user:pass'"
     invalid_client_id: "Invalid client ID, must be a string or between 0 and %{max_id}"
     invalid_io_object: "Invalid IO-like object assigned to RObject#data.  It should be assigned to raw_data instead."
     invalid_function_value: "invalid value for function: %{value}"
+    invalid_multiget_thread_count: "Invalid multiget thread count, must be nil or a positive integer."
     invalid_options: "Invalid configuration options given."
     invalid_phase_type: "type must be :map, :reduce, or :link"
     invalid_ssl_verify_mode: "%{invalid} is not a valid :verify_mode option for SSL. Valid options are 'peer' and 'none'."
@@ -48,7 +59,10 @@ en:
     source_and_root_required: "Riak::Node configuration must include :source and :root keys."
     stale_write_prevented: "Stale write prevented by client."
     stored_function_invalid: "function must have :bucket and :key when a hash"
+    streaming_bucket_list_without_block: "Streaming bucket list was requested but no block was given."
     string_type: "invalid_argument %{string} is not a String"
     too_few_arguments: "too few arguments: %{params}"
     walk_spec_invalid_unless_link: "WalkSpec is only valid for a function when the type is :link"
     wrong_argument_count_walk_spec: "wrong number of arguments (one Hash or bucket,tag,keep required)"
+    zero_length_bucket: "bucket name cannot be a String of zero length"
+    zero_length_key: "key cannot be a String of zero length"

data/lib/riak/multiget.rb

@@ -0,0 +1,123 @@
+require 'riak/client'
+require 'riak/bucket'
+require 'riak/cluster'
+
+module Riak
+  # Coordinates a parallel fetch operation for multiple values.
+  class Multiget
+    include Util::Translation
+
+    # @return [Riak::Client] the associated client
+    attr_reader :client
+
+    # @return [Array<Bucket, String>] fetch_list an {Array} of {Bucket} and {String} keys to fetch
+    attr_reader :fetch_list
+
+    # @return [Hash<fetch_list_entry, RObject] result_hash a {Hash} of {Bucket} and {String} key pairs to {RObject} instances
+    attr_accessor :result_hash
+
+    # @return [Boolean] finished if the fetch operation has completed
+    attr_reader :finished
+
+    # @return [Integer] The number of threads to use
+    attr_accessor :thread_count
+
+    # Perform a Riak Multiget operation.
+    # @param [Client] client the {Riak::Client} that will perform the multiget
+    # @param [Array<Bucket, String>] fetch_list an {Array} of {Bucket} and {String} keys to fetch
+    # @return [Hash<fetch_list_entry, RObject] result_hash a {Hash} of {Bucket} and {String} key pairs to {Robject} instances
+    def self.get_all(client, fetch_list)
+      multi = new client, fetch_list
+      multi.fetch
+      multi.results
+    end
+
+    # Create a Riak Multiget operation.
+    # @param [Client] client the {Riak::Client} that will perform the multiget
+    # @param [Array<Bucket, String>] fetch_list an {Array} of {Bucket} and {String} keys to fetch
+    def initialize(client, fetch_list)
+      raise ArgumentError, t('client_type', :client => client.inspect) unless client.is_a? Riak::Client
+      raise ArgumentError, t('array_type', :array => fetch_list.inspect) unless fetch_list.is_a? Array
+
+      validate_fetch_list fetch_list
+      @client, @fetch_list = client, fetch_list.uniq
+      self.result_hash = Hash.new
+      @finished = false
+      self.thread_count = client.multiget_threads
+    end
+
+    # Starts the parallelized fetch operation
+    # @raise [ArgumentError] when a non-positive-Integer count is given
+    def fetch
+      queue = fetch_list.dup
+      queue_mutex = Mutex.new
+      result_mutex = Mutex.new
+
+      unless thread_count.is_a?(Integer) && thread_count > 0
+        raise ArgumentError, t("invalid_multiget_thread_count")
+      end
+
+      @threads = 1.upto(thread_count).map do |_node|
+        Thread.new do
+          loop do
+            pair = queue_mutex.synchronize do
+              queue.shift
+            end
+
+            break if pair.nil?
+
+            found = attempt_fetch(*pair)
+            result_mutex.synchronize do
+              result_hash[pair] = found
+            end
+          end
+        end
+      end
+    end
+
+    def results
+      wait_for_finish
+      result_hash
+    end
+
+    def finished?
+      set_finished_for_thread_liveness
+      finished
+    end
+
+    def wait_for_finish
+      return if finished?
+      @threads.each {|t| t.join }
+      @finished = true
+    end
+
+    private
+
+    def attempt_fetch(bucket, key)
+      bucket[key]
+    rescue Riak::FailedRequest => e
+      raise e unless e.not_found?
+      nil
+    end
+
+    def set_finished_for_thread_liveness
+      return if @finished # already done
+
+      all_dead = @threads.none? {|t| t.alive? }
+      return unless all_dead # still working
+
+      @finished = true
+      return
+    end
+
+    def validate_fetch_list(fetch_list)
+      return unless erroneous = fetch_list.detect do |e|
+        bucket, key = e
+        next true unless bucket.is_a? Bucket
+        next true unless key.is_a? String
+      end
+
+      raise ArgumentError, t('fetch_list_type', :problem => erroneous)
+    end
+  end
+end