riak-client 1.2.0 → 1.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 31c844652f0920188a681f67ae0421582deb39e6
+  data.tar.gz: a61519055a317cc9a252bff17903bb08126f5abe
+SHA512:
+  metadata.gz: e76e0fe22bedb857baf87de6d99da8529eaccd79fc0fe39b78fe0f850b8d244193e24aa83481793ae5e1b80d09b578f1213efff2e360c6f51cac622e8d921bb1
+  data.tar.gz: cff6a9f0e542399e9249eb9ddf0258137c7f52dc33ff5ead84c16781a1c34d0fe49099256c95b134203b51a26d0f46a698596b8d9c958c37226fd0703dfe8c3d

data/.gitignore

@@ -37,3 +37,4 @@ Gemfile.lock
 **/.riaktest
 vendor/erlang
 vendor/riak
+.ruby-version

data/Gemfile

@@ -12,6 +12,7 @@ end
 
 platforms :mri do
   gem 'yajl-ruby'
+  gem 'debugger'
 end
 
 platforms :jruby do
@@ -21,10 +22,3 @@ end
 platforms :jruby, :rbx do
   gem 'json'
 end
-# platforms :mri_18, :jruby do
-#   gem 'ruby-debug'
-# end
-
-# platforms :mri_19 do
-#   gem 'ruby-debug19'
-# end

data/README.markdown

@@ -136,6 +136,72 @@ client['users'].enable_index!
 client['users'].disable_index!
 ```
 
+## Secondary Index Examples
+
+Riak supports secondary indexes. Secondary indexing, or "2i," gives you the 
+ability to tag objects with multiple queryable values at write time, and then
+query them later.
+
+* [Using Secondary Indexes](http://docs.basho.com/riak/latest/dev/using/2i/)
+* [Secondary Index implementation notes](http://docs.basho.com/riak/latest/dev/advanced/2i/)
+
+### Tagging Objects
+
+Objects are tagged with a hash kept behind the `indexes` method. Secondary index
+storage logic is in `lib/riak/rcontent.rb`.
+
+```ruby
+object = bucket.get_or_new 'cobb.salad'
+
+# Indexes end with the "_bin" suffix to indicate they're binary or string 
+# indexes. They can have multiple values.
+object.indexes[:ingredients_bin] = %w{lettuce tomato bacon egg chives}
+
+# Indexes ending with the "_int" suffix are indexed as integers. They can
+# have multiple values too.
+object.indexes[:calories_int] = [220]
+
+# You must re-save the object to store indexes.
+object.store
+```
+
+### Finding Objects
+
+Secondary index queries return a list of keys exactly matching a scalar or 
+within a range.
+
+```ruby
+# The Bucket#get_index method allows querying by scalar...
+bucket.get_index 'calories_int', 220 # => ['cobb.salad']
+
+# or range.
+bucket.get_index 'calories_int', 100..300 # => ['cobb.salad']
+
+# Binary indexes also support both ranges and scalars.
+bucket.get_index 'ingredients_bin', 'tomata'..'tomatz' # => ['cobb.salad']
+
+# The collection from #get_index also provides a continuation for pagination:
+c = bucket.get_index 'ingredients_bin', 'lettuce', max_results: 5
+c.length # => 5
+c.continuation # => "g2gCbQAAA="
+
+# You can use that continuation to get the next page of results:
+c2 = bucket.get_index 'ingredients_bin', 'lettuce', max_results: 5, continuation: c.continuation
+
+# More complicated operations may benefit by using the `SecondaryIndex` object:
+q = Riak::SecondaryIndex.new bucket, 'ingredients_bin', 'lettuce', max_results: 5
+
+# SecondaryIndex objects give you access to the keys...
+q.keys # => ['cobb.salad', 'wedge.salad', 'buffalo_chicken.wrap', ...]
+
+# but can also fetch values for you in parallel.
+q.values # => [<RObject {recipes,cobb.salad} ...>, <RObject {recipes,wedge...
+
+# They also provide simpler pagination:
+q.has_next_page? # => true
+q2 = q.next_page
+```
+
 ## How to Contribute
 
 * Fork the project on [Github](http://github.com/basho/riak-ruby-client).  If you have already forked, use `git pull --rebase` to reapply your changes on top of the mainline. Example:

data/RELEASE_NOTES.md

@@ -1,5 +1,32 @@
 # Riak Ruby Client Release Notes
 
+## 1.4.0 Feature Release - 2013-08-16
+
+Release 1.4.0 adds support for Riak 1.4 and fixes a few bugs.
+
+Features for all Riak versions:
+
+* Multi-get parallelizes fetching multiple objects from one or more
+  buckets.
+  
+Features for Riak 1.4 and newer:
+
+* Bucket properties are settable and resettable over Protocol Buffers.
+* Distributed counters are implemented by the `Riak::Counter` class.
+* The full set of improvements to Secondary Indexes
+  are available, including pagination, streaming, and return_terms.
+  These features are available through the existing `Bucket#get_index`
+  interface as well as the new `Riak::SecondaryIndex` interface.
+* The new streaming bucket list is available in the Ruby client.
+* Setting timeout values for object CRUD, key listing, and bucket
+  listing is now possible.
+
+Bugfixes:
+
+* Tests pass and don't stall in Ruby 2.0.
+* Zero-length key and bucket names are forbidden in the client.
+* Test server works with Riak 1.4.
+
 ## 1.2.0 Feature Release - 2013-05-15
 
 Release 1.2.0 adds support for Riak 1.3 and fixes a number of bugs.

data/lib/riak/bucket.rb

@@ -1,6 +1,7 @@
 require 'riak/util/translation'
 require 'riak/client'
 require 'riak/robject'
+require 'riak/counter'
 require 'riak/failed_request'
 
 module Riak
@@ -24,6 +25,7 @@ module Riak
     def initialize(client, name)
       raise ArgumentError, t("client_type", :client => client.inspect) unless Client === client
       raise ArgumentError, t("string_type", :string => name.inspect) unless String === name
+      raise ArgumentError, t('zero_length_bucket') if name == ''
       @client, @name = client, name
     end
 
@@ -35,12 +37,12 @@ module Riak
     # @return [Array<String>] Keys in this bucket
     # @note This operation has serious performance implications and
     #    should not be used in production applications.
-    def keys(&block)
+    def keys(options={}, &block)
       warn(t('list_keys', :backtrace => caller.join("\n    "))) unless Riak.disable_list_keys_warnings
       if block_given?
-        @client.list_keys(self, &block)
+        @client.list_keys(self, options, &block)
       else
-        @client.list_keys(self)
+        @client.list_keys(self, options)
       end
     end
 
@@ -98,6 +100,15 @@ module Riak
     end
     alias :[] :get
 
+    # Retrieve multiple keys from this bucket.
+    # @param [Array<String>] array of keys to fetch
+    # @return [Hash<String, Riak::RObject>] hash of keys to objects
+    def get_many(keys)
+      pairs = keys.map{|k| [self, k]}
+      results = Multiget.get_all @client, pairs
+      results.keys.inject(Hash.new) { |mem, var| mem[var[1]] = results[var]; mem }
+    end
+
     # Create a new blank object
     # @param [String] key the key of the new object
     # @return [RObject] the new, unsaved object
@@ -122,6 +133,14 @@ module Riak
       end
     end
 
+    # Gets a counter object. Counters initially hvae a value of zero, and can be
+    # incremented and decremented by integer values.
+    # @param [String] key the key of the counter to fetch
+    # @return [Counter]
+    def counter(key)
+      Riak::Counter.new self, key
+    end
+
     # Checks whether an object exists in Riak.
     # @param [String] key the key to check
     # @param [Hash] options quorum options
@@ -155,8 +174,8 @@ module Riak
     #   Range of values to query
     # @return [Array<String>] a list of keys that match the index
     #   query
-    def get_index(index, query)
-      client.get_index(self, index, query)
+    def get_index(index, query, options={})
+      client.get_index(self, index, query, options)
     end
 
     # @return [true, false] whether the bucket allows divergent siblings

data/lib/riak/client.rb

@@ -14,7 +14,10 @@ require 'riak/client/excon_backend'
 require 'riak/client/protobuffs_backend'
 require 'riak/client/beefcake_protobuffs_backend'
 require 'riak/bucket'
+require 'riak/multiget'
+require 'riak/secondary_index'
 require 'riak/stamp'
+require 'riak/list_buckets'
 
 module Riak
   # A client connection to Riak.
@@ -70,6 +73,9 @@ module Riak
     # @return [Client::Pool] A pool of protobuffs connections
     attr_reader :protobuffs_pool
 
+    # @return [Integer] The number of threads for multiget requests
+    attr_reader :multiget_threads
+
     # Creates a client connection to Riak
     # @param [Hash] options configuration options for the client
     # @option options [Array] :nodes A list of nodes this client connects to.
@@ -119,6 +125,7 @@ module Riak
       self.protobuffs_backend = options[:protobuffs_backend] || :Beefcake
       self.client_id          = options[:client_id]          if options[:client_id]
       self.ssl                = options[:ssl]                if options[:ssl]
+      self.multiget_threads   = options[:multiget_threads]
     end
 
     # Yields a backend for operations that are protocol-independent.
@@ -148,6 +155,7 @@ module Riak
     # @option options [Boolean] :props (false) whether to retreive the bucket properties
     # @return [Bucket] the requested bucket
     def bucket(name, options={})
+      raise ArgumentError, t('zero_length_bucket') if name == ''
       unless (options.keys - [:props]).empty?
         raise ArgumentError, "invalid options"
       end
@@ -162,10 +170,13 @@ module Riak
     # @note This is an expensive operation and should be used only
     #       in development.
     # @return [Array<Bucket>] a list of buckets
-    def buckets
+    def buckets(options={}, &block)
       warn(t('list_buckets', :backtrace => caller.join("\n    "))) unless Riak.disable_list_keys_warnings
+
+      return ListBuckets.new self, options, block if block_given?
+
       backend do |b|
-        b.list_buckets.map {|name| Bucket.new(self, name) }
+        b.list_buckets(options).map {|name| Bucket.new(self, name) }
       end
     end
     alias :list_buckets :buckets
@@ -187,6 +198,24 @@ module Riak
       end
     end
 
+    # Set the number of threads to use for multiget operations.
+    # If set to nil, defaults to twice the number of nodes.
+    # @param [Integer] count The number of threads to use.
+    # @raise [ArgumentError] when a non-nil, non-positive-Integer count is given
+    def multiget_threads=(count)
+      if count.nil?
+        @multiget_threads = nodes.length * 2
+        return
+      end
+
+      if count.is_a?(Integer) && count > 0
+        @multiget_threads = count
+        return
+      end
+
+      raise ArgumentError, t("invalid_multiget_thread_count")
+    end
+
     # Set the client ID for this client. Must be a string or Fixnum value 0 =<
     # value < MAX_CLIENT_ID.
     # @param [String, Fixnum] value The internal client ID used by Riak to route responses
@@ -268,14 +297,20 @@ module Riak
     end
 
     # Queries a secondary index on a bucket. See Bucket#get_index
-    def get_index(bucket, index, query)
+    def get_index(bucket, index, query, options={})
       backend do |b|
-        b.get_index bucket, index, query
+        b.get_index bucket, index, query, options
       end
     end
 
+    # Get multiple objects in parallel.
+    def get_many(pairs)
+      Multiget.get_all self, pairs
+    end
+
     # Get an object. See Bucket#get
     def get_object(bucket, key, options = {})
+      raise ArgumentError, t("zero_length_key") if key == ''
       backend do |b|
         b.fetch_object(bucket, key, options)
       end
@@ -307,14 +342,14 @@ module Riak
     end
 
     # Retrieves a list of keys in the given bucket. See Bucket#keys
-    def list_keys(bucket, &block)
+    def list_keys(bucket, options={}, &block)
       if block_given?
         backend do |b|
-          b.list_keys bucket, &block
+          b.list_keys bucket, options, &block
         end
       else
         backend do |b|
-          b.list_keys bucket
+          b.list_keys bucket, options
         end
       end
     end

data/lib/riak/client/beefcake/message_codes.rb

@@ -0,0 +1,56 @@
+module Riak
+  class Client
+    module BeefcakeMessageCodes
+      MESSAGE_TO_CODE = {
+        :ErrorResp => 0,
+        :PingReq => 1,
+        :PingResp => 2,
+        :GetClientIdReq => 3,
+        :GetClientIdResp => 4,
+        :SetClientIdReq => 5,
+        :SetClientIdResp => 6,
+        :GetServerInfoReq => 7,
+        :GetServerInfoResp => 8,
+        :GetReq => 9,
+        :GetResp => 10,
+        :PutReq => 11,
+        :PutResp => 12,
+        :DelReq => 13,
+        :DelResp => 14,
+        :ListBucketsReq => 15,
+        :ListBucketsResp => 16,
+        :ListKeysReq => 17,
+        :ListKeysResp => 18,
+        :GetBucketReq => 19,
+        :GetBucketResp => 20,
+        :SetBucketReq => 21,
+        :SetBucketResp => 22,
+        :MapRedReq => 23,
+        :MapRedResp => 24,
+        :IndexReq => 25,
+        :IndexResp => 26,
+        :SearchQueryReq => 27,
+        :SearchQueryResp => 28,
+        :ResetBucketReq => 29,
+        :ResetBucketResp => 30,
+        :CSBucketReq => 40,
+        :CSBucketResp => 41,
+        :CounterUpdateReq => 50,
+        :CounterUpdateResp => 51,
+        :CounterGetReq => 52,
+        :CounterGetResp => 53,
+      }
+
+      CODE_TO_MESSAGE = MESSAGE_TO_CODE.invert
+
+      # ugly shims
+      def self.index(message_name)
+        MESSAGE_TO_CODE[message_name]
+      end
+
+      def self.[](message_code)
+        CODE_TO_MESSAGE[message_code]
+      end
+    end
+  end
+end

data/lib/riak/client/beefcake/messages.rb

@@ -11,10 +11,90 @@ module Riak
         optional :value, :bytes, 2
       end
 
+      # module-function pair for commit hooks and other properties that take
+      # functions
+      class RpbModFun
+        include Beefcake::Message
+
+        required :module,   :bytes, 1
+        required :function, :bytes, 2
+      end
+
+      class RpbCommitHook
+        include Beefcake::Message
+
+        optional :modfun, RpbModFun, 1
+        optional :name,   :bytes,    2
+      end
+
       class RpbBucketProps
         include Beefcake::Message
-        optional :n_val,      :uint32, 1
-        optional :allow_mult, :bool,   2
+
+        # riak_core_app
+        optional :n_val,           :uint32,       1
+        optional :allow_mult,      :bool,         2
+        optional :last_write_wins, :bool,         3
+
+        # riak_core values with special handling, see below
+        repeated :precommit,       RpbCommitHook, 4
+        optional :has_precommit,   :bool,         5, :default => false
+        repeated :postcommit,      RpbCommitHook, 6
+        optional :has_postcommit,  :bool,         7, :default => false
+
+        optional :chash_keyfun,    RpbModFun,     8
+
+        # riak_kv_app
+        optional :linkfun,         RpbModFun,     9
+        optional :old_vclock,      :uint32,       10
+        optional :young_vclock,    :uint32,       11
+        optional :big_vclock,      :uint32,       12
+        optional :small_vclock,    :uint32,       13
+        optional :pr,              :uint32,       14
+        optional :r,               :uint32,       15
+        optional :w,               :uint32,       16
+        optional :pw,              :uint32,       17
+        optional :dw,              :uint32,       18
+        optional :rw,              :uint32,       19
+        optional :basic_quorum,    :bool,         20
+        optional :notfound_ok,     :bool,         21
+
+        # riak_kv_multi_backend
+        optional :backend,         :bytes,        22
+
+        # riak_search bucket fixup
+        optional :search,          :bool,         23
+
+        module RpbReplMode
+          FALSE = 0
+          REALTIME = 1
+          FULLSYNC = 2
+          TRUE = 3
+        end
+
+        optional :repl,            RpbReplMode,   24
+
+        # "repeated" elements with zero items are indistinguishable
+        # from a nil, so we have to manage has_precommit/has_postcommit
+        # flags.
+        def precommit=(newval)
+          @precommit = newval
+          @has_precommit = !!newval 
+        end
+
+        def has_precommit=(newval)
+          @has_precommit = newval
+          @precommit ||= [] if newval
+        end
+
+        def postcommit=(newval)
+          @postcommit = newval
+          @has_postcommit = !!newval 
+        end
+
+        def has_postcommit=(newval)
+          @has_postcommit = newval
+          @postcommit ||= [] if newval
+        end
       end
 
       class RpbLink
@@ -72,6 +152,9 @@ module Riak
         optional :if_modified,   :bytes,  7
         optional :head,          :bool,   8
         optional :deletedvclock, :bool,   9
+        optional :timeout,       :uint32, 10
+        optional :sloppy_quorum, :bool,   11
+        optional :n_val,         :uint32, 12
       end
 
       class RpbGetResp
@@ -94,6 +177,10 @@ module Riak
         optional :if_not_modified, :bool,      9
         optional :if_none_match,   :bool,      10
         optional :return_head,     :bool,      11
+        optional :timeout,         :uint32,    12
+        optional :asis,            :bool,      13
+        optional :sloppy_quorum,   :bool,      14
+        optional :n_val,           :uint32,    15
       end
 
       class RpbPutResp
@@ -105,25 +192,36 @@ module Riak
 
       class RpbDelReq
         include Beefcake::Message
-        required :bucket, :bytes,  1
-        required :key,    :bytes,  2
-        optional :rw,     :uint32, 3
-        optional :vclock, :bytes,  4
-        optional :r,      :uint32, 5
-        optional :w,      :uint32, 6
-        optional :pr,     :uint32, 7
-        optional :pw,     :uint32, 8
-        optional :dw,     :uint32, 9
+        required :bucket,        :bytes,  1
+        required :key,           :bytes,  2
+        optional :rw,            :uint32, 3
+        optional :vclock,        :bytes,  4
+        optional :r,             :uint32, 5
+        optional :w,             :uint32, 6
+        optional :pr,            :uint32, 7
+        optional :pw,            :uint32, 8
+        optional :dw,            :uint32, 9
+        optional :timeout,       :uint32, 10
+        optional :sloppy_quorum, :bool,   11
+        optional :n_val,         :uint32, 12
+      end
+
+      class RpbListBucketsReq
+        include Beefcake::Message
+        optional :timeout, :uint32, 1
+        optional :stream, :bool, 2
       end
 
       class RpbListBucketsResp
         include Beefcake::Message
         repeated :buckets, :bytes, 1
+        optional :done, :bool, 2
       end
 
       class RpbListKeysReq
         include Beefcake::Message
         required :bucket, :bytes, 1
+        optional :timeout, :uint32, 2
       end
 
       class RpbListKeysResp
@@ -148,6 +246,11 @@ module Riak
         required :props,  RpbBucketProps, 2
       end
 
+      class RpbResetBucketReq
+        include Beefcake::Message
+        required :bucket, :bytes, 1
+      end
+
       class RpbMapRedReq
         include Beefcake::Message
         required :request,      :bytes, 1
@@ -168,17 +271,25 @@ module Riak
           RANGE = 1
         end
 
-        required :bucket,    :bytes,         1
-        required :index,     :bytes,         2
-        required :qtype,     IndexQueryType, 3
-        optional :key,       :bytes,         4
-        optional :range_min, :bytes,         5
-        optional :range_max, :bytes,         6
+        required :bucket,       :bytes,         1
+        required :index,        :bytes,         2
+        required :qtype,        IndexQueryType, 3
+        optional :key,          :bytes,         4
+        optional :range_min,    :bytes,         5
+        optional :range_max,    :bytes,         6
+        optional :return_terms, :bool,          7
+        optional :stream,       :bool,          8
+        optional :max_results,  :uint32,        9
+        optional :continuation, :bytes,         10
+        optional :timeout,      :uint32,        11
       end
 
       class RpbIndexResp
         include Beefcake::Message
-        repeated :keys, :bytes, 1
+        repeated :keys,         :bytes,  1
+        repeated :results,      RpbPair, 2
+        optional :continuation, :bytes,  3
+        optional :done,         :bool,   4
       end
 
       class RpbSearchDoc
@@ -208,6 +319,67 @@ module Riak
         optional :max_score, :float,  2
         optional :num_found, :uint32, 3
       end
+
+      class RpbResetBucketReq
+        include Beefcake::Message
+        required :bucket, :bytes, 1
+      end
+
+      class RpbCSBucketReq
+        include Beefcake::Message
+        required :bucket,       :bytes,  1
+        required :start_key,    :bytes,  2
+        optional :end_key,      :bytes,  3
+        optional :start_incl,   :bool,   4, default: true
+        optional :end_incl,     :bool,   5, default: false
+        optional :continuation, :bytes,  6
+        optional :max_results,  :uint32, 7
+        optional :timeout,      :uint32, 8
+      end
+      
+      class RpbIndexObject
+        include Beefcake::Message
+        required :key,    :bytes,     1
+        required :object, RpbGetResp, 2
+      end
+
+      class RpbCSBucketResp
+        include Beefcake::Message
+        repeated :objects,      RpbIndexObject, 1
+        optional :continuation, :bytes,         2
+        optional :done,         :bool,          3
+      end
+
+      class RpbCounterUpdateReq
+        include Beefcake::Message
+        required :bucket,      :bytes,  1
+        required :key,         :bytes,  2
+        required :amount,      :sint64, 3
+        optional :w,           :uint32, 4
+        optional :dw,          :uint32, 5
+        optional :pw,          :uint32, 6
+        optional :returnvalue, :bool,   7
+      end
+
+      class RpbCounterUpdateResp
+        include Beefcake::Message
+        optional :value, :sint64, 1
+      end
+
+      class RpbCounterGetReq
+        include Beefcake::Message
+        required :bucket,       :bytes,  1
+        required :key,          :bytes,  2
+        optional :r,            :uint32, 3
+        optional :pr,           :uint32, 4
+        optional :basic_quorum, :bool,   5
+        optional :notfound_ok,  :bool,   6
+      end
+
+      class RpbCounterGetResp
+        include Beefcake::Message
+        optional :value, :sint64, 1
+      end
     end
   end
 end