riak-client 1.0.5 → 1.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -1,3 +1,7 @@
+## NOTE: This file must be manually kept in sync with the gemspec, but
+## should not normally need to be modified unless new top-level files
+## or directory trees are being added.
+
 ## MAC OS
 .DS_Store
 
@@ -18,9 +22,6 @@ coverage
 rdoc
 pkg
 
-## This line is for the dirglob only
-pkg/*
-
 ## PROJECT::SPECIFIC
 _notes
 doc

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -2,6 +2,7 @@ source :rubygems
 
 gemspec
 gem 'bundler'
+gem 'rake'
 
 group :guard do
   gem 'guard-rspec'

data/RELEASE_NOTES.md

@@ -1,5 +1,52 @@
 # Riak Ruby Client Release Notes
 
+## 1.1.0 Feature Release - 2012-11-07
+
+Release 1.1.0 includes full Riak 1.2 compatibility, and includes
+improvements to the handling of siblings, the node generation
+tools, and resolves a number of important bugs.
+
+Features:
+
+* Client features are enabled or disabled based on the detected Riak
+  version.
+* Riak 1.2 compatibility, including search and 2I over Protocol
+  Buffers.
+* Phaseless MapReduce (which was available in 1.1) is allowed, using
+  feature detection to determine whether an exception is raised.
+* Conditional store_object operations on Protocol Buffers use the
+  message features available since Riak 1.0.
+* The integration test-suite can be run without generating a test
+  node, which lets us support riak_test.
+
+Bugfixes:
+
+* URL-escaping now allows some normally URI-safe characters to be
+  escaped.
+* JRuby should be more reliable when attaching to a generated node's
+  console.
+* The client backend pool has been extracted to the Innertube gem,
+  which is now a dependency.
+* Fix a documentation issue around key-filters.
+* Fix RSpec formatter and deprecation errors.
+* Object siblings are now a separate class (RContent) rather than
+  being unclean copies of the parent RObject. If only one sibling
+  exists, the original accessors (e.g. `content_type`, `data`) will
+  behave as expected. When more than one sibling exists, they will
+  raise `Riak::Conflict`. This should prevent unintentional storing of
+  unresolved objects back into Riak as `multipart/mixed` values.
+* `Riak::Client#ssl=` won't blow away existing `ssl_options` if set to
+  `true`.
+* Generated nodes will ensure that the source's
+  `ssl_distribution.args_file` exists by invoking `riak chkconfig`.
+* Copy fixes for the `$key` index on the memory/test backend from
+  riak_kv.
+* The shape of MapReduce results will no longer be changed by the
+  Protocol Buffers backend, which manifested as kept phases without
+  results being removed from the return value. Implementing this
+  required all HTTP requests to use streaming, even if invoked without
+  a block.
+
 ## 1.0.5 Packaging Fix Release - 2012-10-12
 
 Release 1.0.5 fixes a bug with the RubyGems packaging that

data/Rakefile

@@ -69,5 +69,4 @@ desc "Run All Specs (including slow specs)"
 RSpec::Core::RakeTask.new(:ci) do |spec|
   spec.rspec_opts = %w[--profile]
 end
-
 task :default => :ci

data/erl_src/riak_kv_test_backend.erl

@@ -61,6 +61,7 @@
 
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
+-compile([export_all]).
 -endif.
 
 -define(API_VERSION, 1).
@@ -427,6 +428,9 @@ fold_keys_fun(FoldKeysFun, {index, FilterBucket, {eq, <<"$bucket">>, _}}) ->
 fold_keys_fun(FoldKeysFun, {index, FilterBucket, {range, <<"$key">>, _, _}}) ->
     %% 2I range query on special $key field...
     fold_keys_fun(FoldKeysFun, {bucket, FilterBucket});
+fold_keys_fun(FoldKeysFun, {index, FilterBucket, {eq, <<"$key">>, _}}) ->
+    %% 2I eq query on special $key field...
+    fold_keys_fun(FoldKeysFun, {bucket, FilterBucket});
 fold_keys_fun(FoldKeysFun, {index, _FilterBucket, _Query}) ->
     fun({{Bucket, _FilterField, _FilterTerm, Key}, _}, Acc) ->
             FoldKeysFun(Bucket, Key, Acc);
@@ -469,6 +473,8 @@ get_index_folder(Folder, Acc0, {index, Bucket, {range, <<"$key">>, Min, Max}}, D
     fun() ->
             key_range_folder(Folder, Acc0, DataRef, {Bucket, Min}, {Bucket, Min, Max})
     end;
+get_index_folder(Folder, Acc0, {index, Bucket, {eq, <<"$key">>, Val}}, DataRef, IndexRef) ->
+    get_index_folder(Folder, Acc0, {index, Bucket, {range, <<"$key">>, Val, Val}}, DataRef, IndexRef);
 get_index_folder(Folder, Acc0, {index, Bucket, {eq, Field, Term}}, _, IndexRef) ->
     fun() ->
             index_range_folder(Folder, Acc0, IndexRef, {Bucket, Field, Term, undefined}, {Bucket, Field, Term, Term})
@@ -668,6 +674,34 @@ max_memory_test_() ->
      ?_assertEqual({ok, Value2, State2}, get(Bucket, Key2, State2))
     ].
 
+regression_367_key_range_test_() ->
+    {ok, State} = start(142, []),
+    Keys = [begin
+                Bin = list_to_binary(integer_to_list(I)),
+                if I < 10 ->
+                        <<"obj0", Bin/binary>>;
+                   true -> <<"obj", Bin/binary>>
+                end
+            end || I <- lists:seq(1,30) ],
+    Bucket = <<"keyrange">>,
+    Value = <<"foobarbaz">>,
+    State1 = lists:foldl(fun(Key, IState) ->
+                                 {ok, NewState} = put(Bucket, Key, [], Value, IState),
+                                 NewState
+                         end, State, Keys),
+    Folder = fun(_B, K, Acc) ->
+                     Acc ++ [K]
+             end,
+    [
+     ?_assertEqual({ok, [<<"obj01">>]}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj01">>, <<"obj01">>}}], State1)),
+     ?_assertEqual({ok, [<<"obj10">>,<<"obj11">>]}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj10">>, <<"obj11">>}}], State1)),
+     ?_assertEqual({ok, [<<"obj01">>]}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj00">>, <<"obj01">>}}], State1)),
+     ?_assertEqual({ok, lists:sort(Keys)}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj0">>, <<"obj31">>}}], State1)),
+     ?_assertEqual({ok, []}, fold_keys(Folder, [], [{index, Bucket, {range, <<"$key">>, <<"obj31">>, <<"obj32">>}}], State1)),
+     ?_assertEqual({ok, [<<"obj01">>]}, fold_keys(Folder, [], [{index, Bucket, {eq, <<"$key">>, <<"obj01">>}}], State1)),
+     ?_assertEqual(ok, stop(State1))
+    ].
+
 -ifdef(EQC).
 
 eqc_test_() ->

data/lib/riak/client.rb

@@ -1,10 +1,10 @@
 require 'tempfile'
 require 'delegate'
+require 'innertube'
 require 'riak'
 require 'riak/util/translation'
 require 'riak/util/escape'
 require 'riak/failed_request'
-require 'riak/client/pool'
 require 'riak/client/decaying'
 require 'riak/client/node'
 require 'riak/client/search'
@@ -47,6 +47,8 @@ module Riak
       SystemCallError,
     ]
 
+    Pool = ::Innertube::Pool
+
     # @return [String] The protocol to use for the Riak endpoint
     attr_reader :protocol
 

data/lib/riak/client/beefcake/messages.rb

@@ -89,7 +89,7 @@ module Riak
         required :content,         RpbContent, 4
         optional :w,               :uint32,    5
         optional :dw,              :uint32,    6
-        optional :returnbody,     :bool,      7
+        optional :returnbody,      :bool,      7
         optional :pw,              :uint32,    8
         optional :if_not_modified, :bool,      9
         optional :if_none_match,   :bool,      10
@@ -160,6 +160,54 @@ module Riak
         optional :response, :bytes,  2
         optional :done,     :bool,   3
       end
+
+      class RpbIndexReq
+        include Beefcake::Message
+        module IndexQueryType
+          EQ = 0
+          RANGE = 1
+        end
+
+        required :bucket,    :bytes,         1
+        required :index,     :bytes,         2
+        required :qtype,     IndexQueryType, 3
+        optional :key,       :bytes,         4
+        optional :range_min, :bytes,         5
+        optional :range_max, :bytes,         6
+      end
+
+      class RpbIndexResp
+        include Beefcake::Message
+        repeated :keys, :bytes, 1
+      end
+
+      class RpbSearchDoc
+        include Beefcake::Message
+        # We have to name this differently than the .proto file does
+        # because Beefcake uses 'fields' as an instance method.
+        repeated :properties, RpbPair, 1
+      end
+
+      class RpbSearchQueryReq
+        include Beefcake::Message
+        required :q,       :bytes,  1
+        required :index,   :bytes,  2
+        optional :rows,    :uint32, 3
+        optional :start,   :uint32, 4
+        optional :sort,    :bytes,  5
+        optional :filter,  :bytes,  6
+        optional :df,      :bytes,  7
+        optional :op,      :bytes,  8
+        repeated :fl,      :bytes,  9
+        optional :presort, :bytes, 10
+      end
+
+      class RpbSearchQueryResp
+        include Beefcake::Message
+        repeated :docs, RpbSearchDoc, 1, :default => []
+        optional :max_score, :float,  2
+        optional :num_found, :uint32, 3
+      end
     end
   end
 end

data/lib/riak/client/beefcake/object_methods.rb

@@ -31,40 +31,33 @@ module Riak
           return robject if pbuf.respond_to?(:unchanged) && pbuf.unchanged # Reloading
           robject.vclock = Base64.encode64(pbuf.vclock).chomp if pbuf.vclock
           robject.key = maybe_unescape(pbuf.key) if pbuf.respond_to?(:key) && pbuf.key # Put w/o key
-          if pbuf.content.size > 1
-            robject.conflict = true
-            robject.siblings = pbuf.content.map do |c|
-              sibling = RObject.new(robject.bucket, robject.key)
-              sibling.vclock = robject.vclock
+          robject.siblings = pbuf.content.map do |c|
+            RContent.new(robject) do |sibling|
               load_content(c, sibling)
             end
-
-            return robject.attempt_conflict_resolution
-          else
-            load_content(pbuf.content.first, robject)
           end
-          robject
+          robject.conflict? ? robject.attempt_conflict_resolution : robject
         end
 
         private
-        def load_content(pbuf, robject)
+        def load_content(pbuf, rcontent)
           if ENCODING && pbuf.charset.present?
             pbuf.value.force_encoding(pbuf.charset) if Encoding.find(pbuf.charset)
           end
-          robject.raw_data = pbuf.value
-          robject.etag = pbuf.vtag if pbuf.vtag.present?
-          robject.content_type = pbuf.content_type if pbuf.content_type.present?
-          robject.links = pbuf.links.map(&method(:decode_link)) if pbuf.links.present?
-          pbuf.usermeta.each {|pair| decode_meta(pair, robject.meta) } if pbuf.usermeta.present?
+          rcontent.raw_data = pbuf.value
+          rcontent.etag = pbuf.vtag if pbuf.vtag.present?
+          rcontent.content_type = pbuf.content_type if pbuf.content_type.present?
+          rcontent.links = pbuf.links.map(&method(:decode_link)) if pbuf.links.present?
+          pbuf.usermeta.each {|pair| decode_meta(pair, rcontent.meta) } if pbuf.usermeta.present?
           if pbuf.indexes.present?
-            robject.indexes.clear
-            pbuf.indexes.each {|pair| decode_index(pair, robject.indexes) }
+            rcontent.indexes.clear
+            pbuf.indexes.each {|pair| decode_index(pair, rcontent.indexes) }
           end
           if pbuf.last_mod.present?
-            robject.last_modified = Time.at(pbuf.last_mod)
-            robject.last_modified += pbuf.last_mod_usecs / 1000000 if pbuf.last_mod_usecs.present?
+            rcontent.last_modified = Time.at(pbuf.last_mod)
+            rcontent.last_modified += pbuf.last_mod_usecs / 1000000 if pbuf.last_mod_usecs.present?
           end
-          robject
+          rcontent
         end
 
         def decode_link(pbuf)

data/lib/riak/client/beefcake_protobuffs_backend.rb

@@ -31,7 +31,7 @@ module Riak
       end
 
       def fetch_object(bucket, key, options={})
-        options = normalize_quorums(options)
+        options = prune_unsupported_options(:GetReq, normalize_quorums(options))
         bucket = Bucket === bucket ? bucket.name : bucket
         req = RpbGetReq.new(options.merge(:bucket => maybe_encode(bucket), :key => maybe_encode(key)))
         write_protobuff(:GetReq, req)
@@ -43,18 +43,25 @@ module Riak
         options[:bucket] = maybe_encode(robject.bucket.name)
         options[:key] = maybe_encode(robject.key)
         options[:if_modified] = maybe_encode Base64.decode64(robject.vclock) if robject.vclock
-        req = RpbGetReq.new(options)
+        req = RpbGetReq.new(prune_unsupported_options(:GetReq, options))
         write_protobuff(:GetReq, req)
         decode_response(robject)
       end
 
       def store_object(robject, options={})
+        options = normalize_quorums(options)
         if robject.prevent_stale_writes
-          other = fetch_object(robject.bucket, robject.key)
-          raise Riak::ProtobuffsFailedRequest(:stale_object, t("stale_write_prevented")) unless other.vclock == robject.vclock
+          unless pb_conditionals?
+            other = fetch_object(robject.bucket, robject.key)
+            raise Riak::ProtobuffsFailedRequest.new(:stale_object, t("stale_write_prevented")) unless other.vclock == robject.vclock
+          end
+          if robject.vclock
+            options[:if_not_modified] = true
+          else
+            options[:if_none_match] = true
+          end
         end
-        options = normalize_quorums(options)
-        req = dump_object(robject, options)
+        req = dump_object(robject, prune_unsupported_options(:PutReq, options))
         write_protobuff(:PutReq, req)
         decode_response(robject)
       end
@@ -65,7 +72,7 @@ module Riak
         options[:bucket] = maybe_encode(bucket)
         options[:key] = maybe_encode(key)
         options[:vclock] = Base64.decode64(options[:vclock]) if options[:vclock]
-        req = RpbDelReq.new(options)
+        req = RpbDelReq.new(prune_unsupported_options(:DelReq, options))
         write_protobuff(:DelReq, req)
         decode_response
       end
@@ -102,19 +109,47 @@ module Riak
       end
 
       def mapred(mr, &block)
+        raise MapReduceError.new(t("empty_map_reduce_query")) if mr.query.empty? && !mapred_phaseless?
         req = RpbMapRedReq.new(:request => mr.to_json, :content_type => "application/json")
         write_protobuff(:MapRedReq, req)
-        results = []
+        results = MapReduce::Results.new(mr)
         while msg = decode_response
           break if msg.done
           if block_given?
             yield msg.phase, JSON.parse(msg.response)
           else
-            results[msg.phase] ||= []
-            results[msg.phase] += JSON.parse(msg.response)
+            results.add msg.phase, JSON.parse(msg.response)
           end
         end
-        block_given? || results.compact.size == 1 ? results.last : results
+        block_given? || results.report
+      end
+
+      def get_index(bucket, index, query)
+        return super unless pb_indexes?
+        if Range === query
+          options = {
+            :qtype => RpbIndexReq::IndexQueryType::RANGE,
+            :range_min => query.begin.to_s,
+            :range_max => query.end.to_s
+          }
+        else
+          options = {
+            :qtype => RpbIndexReq::IndexQueryType::EQ,
+            :key => query.to_s
+          }
+        end
+        req = RpbIndexReq.new(options.merge(:bucket => bucket, :index => index))
+        write_protobuff(:IndexReq, req)
+        decode_response
+      end
+
+      def search(index, query, options={})
+        return super unless pb_search?
+        options = options.symbolize_keys
+        options[:op] = options.delete(:'q.op') if options[:'q.op']
+        req = RpbSearchQueryReq.new(options.merge(:index => index || 'search', :q => query))
+        write_protobuff(:SearchQueryReq, req)
+        decode_response
       end
 
       private
@@ -167,12 +202,23 @@ module Riak
             {'n_val' => res.props.n_val, 'allow_mult' => res.props.allow_mult}
           when :MapRedResp
             RpbMapRedResp.decode(message)
+          when :IndexResp
+            res = RpbIndexResp.decode(message)
+            res.keys
+          when :SearchQueryResp
+            res = RpbSearchQueryResp.decode(message)
+            { 'docs' => res.docs.map {|d| decode_doc(d) },
+              'max_score' => res.max_score,
+              'num_found' => res.num_found }
           end
         end
       rescue SystemCallError, SocketError => e
         reset_socket
         raise
-        #raise Riak::ProtobuffsFailedRequest.new(:server_error, e.message)
+      end
+
+      def decode_doc(doc)
+        Hash[doc.properties.map {|p| [ p.key, p.value ] }]
       end
     end
   end

data/lib/riak/client/decaying.rb

@@ -1,28 +1,36 @@
-class Riak::Client::Decaying
-  # A float which decays exponentially with time.
+module Riak
+  class Client
+    # A float value which decays exponentially toward 0 over time.
+    # @private
+    class Decaying
+      attr_accessor :e
+      attr_accessor :p
 
-  attr_accessor :e
-  attr_accessor :p
-  # @param[:p] The initial value
-  # @param[:e] Exponent base
-  # @param[:r] Timescale
-  def initialize(opts = {})
-    @p = opts[:p] || 0
-    @e = opts[:e] || Math::E
-    @r = opts[:r] || Math.log(0.5) / 10
-    @t0 = Time.now
-  end
+      # @param [Hash] opts options
+      # @option options [Float] :p (0.0) The initial value
+      # @option options [Float] :e (Math::E) Exponent base
+      # @option options [Float] :r (Math.log(0.5) / 10) Timescale
+      #   factor - defaulting to decay 50% every 10 seconds
+      def initialize(opts = {})
+        @p = opts[:p] || 0.0
+        @e = opts[:e] || Math::E
+        @r = opts[:r] || Math.log(0.5) / 10
+        @t0 = Time.now
+      end
 
-  # Add d to current value.
-  def <<(d)
-    @p = value + d
-  end
+      # Add to current value.
+      # @param [Float] d the value to add
+      def <<(d)
+        @p = value + d
+      end
 
-  # Return current value
-  def value
-    now = Time.now
-    dt = now - @t0
-    @t0 = now
-    @p = @p * (@e ** (@r * dt))
+      # @return [Float] the current value (adjusted for the time decay)
+      def value
+        now = Time.now
+        dt = now - @t0
+        @t0 = now
+        @p = @p * (@e ** (@r * dt))
+      end
+    end
   end
 end