riak-client 0.9.0.beta → 0.9.0.beta2

data/Gemfile

@@ -1,22 +1,25 @@
-source :gemcutter
+source :rubygems
 
 gem 'i18n'
 gem 'builder'
-gem 'rspec', "~>2.0.0"
+gem 'rspec', "~>2.4.0"
 gem 'fakeweb', ">=1.2"
 gem 'rack', '>=1.0'
 gem 'rake'
 gem 'bundler'
-gem 'excon', "~>0.3.4"
+gem 'excon', "~>0.5.7"
+gem 'beefcake', '~>0.2.0'
 
-if defined? JRUBY_VERSION
-  gem 'json'
-  gem 'jruby-openssl'
-else
+platforms :mri do
   gem 'curb', '>=0.6'
   gem 'yajl-ruby'
 end
 
+platforms :jruby do
+  gem 'json'
+  gem 'jruby-openssl'
+end
+
 group :integration do
   gem 'activesupport', '~>3.0'
 end

data/Rakefile

@@ -9,13 +9,14 @@ gemspec = Gem::Specification.new do |gem|
   gem.email = "sean@basho.com"
   gem.homepage = "http://seancribbs.github.com/ripple"
   gem.authors = ["Sean Cribbs"]
-  gem.add_development_dependency "rspec", "~>2.0.0"
+  gem.add_development_dependency "rspec", "~>2.4.0"
   gem.add_development_dependency "fakeweb", ">=1.2"
   gem.add_development_dependency "rack", ">=1.0"
   gem.add_development_dependency "curb", ">=0.6"
-  gem.add_development_dependency "excon", "~>0.3.4"
+  gem.add_development_dependency "excon", "~>0.5.7"
   gem.add_dependency "i18n", ">=0.4.0"
   gem.add_dependency "builder", "~>2.1.2"
+  gem.add_dependency "beefcake", "~>0.2.0"
 
   gem.files = %W{
                   erl_src/riak_kv_test_backend.beam
@@ -24,6 +25,9 @@ gemspec = Gem::Specification.new do |gem|
                   lib/active_support/cache/riak_store.rb
                   lib/riak/bucket.rb
                   lib/riak/cache_store.rb
+                  lib/riak/client/beefcake/messages.rb
+                  lib/riak/client/beefcake/object_methods.rb
+                  lib/riak/client/beefcake_protobuffs_backend.rb
                   lib/riak/client/curb_backend.rb
                   lib/riak/client/excon_backend.rb
                   lib/riak/client/http_backend/configuration.rb
@@ -32,6 +36,8 @@ gemspec = Gem::Specification.new do |gem|
                   lib/riak/client/http_backend/transport_methods.rb
                   lib/riak/client/http_backend.rb
                   lib/riak/client/net_http_backend.rb
+                  lib/riak/client/protobuffs_backend.rb
+                  lib/riak/client/pump.rb
                   lib/riak/client.rb
                   lib/riak/core_ext/blank.rb
                   lib/riak/core_ext/extract_options.rb
@@ -56,6 +62,7 @@ gemspec = Gem::Specification.new do |gem|
                   lib/riak/util/escape.rb
                   lib/riak/util/fiber1.8.rb
                   lib/riak/util/headers.rb
+                  lib/riak/util/multipart/stream_parser.rb
                   lib/riak/util/multipart.rb
                   lib/riak/util/tcp_socket_extensions.rb
                   lib/riak/util/translation.rb
@@ -66,7 +73,12 @@ gemspec = Gem::Specification.new do |gem|
                   spec/fixtures/cat.jpg
                   spec/fixtures/multipart-blank.txt
                   spec/fixtures/multipart-with-body.txt
+                  spec/fixtures/server.cert.crt
+                  spec/fixtures/server.cert.key
+                  spec/fixtures/test.pem
                   spec/integration/riak/cache_store_spec.rb
+                  spec/integration/riak/http_backends_spec.rb
+                  spec/integration/riak/protobuffs_backends_spec.rb
                   spec/integration/riak/test_server_spec.rb
                   spec/riak/bucket_spec.rb
                   spec/riak/client_spec.rb
@@ -74,14 +86,19 @@ gemspec = Gem::Specification.new do |gem|
                   spec/riak/escape_spec.rb
                   spec/riak/excon_backend_spec.rb
                   spec/riak/headers_spec.rb
+                  spec/riak/http_backend/configuration_spec.rb
                   spec/riak/http_backend/object_methods_spec.rb
+                  spec/riak/http_backend/transport_methods_spec.rb
                   spec/riak/http_backend_spec.rb
                   spec/riak/link_spec.rb
+                  spec/riak/map_reduce/filter_builder_spec.rb
+                  spec/riak/map_reduce/phase_spec.rb
                   spec/riak/map_reduce_spec.rb
                   spec/riak/multipart_spec.rb
                   spec/riak/net_http_backend_spec.rb
-                  spec/riak/object_spec.rb
+                  spec/riak/robject_spec.rb
                   spec/riak/search_spec.rb
+                  spec/riak/stream_parser_spec.rb
                   spec/riak/walk_spec_spec.rb
                   spec/spec_helper.rb
                   spec/support/drb_mock_server.rb
@@ -89,6 +106,7 @@ gemspec = Gem::Specification.new do |gem|
                   spec/support/mock_server.rb
                   spec/support/mocks.rb
                   spec/support/test_server.yml.example
+                  spec/support/unified_backend_examples.rb
               }
 
   gem.test_files = gem.files.grep(/_spec\.rb$/)

data/erl_src/riak_kv_test_backend.erl

@@ -20,7 +20,8 @@
 %%
 %% -------------------------------------------------------------------
 
-% @doc riak_kv_test_backend is a Riak storage backend using ets that exposes a reset function for efficiently clearing stored data.
+% @doc riak_kv_test_backend is a Riak storage backend using ets that
+%      exposes a reset function for efficiently clearing stored data.
 
 -module(riak_kv_test_backend).
 -behavior(riak_kv_backend).
@@ -34,29 +35,47 @@
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
+
 % @type state() = term().
--record(state, {t}).
+-record(state, {t, p}).
 
 % @spec start(Partition :: integer(), Config :: proplist()) ->
 %                        {ok, state()} | {{error, Reason :: term()}, state()}
 start(Partition, _Config) ->
     gen_server:start_link(?MODULE, [Partition], []).
 
-% @spec reset() -> ok
+% @spec reset() -> ok | {error, timeout}
 reset() ->
-    Pids = lists:filter(fun(Item) ->
-                                lists:prefix("test_backend", atom_to_list(Item))
-                        end, registered()),
-    [gen_server:call(Pid, reset) || Pid <- Pids],
-    ok.
+    Pids = lists:foldl(fun(Item, Acc) ->
+                               case lists:prefix("test_backend", atom_to_list(Item)) of
+                                   true -> [whereis(Item)|Acc];
+                                   _ -> Acc
+                               end
+                       end, [], registered()),
+    [gen_server:cast(Pid,{reset, self()})|| Pid <- Pids],
+    receive_reset(Pids).
+
+receive_reset([]) -> ok;
+receive_reset(Pids) ->
+    receive
+        {reset, Pid} ->
+            receive_reset(lists:delete(Pid, Pids))
+    after 1000 ->
+            {error, timeout}
+    end.
 
 %% @private
 init([Partition]) ->
     PName = list_to_atom("test_backend" ++ integer_to_list(Partition)),
+    P = list_to_atom(integer_to_list(Partition)),
     register(PName, self()),
-    {ok, #state{t=ets:new(list_to_atom(integer_to_list(Partition)),[])}}.
+    {ok, #state{t=ets:new(P,[]), p=P}}.
 
 %% @private
+handle_cast({reset,From}, State) ->
+    ets:delete_all_objects(State#state.t),
+    From ! {reset, self()},
+    {noreply, State};
 handle_cast(_, State) -> {noreply, State}.
 
 %% @private
@@ -76,10 +95,7 @@ handle_call(drop, _From, State) ->
 handle_call({fold, Fun0, Acc}, _From, State) ->
     Fun = fun({{B,K}, V}, AccIn) -> Fun0({B,K}, V, AccIn) end,
     Reply = ets:foldl(Fun, Acc, State#state.t),
-    {reply, Reply, State};
-handle_call(reset, _From, State) ->
-    ets:delete_all_objects(State#state.t),
-    {reply, ok, State}.
+    {reply, Reply, State}.
 
 % @spec stop(state()) -> ok | {error, Reason :: term()}
 stop(SrvRef) -> gen_server:call(SrvRef,stop).

data/lib/riak/bucket.rb

@@ -113,7 +113,7 @@ module Riak
       begin
         get(key, options)
       rescue Riak::FailedRequest => fr
-        if fr.code.to_i == 404
+        if fr.not_found?
           new(key)
         else
           raise fr

data/lib/riak/cache_store.rb

@@ -77,7 +77,7 @@ module Riak
       begin
         bucket.get(key).data
       rescue Riak::FailedRequest => fr
-        raise fr unless fr.code.to_i == 404
+        raise fr unless fr.not_found?
         nil
       end
     end

data/lib/riak/client.rb

@@ -14,6 +14,7 @@
 require 'riak'
 require 'tempfile'
 require 'delegate'
+require 'riak/failed_request'
 
 module Riak
   # A client connection to Riak.
@@ -21,26 +22,45 @@ module Riak
     include Util::Translation
     include Util::Escape
 
+    autoload :Pump,           "riak/client/pump"
     autoload :HTTPBackend,    "riak/client/http_backend"
     autoload :NetHTTPBackend, "riak/client/net_http_backend"
     autoload :CurbBackend,    "riak/client/curb_backend"
     autoload :ExconBackend,   "riak/client/excon_backend"
 
+    autoload :ProtobuffsBackend, "riak/client/protobuffs_backend"
+    autoload :BeefcakeProtobuffsBackend, "riak/client/beefcake_protobuffs_backend"
+
     # When using integer client IDs, the exclusive upper-bound of valid values.
     MAX_CLIENT_ID = 4294967296
 
+    # Array of valid protocols
+    PROTOCOLS = %w[http https pbc]
+
     # Regexp for validating hostnames, lifted from uri.rb in Ruby 1.8.6
     HOST_REGEX = /^(?:(?:(?:[a-zA-Z\d](?:[-a-zA-Z\d]*[a-zA-Z\d])?)\.)*(?:[a-zA-Z](?:[-a-zA-Z\d]*[a-zA-Z\d])?)\.?|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|\[(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:(?:[a-fA-F\d]{1,4}:)*[a-fA-F\d]{1,4})?::(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))?)\])$/n
 
+    # @return [String] The protocol to use for the Riak endpoint
+    attr_reader :protocol
+
     # @return [String] The host or IP address for the Riak endpoint
     attr_reader :host
 
     # @return [Fixnum] The port of the Riak HTTP endpoint
     attr_reader :port
 
+    # @return [String] The user:pass for http basic authentication
+    attr_reader :basic_auth
+
     # @return [String] The internal client ID used by Riak to route responses
     attr_reader :client_id
 
+    # @return [Hash|nil] The SSL options that get built when using SSL
+    attr_reader :ssl_options
+
+    # @return [Hash|nil] The writer that will build valid SSL options from the provided config
+    attr_writer :ssl
+
     # @return [String] The URL path prefix to the "raw" HTTP endpoint
     attr_accessor :prefix
 
@@ -53,6 +73,9 @@ module Riak
     # @return [Symbol] The HTTP backend/client to use
     attr_accessor :http_backend
 
+    # @return [Symbol] The Protocol Buffers backend/client to use
+    attr_accessor :protobuffs_backend
+
     # Creates a client connection to Riak
     # @param [Hash] options configuration options for the client
     # @option options [String] :host ('127.0.0.1') The host or IP address for the Riak endpoint
@@ -61,19 +84,23 @@ module Riak
     # @option options [String] :mapred ('/mapred') The path to the map-reduce HTTP endpoint
     # @option options [Fixnum, String] :client_id (rand(MAX_CLIENT_ID)) The internal client ID used by Riak to route responses
     # @option options [String, Symbol] :http_backend (:NetHTTP) which  HTTP backend to use
-    # @raise [ArgumentError] raised if any options are invalid
+    # @option options [String, Symbol] :protobuffs_backend (:Beefcake) which Protocol Buffers backend to use
+    # @raise [ArgumentError] raised if any invalid options are given
     def initialize(options={})
-      unless (options.keys - [:host, :port, :prefix, :client_id, :mapred, :luwak, :http_backend]).empty?
-        raise ArgumentError, "invalid options"
+      unless (options.keys - [:protocol, :host, :port, :prefix, :client_id, :mapred, :luwak, :http_backend, :protobuffs_backend, :ssl, :basic_auth]).empty?
+        raise ArgumentError, t("invalid options")
       end
+      self.ssl          = options[:ssl]
+      self.protocol     = options[:protocol]     || "http"
       self.host         = options[:host]         || "127.0.0.1"
-      self.port         = options[:port]         || 8098
+      self.port         = options[:port]         || ((protocol == "pbc") ? 8087 : 8098)
       self.client_id    = options[:client_id]    || make_client_id
       self.prefix       = options[:prefix]       || "/riak/"
       self.mapred       = options[:mapred]       || "/mapred"
       self.luwak        = options[:luwak]        || "/luwak"
       self.http_backend = options[:http_backend] || :NetHTTP
-      raise ArgumentError, t("missing_host_and_port") unless @host && @port
+      self.protobuffs_backend = options[:protobuffs_backend] || :Beefcake
+      self.basic_auth   = options[:basic_auth] if options[:basic_auth]
     end
 
     # Set the client ID for this client. Must be a string or Fixnum value 0 =< value < MAX_CLIENT_ID.
@@ -91,6 +118,19 @@ module Riak
                    end
     end
 
+    # Set the protocol of the Riak endpoint.  Value must be in the
+    # Riak::Client::PROTOCOLS array.
+    # @raise [ArgumentError] if the protocol is not in PROTOCOLS
+    # @return [String] the protocol being assigned
+    def protocol=(value)
+      unless PROTOCOLS.include?(value.to_s)
+        raise ArgumentError, t("protocol_invalid", :invalid => value, :valid => PROTOCOLS.join(', '))
+      end
+      @ssl_options ||= {} if value === 'https'
+      @backend = nil
+      @protocol = value
+    end
+
     # Set the hostname of the Riak endpoint. Must be an IPv4, IPv6, or valid hostname
     # @param [String] value The host or IP address for the Riak endpoint
     # @raise [ArgumentError] if an invalid hostname is given
@@ -109,12 +149,34 @@ module Riak
       @port = value
     end
 
+    def basic_auth=(value)
+      raise ArgumentError, t("invalid_basic_auth") unless value.to_s.split(':').length === 2
+      @basic_auth = value
+    end
+
     # Sets the desired HTTP backend
     def http_backend=(value)
-      @http = nil
+      @http, @backend = nil, nil
       @http_backend = value
     end
 
+    # Sets the desired Protocol Buffers backend
+    def protobuffs_backend=(value)
+      @protobuffs, @backend = nil, nil
+      @protobuffs_backend = value
+    end
+
+    # Enables or disables SSL on the client to be utilized by the HTTP Backends
+    def ssl=(value)
+      @ssl_options = Hash === value ? value : {}
+      value ? ssl_enable : ssl_disable
+    end
+
+    # Checks if the current protocol is https
+    def ssl_enabled?
+      protocol === 'https'
+    end
+
     # Automatically detects and returns an appropriate HTTP backend.
     # The HTTP backend is used internally by the Riak client, but can also
     # be used to access the server directly.
@@ -130,7 +192,41 @@ module Riak
                 end
     end
 
-    alias :backend :http
+    # Automatically detects and returns an appropriate Protocol
+    # Buffers backend.  The Protocol Buffers backend is used
+    # internally by the Riak client but can also be used to access the
+    # server directly.
+    # @return [ProtobuffsBackend] the Protocol Buffers backend for
+    #    this client
+    def protobuffs
+      @protobuffs ||= begin
+                        klass = self.class.const_get("#{@protobuffs_backend}ProtobuffsBackend")
+                        if klass.configured?
+                          klass.new(self)
+                        else
+                          raise t('protobuffs_configuration', :backend => @protobuffs_backend)
+                        end
+                      end
+    end
+
+    # Returns a backend for operations that are protocol-independent.
+    # You can change which type of backend is used by setting the
+    # {#protocol}.
+    # @return [HTTPBackend,ProtobuffsBackend] an appropriate client backend
+    def backend
+      @backend ||= case @protocol.to_s
+                   when /https?/i
+                     http
+                   when /pbc/i
+                     protobuffs
+                   end
+    end
+
+    # Pings the Riak server to check for liveness.
+    # @return [true,false] whether the Riak server is alive and reachable
+    def ping
+      backend.ping
+    end
 
     # Retrieves a bucket from Riak.
     # @param [String] bucket the bucket to retrieve
@@ -228,7 +324,7 @@ module Riak
 
     # @return [String] A representation suitable for IRB and debugging output.
     def inspect
-      "#<Riak::Client #{http.root_uri.to_s}>"
+      "#<Riak::Client #{protocol}://#{host}:#{port}>"
     end
 
     private
@@ -240,6 +336,21 @@ module Riak
       Base64.encode64([n].pack("N")).chomp
     end
 
+    def ssl_enable
+      self.protocol = 'https'
+      @ssl_options[:pem] = File.read(@ssl_options[:pem_file]) if @ssl_options[:pem_file]
+      @ssl_options[:verify_mode] ||= "peer" if @ssl_options.stringify_keys.any? {|k,v| %w[pem ca_file ca_path].include?(k)}
+      @ssl_options[:verify_mode] ||= "none"
+      raise ArgumentError.new unless %w[none peer].include?(@ssl_options[:verify_mode])
+
+      @ssl_options
+    end
+
+    def ssl_disable
+      self.protocol = 'http'
+      @ssl_options  = nil
+    end
+
     # @private
     class LuwakFile < DelegateClass(Tempfile)
       attr_accessor :original_filename, :content_type

data/lib/riak/client/beefcake/messages.rb

@@ -0,0 +1,162 @@
+# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+require 'riak'
+require 'riak/link'
+require 'beefcake'
+
+module Riak
+  class Client
+    # @private
+    class BeefcakeProtobuffsBackend
+      # Embedded messages
+      class RpbPair
+        include Beefcake::Message
+        required :key,   :bytes, 1
+        optional :value, :bytes, 2
+      end
+
+      class RpbBucketProps
+        include Beefcake::Message
+        optional :n_val,      :uint32, 1
+        optional :allow_mult, :bool,   2
+      end
+
+      class RpbLink
+        include Beefcake::Message
+        optional :bucket, :bytes, 1
+        optional :key,    :bytes, 2
+        optional :tag,    :bytes, 3
+      end
+
+      class RpbContent
+        include Beefcake::Message
+        required :value,            :bytes,  1
+        optional :content_type,     :bytes,  2
+        optional :charset,          :bytes,  3
+        optional :content_encoding, :bytes,  4
+        optional :vtag,             :bytes,  5
+        repeated :links,            RpbLink, 6
+        optional :last_mod,         :uint32, 7
+        optional :last_mod_usecs,   :uint32, 8
+        repeated :usermeta,         RpbPair, 9
+      end
+
+      # Primary messages
+      class RpbErrorResp
+        include Beefcake::Message
+        required :errmsg,  :bytes,  1
+        required :errcode, :uint32, 2
+      end
+
+      class RpbGetClientIdResp
+        include Beefcake::Message
+        required :client_id, :bytes, 1
+      end
+
+      class RpbSetClientIdReq
+        include Beefcake::Message
+        required :client_id, :bytes, 1
+      end
+
+      class RpbGetServerInfoResp
+        include Beefcake::Message
+        optional :node,           :bytes, 1
+        optional :server_version, :bytes, 2
+      end
+
+      class RpbGetReq
+        include Beefcake::Message
+        required :bucket, :bytes,  1
+        required :key,    :bytes,  2
+        optional :r,      :uint32, 3
+      end
+
+      class RpbGetResp
+        include Beefcake::Message
+        repeated :content, RpbContent, 1
+        optional :vclock,  :bytes,  2
+      end
+
+      class RpbPutReq
+        include Beefcake::Message
+        required :bucket,      :bytes,  1
+        required :key,         :bytes,  2
+        optional :vclock,      :bytes,  3
+        required :content,     RpbContent, 4
+        optional :w,           :uint32, 5
+        optional :dw,          :uint32, 6
+        optional :return_body, :bool,   7
+      end
+
+      # Optional since it has the same structure as GetResp
+      # class RpbPutResp
+      #   include Beefcake::Message
+      #   repeated :content, RpbContent, 1
+      #   optional :vclock,  :bytes,  2
+      # end
+
+      class RpbDelReq
+        include Beefcake::Message
+        required :bucket, :bytes,  1
+        required :key,    :bytes,  2
+        optional :rw,     :uint32, 3
+      end
+
+      class RpbListBucketsResp
+        include Beefcake::Message
+        repeated :buckets, :bytes, 1
+      end
+
+      class RpbListKeysReq
+        include Beefcake::Message
+        required :bucket, :bytes, 1
+      end
+
+      class RpbListKeysResp
+        include Beefcake::Message
+        repeated :keys, :bytes, 1
+        optional :done, :bool,  2
+      end
+
+      class RpbGetBucketReq
+        include Beefcake::Message
+        required :bucket, :bytes, 1
+      end
+
+      class RpbGetBucketResp
+        include Beefcake::Message
+        required :props, RpbBucketProps, 1
+      end
+
+      class RpbSetBucketReq
+        include Beefcake::Message
+        required :bucket, :bytes,         1
+        required :props,  RpbBucketProps, 2
+      end
+
+      class RpbMapRedReq
+        include Beefcake::Message
+        required :request,      :bytes, 1
+        required :content_type, :bytes, 2
+      end
+
+      class RpbMapRedResp
+        include Beefcake::Message
+        optional :phase,    :uint32, 1
+        optional :response, :bytes,  2
+        optional :done,     :bool,   3
+      end
+    end
+  end
+end