riak-client 0.9.0.beta → 0.9.0.beta2

data/lib/riak/client/beefcake/object_methods.rb

@@ -0,0 +1,92 @@
+# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+require 'riak'
+
+module Riak
+  class Client
+    class BeefcakeProtobuffsBackend
+      module ObjectMethods
+        # Returns RpbPutReq
+        def dump_object(robject)
+          pbuf = RpbPutReq.new(:bucket => robject.bucket.name)
+          pbuf.key = robject.key || generate_key
+          pbuf.vclock = Base64.decode64(robject.vclock) if robject.vclock
+          pbuf.content = RpbContent.new(:value => robject.raw_data,
+                                        :content_type => robject.content_type,
+                                        :links => robject.links.map {|l| encode_link(l) }.compact)
+
+          pbuf.content.usermeta = robject.meta.map {|k,v| encode_meta(k,v)} if robject.meta.any?
+          pbuf.content.vtag = robject.etag if robject.etag.present?
+          if robject.raw_data.respond_to?(:encoding) # 1.9 support
+            pbuf.content.charset = robject.raw_data.encoding.name
+          end
+          pbuf
+        end
+
+        # Returns RObject
+        def load_object(pbuf, robject)
+          robject.vclock = Base64.encode64(pbuf.vclock).chomp if pbuf.vclock
+          if pbuf.content.size > 1
+            robject.conflict = true
+            robject.siblings = pbuf.content.map do |c|
+              sibling = RObject.new(robject.bucket, robject.key)
+              sibling.vclock = robject.vclock
+              load_content(c, sibling)
+            end
+          else
+            load_content(pbuf.content.first, robject)
+          end
+          robject
+        end
+
+        private
+        def load_content(pbuf, robject)
+          if pbuf.value.respond_to?(:force_encoding) && pbuf.charset.present?
+            pbuf.value.force_encoding(pbuf.charset) if Encoding.find(pbuf.charset)
+          end
+          robject.raw_data = pbuf.value
+          robject.etag = pbuf.vtag if pbuf.vtag.present?
+          robject.content_type = pbuf.content_type if pbuf.content_type.present?
+          robject.links = pbuf.links.map(&method(:decode_link)) if pbuf.links.present?
+          pbuf.usermeta.each {|pair| decode_meta(pair, robject.meta) } if pbuf.usermeta.present?
+          if pbuf.last_mod.present?
+            robject.last_modified = Time.at(pbuf.last_mod)
+            robject.last_modified += pbuf.last_mod_usecs / 1000000 if pbuf.last_mod_usecs.present?
+          end
+          robject
+        end
+
+        def decode_link(pbuf)
+          Riak::Link.new(pbuf.bucket, pbuf.key, pbuf.tag)
+        end
+
+        def encode_link(link)
+          return nil unless link.key.present?
+          RpbLink.new(:bucket => link.bucket.to_s, :key => link.key.to_s, :tag => link.tag.to_s)
+        end
+
+        def decode_meta(pbuf, hash)
+          hash[pbuf.key] = pbuf.value
+        end
+
+        def encode_meta(key,value)
+          return nil unless value.present?
+          RpbPair.new(:key.to_s, :value => value.to_s)
+        end
+      end
+
+      include ObjectMethods
+    end
+  end
+end

data/lib/riak/client/beefcake_protobuffs_backend.rb

@@ -0,0 +1,186 @@
+# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+require 'riak'
+require 'riak/client/protobuffs_backend'
+require 'riak/client/pump'
+
+module Riak
+  class Client
+    class BeefcakeProtobuffsBackend < ProtobuffsBackend
+      def self.configured?
+        begin
+          require 'beefcake'
+          require 'riak/client/beefcake/messages'
+          require "riak/client/beefcake/object_methods"
+          true
+        rescue LoadError, NameError
+          false
+        end
+      end
+
+      def set_client_id(id)
+        value = case id
+                when Integer
+                  [id].pack("N")
+                else
+                  id.to_s
+                end
+        req = RpbSetClientIdReq.new(:client_id => value)
+        write_protobuff(:SetClientIdReq, req)
+        decode_response
+      end
+
+      def fetch_object(bucket, key, r=nil)
+        bucket = Bucket === bucket ? bucket.name : bucket
+        req = RpbGetReq.new(:bucket => bucket, :key => key)
+        req.r = normalize_quorum_value(r) if r
+        write_protobuff(:GetReq, req)
+        decode_response(RObject.new(client.bucket(bucket), key))
+      end
+
+      def reload_object(robject, r=nil)
+        req = RpbGetReq.new(:bucket => robject.bucket.name, :key => robject.key)
+        req.r = normalize_quorum_value(r) if r
+        write_protobuff(:GetReq, req)
+        decode_response(robject)
+      end
+
+      def store_object(robject, returnbody=false, w=nil, dw=nil)
+        if robject.prevent_stale_writes
+          other = fetch_object(robject.bucket, robject.key)
+          raise Riak::ProtobuffsFailedRequest(:stale_object, t("stale_write_prevented")) unless other.vclock == robject.vclock
+        end
+        req = dump_object(robject)
+        req.w = normalize_quorum_value(w) if w
+        req.dw = normalize_quorum_value(dw) if dw
+        req.return_body = returnbody
+        write_protobuff(:PutReq, req)
+        decode_response(robject)
+      end
+
+      def delete_object(bucket, key, rw=nil)
+        bucket = Bucket === bucket ? bucket.name : bucket
+        req = RpbDelReq.new(:bucket => bucket, :key => key)
+        req.rw = normalize_quorum_value(rw) if rw
+        write_protobuff(:DelReq, req)
+        decode_response
+      end
+
+      def get_bucket_props(bucket)
+        bucket = bucket.name if Bucket === bucket
+        req = RpbGetBucketReq.new(:bucket => bucket)
+        write_protobuff(:GetBucketReq, req)
+        decode_response
+      end
+
+      def set_bucket_props(bucket, props)
+        bucket = bucket.name if Bucket === bucket
+        props = props.slice('n_val', 'allow_mult')
+        req = RpbSetBucketReq.new(:bucket => bucket, :props => RpbBucketProps.new(props))
+        write_protobuff(:SetBucketReq, req)
+        decode_response
+      end
+
+      def list_keys(bucket, &block)
+        bucket = bucket.name if Bucket === bucket
+        req = RpbListKeysReq.new(:bucket => bucket)
+        write_protobuff(:ListKeysReq, req)
+        keys = []
+        pump = Pump.new(block) if block_given?
+        while msg = decode_response
+          break if msg.done
+          if pump
+            pump.pump msg.keys
+          else
+            keys += msg.keys
+          end
+        end
+        block_given? || keys
+      end
+
+      def mapred(mr, &block)
+        req = RpbMapRedReq.new(:request => mr.to_json, :content_type => "application/json")
+        write_protobuff(:MapRedReq, req)
+        results = []
+        pump = Pump.new(lambda do |msg|
+          block.call msg.phase, JSON.parse(msg.response)
+        end) if block_given?
+        while msg = decode_response
+          break if msg.done
+          if pump
+            pump.pump msg
+          else
+            results[msg.phase] ||= []
+            results[msg.phase] += JSON.parse(msg.response)
+          end
+        end
+        block_given? || results.size == 1 ? results.first : results
+      end
+
+      private
+      def write_protobuff(code, message)
+        encoded = message.encode
+        socket.write([encoded.length+1, MESSAGE_CODES.index(code)].pack("NC"))
+        socket.write(encoded)
+      end
+
+      def decode_response(*args)
+        header = socket.read(5)
+        raise SocketError, "Unexpected EOF on PBC socket" if header.nil?
+        msglen, msgcode = header.unpack("NC")
+        if msglen == 1
+          case MESSAGE_CODES[msgcode]
+          when :PingResp, :SetClientIdResp, :PutResp, :DelResp, :SetBucketResp
+            true
+          when :ListBucketsResp, :ListKeysResp
+            []
+          when :GetResp
+            raise Riak::ProtobuffsFailedRequest.new(:not_found, t('not_found'))
+          else
+            false
+          end
+        else
+          message = socket.read(msglen-1)
+          case MESSAGE_CODES[msgcode]
+          when :ErrorResp
+            res = RpbErrorResp.decode(message)
+            raise Riak::ProtobuffsFailedRequest.new(res.errcode, res.errmsg)
+          when :GetClientIdResp
+            res = RpbGetClientIdResp.decode(message)
+            res.client_id
+          when :GetServerInfoResp
+            res = RpbGetServerInfoResp.decode(message)
+            {:node => res.node, :server_version => res.server_version}
+          when :GetResp, :PutResp
+            res = RpbGetResp.decode(message)
+            load_object(res, args.first)
+          when :ListBucketsResp
+            res = RpbListBucketsResp.decode(message)
+            res.buckets
+          when :ListKeysResp
+            RpbListKeysResp.decode(message)
+          when :GetBucketResp
+            res = RpbGetBucketResp.decode(message)
+            {'n_val' => res.props.n_val, 'allow_mult' => res.props.allow_mult}
+          when :MapRedResp
+            RpbMapRedResp.decode(message)
+          end
+        end
+      rescue SocketError => e
+        reset_socket
+        raise Riak::ProtobuffsFailedRequest.new(:server_error, e.message)
+      end
+    end
+  end
+end

data/lib/riak/client/curb_backend.rb

@@ -12,12 +12,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 require 'riak'
-
-begin
-  require 'fiber'
-rescue LoadError
-  require 'riak/util/fiber1.8'
-end
+require 'riak/client/pump'
 
 module Riak
   class Client
@@ -34,20 +29,13 @@ module Riak
       end
 
       private
-      def perform(method, uri, headers, expect, data=nil)
+      def perform(method, uri, headers, expect, data=nil, &block)
         # Setup
         curl.headers = RequestHeaders.new(headers).to_a
         curl.url = uri.to_s
         response_headers.initialize_http_header(nil)
         if block_given?
-          _curl = curl
-          Fiber.new {
-            f = Fiber.current
-            _curl.on_body {|chunk| f.resume(chunk); chunk.size }
-            loop do
-              yield Fiber.yield
-            end
-          }.resume
+          curl.on_body(&Pump.new(block))
         else
           curl.on_body
         end
@@ -74,12 +62,14 @@ module Riak
           end
           result
         else
-          raise FailedRequest.new(method, expect, curl.response_code, response_headers.to_hash, curl.body_str)
+          raise HTTPFailedRequest.new(method, expect, curl.response_code, response_headers.to_hash, curl.body_str)
         end
       end
 
       def curl
         Thread.current[:curl_easy_handle] ||= Curl::Easy.new.tap do |c|
+          configure_ssl(c) if @client.ssl_enabled?
+
           c.follow_location = false
           c.on_header do |header_line|
             response_headers.parse(header_line)
@@ -87,6 +77,10 @@ module Riak
           end
         end
       end
+
+      def configure_ssl(curl)
+        curl.ssl_verify_peer = @client.ssl_options[:verify_mode] == "peer"
+      end
     end
   end
 end

data/lib/riak/client/excon_backend.rb

@@ -12,22 +12,18 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 require 'riak'
+require 'riak/client/pump'
 
 module Riak
   class Client
     # An HTTP backend for Riak::Client that uses Wesley Beary's Excon
-    # HTTP library. Comforms to the Riak::Client::HTTPBackend
+    # HTTP library. Conforms to the Riak::Client::HTTPBackend
     # interface.
     class ExconBackend < HTTPBackend
       def self.configured?
         begin
-          begin
-            require 'fiber'
-          rescue LoadError
-            require 'riak/util/fiber1.8'
-          end
           require 'excon'
-          Excon::VERSION >= "0.3.4"
+          Excon::VERSION >= "0.5.7"
         rescue LoadError
           false
         end
@@ -35,6 +31,8 @@ module Riak
 
       private
       def perform(method, uri, headers, expect, data=nil, &block)
+        configure_ssl if @client.ssl_enabled?
+
         params = {
           :method => method.to_s.upcase,
           :headers => RequestHeaders.new(headers).to_hash,
@@ -43,16 +41,9 @@ module Riak
         params[:query] = uri.query if uri.query
         params[:body] = data if [:put,:post].include?(method)
         params[:idempotent] = (method != :post)
-        if block_given?
-          passed_block = block
-          Fiber.new {
-            f = Fiber.current
-            block = lambda {|chunk| f.resume(chunk) }
-            loop do
-              passed_block.call Fiber.yield
-            end
-          }.resume
-        end
+
+        block = Pump.new(block) if block_given?
+
         response = connection.request(params, &block)
         if valid_response?(expect, response.status)
           response_headers.initialize_http_header(response.headers)
@@ -62,13 +53,18 @@ module Riak
           end
           result
         else
-          raise FailedRequest.new(method, expect, response.status, response.headers, response.body)
+          raise HTTPFailedRequest.new(method, expect, response.status, response.headers, response.body)
         end
       end
 
       def connection
         @connection ||= Excon::Connection.new(root_uri.to_s)
       end
+
+      def configure_ssl
+        Excon.ssl_verify_peer = @client.ssl_options[:verify_mode].to_s === "peer"
+        Excon.ssl_ca_path     = @client.ssl_options[:ca_path] if @client.ssl_options[:ca_path]
+      end
     end
   end
 end

data/lib/riak/client/http_backend.rb

@@ -164,20 +164,20 @@ module Riak
       # @return [Array<Object>] the list of results, if no block was
       #        given
       def mapred(mr)
-        response = post(200, riak_kv_wm_mapred, mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"})
-        data = begin
-                 JSON.parse(response[:body])
-               rescue
-                 response
-               end
-        # This fakes streaming until the streaming MIME parser works.
         if block_given?
-          data = [data] if mr.query.count {|p| p.keep } == 1
-          data.each_with_index do |phase, idx|
-            phase.each {|obj| yield idx, obj }
+          parser = Riak::Util::Multipart::StreamParser.new do |response|
+            result = JSON.parse(response[:body])
+            yield result['phase'], result['data']
           end
+          post(200, riak_kv_wm_mapred, {:chunked => true}, mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"}, &parser)
+          nil
         else
-          data
+          response = post(200, riak_kv_wm_mapred, mr.to_json, {"Content-Type" => "application/json", "Accept" => "application/json"})
+          begin
+            JSON.parse(response[:body])
+          rescue
+            response
+          end
         end
       end
 
@@ -200,8 +200,8 @@ module Riak
           Util::Multipart.parse(response[:body], boundary).map do |group|
             group.map do |obj|
               if obj[:headers] && obj[:body] && obj[:headers]['location']
-                bucket, key = $1, $2 if obj[:headers]['location'].first =~ %r{/.*/(.*)/(.*)$}
-                load_object(RObject.new(client.bucket(bucket), key), obj)
+                bucket = $1 if obj[:headers]['location'].first =~ %r{/.*/(.*)/.*$}
+                load_object(RObject.new(client.bucket(bucket), nil), obj)
               end
             end
           end