riak-client 0.9.0.beta → 0.9.0.beta2

data/lib/riak/client/http_backend/object_methods.rb

@@ -54,7 +54,7 @@ module Riak
         # Load object data from an HTTP response
         # @param [Hash] response a response from {Riak::Client::HTTPBackend}
         def load_object(robject, response)
-          extract_header(robject, response, "location", :key) {|v| URI.unescape(v.split("/").last) }
+          extract_header(robject, response, "location", :key) {|v| URI.unescape(v.match(%r{.*/(.*?)(\?.*)?$})[1]) }
           extract_header(robject, response, "content-type", :content_type)
           extract_header(robject, response, "x-riak-vclock", :vclock)
           extract_header(robject, response, "link", :links) {|v| Set.new(Link.parse(v)) }

data/lib/riak/client/http_backend/transport_methods.rb

@@ -138,12 +138,16 @@ module Riak
           {
             "Accept" => "multipart/mixed, application/json;q=0.7, */*;q=0.5",
             "X-Riak-ClientId" => @client.client_id
-          }
+          }.merge(basic_auth_header)
+        end
+
+        def basic_auth_header
+          @client.basic_auth ? {"Authorization" => "Basic #{Base64::encode64(@client.basic_auth)}"} : {}
         end
 
         # @return [URI] The calculated root URI for the Riak HTTP endpoint
         def root_uri
-          URI.parse("http://#{client.host}:#{client.port}")
+          URI.parse("#{client.protocol}://#{client.host}:#{client.port}")
         end
 
         # Calculates an absolute URI from a relative path specification

data/lib/riak/client/net_http_backend.rb

@@ -16,8 +16,6 @@ require 'riak'
 module Riak
   class Client
     # Uses the Ruby standard library Net::HTTP to connect to Riak.
-    # We recommend using the CurbBackend, which will
-    # be preferred when the 'curb' library is available.
     # Conforms to the Riak::Client::HTTPBackend interface.
     class NetHTTPBackend < HTTPBackend
       def self.configured?
@@ -31,30 +29,45 @@ module Riak
 
       private
       def perform(method, uri, headers, expect, data=nil) #:nodoc:
-        Net::HTTP.start(uri.host, uri.port) do |http|
-          request = Net::HTTP.const_get(method.to_s.capitalize).new(uri.request_uri, headers)
-          case data
-          when String
-            request.body = data
-          when IO
-            request.body_stream = data
-          end
+        http = Net::HTTP.new(uri.host, uri.port)
+        configure_ssl(http) if @client.ssl_enabled?
+
+        request = Net::HTTP.const_get(method.to_s.capitalize).new(uri.request_uri, headers)
+        case data
+        when String
+          request.body = data
+        when IO
+          request.body_stream = data
+        end
 
-          {}.tap do |result|
-            http.request(request) do |response|
-              if valid_response?(expect, response.code)
-                result.merge!({:headers => response.to_hash, :code => response.code.to_i})
-                response.read_body {|chunk| yield chunk } if block_given?
-                if return_body?(method, response.code, block_given?)
-                  result[:body] = response.body
-                end
-              else
-                raise FailedRequest.new(method, expect, response.code.to_i, response.to_hash, response.body)
+        {}.tap do |result|
+          http.request(request) do |response|
+            if valid_response?(expect, response.code)
+              result.merge!({:headers => response.to_hash, :code => response.code.to_i})
+              response.read_body {|chunk| yield chunk } if block_given?
+              if return_body?(method, response.code, block_given?)
+                result[:body] = response.body
               end
+            else
+              raise Riak::HTTPFailedRequest.new(method, expect, response.code.to_i, response.to_hash, response.body)
             end
           end
         end
       end
+
+      def configure_ssl(http)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL.const_get("VERIFY_#{@client.ssl_options[:verify_mode].upcase}")
+
+        if @client.ssl_options[:pem]
+          http.cert = OpenSSL::X510::Certificate.new(@client.ssl_options[:pem])
+          http.key  = OpenSSL::PKey::RSA.new(@client.ssl_options[:pem], @client.ssl_options[:pem_password])
+        end
+
+        http.ca_file = @client.ssl_options[:ca_file] if @client.ssl_options[:ca_file]
+
+        http.ca_path = @client.ssl_options[:ca_path] if @client.ssl_options[:ca_path]
+      end
     end
   end
 end

data/lib/riak/client/protobuffs_backend.rb

@@ -0,0 +1,103 @@
+# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+require 'riak'
+require 'socket'
+require 'base64'
+require 'digest/sha1'
+
+module Riak
+  class Client
+    class ProtobuffsBackend
+      include Util::Translation
+      
+      # Message Codes Enum
+      MESSAGE_CODES = %W[
+          ErrorResp
+          PingReq
+          PingResp
+          GetClientIdReq
+          GetClientIdResp
+          SetClientIdReq
+          SetClientIdResp
+          GetServerInfoReq
+          GetServerInfoResp
+          GetReq
+          GetResp
+          PutReq
+          PutResp
+          DelReq
+          DelResp
+          ListBucketsReq
+          ListBucketsResp
+          ListKeysReq
+          ListKeysResp
+          GetBucketReq
+          GetBucketResp
+          SetBucketReq
+          SetBucketResp
+          MapRedReq
+          MapRedResp
+       ].map {|s| s.intern }.freeze
+
+      def self.simple(method, code)
+        define_method method do
+          socket.write([1, MESSAGE_CODES.index(code)].pack('NC'))
+          decode_response
+        end
+      end
+
+      attr_accessor :client
+      def initialize(client)
+        @client = client
+      end
+
+      simple :ping,          :PingReq
+      simple :get_client_id, :GetClientIdReq
+      simple :server_info,   :GetServerInfoReq
+      simple :list_buckets,  :ListBucketsReq
+
+      private
+      # Implemented by subclasses
+      def decode_response
+        raise NotImplementedError
+      end
+
+      def socket
+        Thread.current[:riakpbc_socket] ||= TCPSocket.new(@client.host, @client.port)
+      end
+
+      def reset_socket
+        socket.close
+        Thread.current[:riakpbc_socket] = nil
+      end
+
+      UINTMAX = 0xffffffff
+      QUORUMS = {
+        "one" => UINTMAX - 1,
+        "quorum" => UINTMAX - 2,
+        "all" => UINTMAX - 3,
+        "default" => UINTMAX - 4
+      }.freeze
+
+      def normalize_quorum_value(q)
+        QUORUMS[q.to_s] || q.to_i
+      end
+
+      # This doesn't give us exactly the keygen that Riak uses, but close.
+      def generate_key
+        Base64.encode64(Digest::SHA1.digest(Socket.gethostname + Time.now.iso8601(3))).tr("+/","-_").sub(/=+\n$/,'')
+      end
+    end
+  end
+end

data/lib/riak/client/pump.rb

@@ -0,0 +1,44 @@
+# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+require 'riak'
+begin
+  require 'fiber'
+rescue LoadError
+  require 'riak/util/fiber1.8'
+end
+
+module Riak
+  class Client
+    # @private
+    class Pump      
+      def initialize(block)
+        @fiber = Fiber.new do
+          loop do
+            block.call Fiber.yield
+          end
+        end
+        @fiber.resume
+      end
+
+      def pump(input)
+        @fiber.resume input
+        input.size if input.respond_to?(:size) # for curb
+      end
+
+      def to_proc
+        method(:pump).to_proc
+      end
+    end
+  end
+end

data/lib/riak/failed_request.rb

@@ -14,10 +14,19 @@
 require 'riak'
 
 module Riak
-  # Exception raised when the expected response code from Riak
-  # fails to match the actual response code.
+  # Exception raised when receiving an unexpected client response from
+  # Riak.
   class FailedRequest < StandardError
     include Util::Translation
+
+    def initialize(message)
+      super(message ||  t('failed_request'))
+    end
+  end
+
+  # Exception raised when the expected HTTP response code from Riak
+  # fails to match the actual response code.
+  class HTTPFailedRequest < FailedRequest
     # @return [Symbol] the HTTP method, one of :head, :get, :post, :put, :delete
     attr_reader :method
     # @return [Fixnum] the expected response code
@@ -31,7 +40,53 @@ module Riak
 
     def initialize(method, expected_code, received_code, headers, body)
       @method, @expected, @code, @headers, @body = method, expected_code, received_code, headers, body
-      super t("failed_request", :expected => @expected.inspect, :code => @code, :body => @body)
+      super t("http_failed_request", :expected => @expected.inspect, :code => @code, :body => @body)
+    end
+
+    def is_json?
+      headers['content-type'].include?('application/json')
+    end
+    
+    # @return [true,false] whether the error represents a "not found" response
+    def not_found?
+      @code.to_i == 404
+    end
+
+    # @return [true,false] whether the error represents an internal
+    #   server error
+    def server_error?
+      @code.to_i == 500
+    end
+  end
+
+  # Exception raised when receiving an unexpected Protocol Buffers response from Riak
+  class ProtobuffsFailedRequest < FailedRequest
+    def initialize(code, message)
+      super t('protobuffs_failed_request', :code => code, :body => message)
+      @original_message = message
+      @not_found = code == :not_found
+      @server_error = code == :server_error
+    end
+
+    # @return [true, false] whether the error response is in JSON
+    def is_json?
+      begin
+        JSON.parse(original_message)
+        true
+      rescue
+        false
+      end
+    end
+    
+    # @return [true,false] whether the error represents a "not found" response
+    def not_found?
+      @not_found
+    end
+
+    # @return [true,false] whether the error represents an internal
+    #   server error
+    def server_error?
+      @server_error
     end
   end
 end

data/lib/riak/locale/en.yml

@@ -17,24 +17,32 @@ en:
     client_type: "invalid argument %{client} is not a Riak::Client"
     content_type_undefined: "content_type is not defined!"
     empty_map_reduce_query: "Specify one or more query phases to your MapReduce."
-    failed_request: "Expected %{expected} from Riak but received %{code}. %{body}"
+    failed_request: "Client request failed."
     filter_needs_block: "Filter %{filter} expects a block."
     filter_arity_mismatch: "Filter %{filter} expects %{expected} arguments but %{received} were given."
     hash_type: "invalid argument %{hash} is not a Hash"
     http_configuration: "The %{backend} HTTP backend cannot be used. Please check its requirements."
+    http_failed_request: "Expected %{expected} from Riak but received %{code}. %{body}"
     hostname_invalid: "host must be a valid hostname"
+    protocol_invalid: "'%{invalid}' is not a valid protocol, valid values are %{valid}"
+    invalid_basic_auth: "basic auth must be set using 'user:pass'"
     invalid_client_id: "Invalid client ID, must be a string or between 0 and %{max_id}"
     invalid_function_value: "invalid value for function: %{value}"
     invalid_phase_type: "type must be :map, :reduce, or :link"
-    loading_bucket: "while loading bucket '%{name}'"
+    invalid_options: "Invalid configuration options given."
+    loading_bucket: "while loading bucket '%{name}'"    
+    missing_block: "A block must be given."
     missing_host_and_port: "You must specify a host and port, or use the defaults of 127.0.0.1:8098"
     module_function_pair_required: "function must have two elements when an array"
+    not_found: "The requested object was not found."
     path_and_body_required: "You must supply both a resource path and a body."
     port_invalid: "port must be an integer between 0 and 65535"
-    request_body_type: "Request body must be a string or IO."
+    protobuffs_failed_request: "Expected success from Riak but received %{code}. %{body}"
+    request_body_type: "Request body must be a String or IO."
     resource_path_short: "Resource path too short"
     search_docs_require_id: "Search index documents must include the 'id' field."
     search_remove_requires_id_or_query: "Search index documents to be removed must have 'id' or 'query' keys."
+    stale_write_prevented: "Stale write prevented by client."
     stored_function_invalid: "function must have :bucket and :key when a hash"
     string_type: "invalid_argument %{string} is not a String"
     too_few_arguments: "too few arguments: %{params}"

data/lib/riak/map_reduce.rb

@@ -83,7 +83,7 @@ module Riak
         bucket = params.shift
         bucket = bucket.name if Bucket === bucket
         if Array === params.first
-          @inputs = {:bucket => escape(bucket), :filters => params.first }
+          @inputs = {:bucket => escape(bucket), :key_filters => params.first }
         else
           key = params.shift
           @inputs << params.unshift(escape(key)).unshift(escape(bucket))
@@ -171,15 +171,24 @@ module Riak
     end
 
     # Executes this map-reduce job.
-    # @return [Array<Array>] similar to link-walking, each element is
-    #   an array of results from a phase where "keep" is true. If there
-    #   is only one "keep" phase, only the results from that phase will
-    #   be returned.
+    # @overload run
+    #   Return the entire collection of results.
+    #   @return [Array<Array>] similar to link-walking, each element is
+    #     an array of results from a phase where "keep" is true. If there
+    #     is only one "keep" phase, only the results from that phase will
+    #     be returned.
+    # @overload run
+    #   Stream the results through the given block without accumulating.
+    #   @yield [phase, data] A block to stream results through
+    #   @yieldparam [Fixnum] phase the phase from which the results were
+    #          generated
+    #   @yieldparam [Array] data a list of results from the phase
+    #   @return [nil] nothing
     def run(&block)
       raise MapReduceError.new(t("empty_map_reduce_query")) if @query.empty?
       @client.backend.mapred(self, &block)
     rescue FailedRequest => fr
-      if fr.code == 500 && fr.headers['content-type'].include?("application/json")
+      if fr.server_error? && fr.is_json?
         raise MapReduceError.new(fr.body)
       else
         raise fr

data/lib/riak/map_reduce/filter_builder.rb

@@ -44,7 +44,7 @@ module Riak
         :matches => 1,
         :neq => 1,
         :eq => 1,
-        :set_member => 1,
+        :set_member => -1,
         :similar_to => 2,
         :starts_with => 1,
         :ends_with => 1
@@ -58,8 +58,8 @@ module Riak
       #    FilterBuilder.new do
       #      string_to_int
       #      AND do
-      #        greater_than_eq 50
-      #        neq 100
+      #        seq { greater_than_eq 50 }
+      #        seq { neq 100 }
       #      end
       #    end
       LOGICAL_OPERATIONS = %w{and or not}
@@ -67,7 +67,7 @@ module Riak
       FILTERS.each do |f,arity|
         class_eval <<-CODE
           def #{f}(*args)
-            raise ArgumentError.new(t("filter_arity_mismatch", :filter => :#{f}, :expected => #{arity.inspect}, :received => args.size)) unless Array(#{arity.inspect}).include?(args.size)
+            raise ArgumentError.new(t("filter_arity_mismatch", :filter => :#{f}, :expected => #{arity.inspect}, :received => args.size)) unless #{arity.inspect} == -1 || Array(#{arity.inspect}).include?(args.size)
             @filters << ([:#{f}] + args)
           end
         CODE

data/lib/riak/test_server.rb

@@ -33,7 +33,10 @@ module Riak
         :js_vm_count => 8,
         :js_max_vm_mem => 8,
         :js_thread_stack => 16,
-        :riak_kv_stat => true
+        :riak_kv_stat => true,
+        # Turn off map caching
+        :map_cache_size => 0,     # 0.14
+        :vnode_cache_entries => 0 # 0.13
       },
       :luwak => {
         :enabled => false
@@ -65,6 +68,7 @@ module Riak
       @vm_args    = options[:vm_args]
       # For synchronizing start/stop/recycle
       @mutex = Mutex.new
+      cleanup # Should prevent some errors related to unclean startup
     end
 
     # Sets up the proper scripts, configuration and directories for