riak-client 1.4.5 → 2.0.0.rc1

data/Rakefile

@@ -1,5 +1,6 @@
 require 'rubygems'
 require 'rubygems/package_task'
+require 'yard'
 require 'rspec/core'
 require 'rspec/core/rake_task'
 
@@ -12,6 +13,17 @@ Gem::PackageTask.new(gemspec) do |pkg|
   pkg.need_tar = false
 end
 
+YARD::Rake::YardocTask.new :doc do |doc|
+  doc.options = ["--markup markdown",
+                 "--markup-provider=redcarpet",
+                 "--charset utf-8",
+                 '-',
+                 'lib/**/*.rb',
+                 '*.md',
+                 '*.markdown'
+                 ].map{|e| e.split(' ')}.flatten
+end
+
 task :gem => :gemspec
 
 desc %{Validate the gemspec file.}
@@ -70,3 +82,36 @@ RSpec::Core::RakeTask.new(:ci) do |spec|
   spec.rspec_opts = %w[--profile]
 end
 task :default => :ci
+
+
+desc "Generate Protocol Buffers message definitions from riak_pb"
+task :pb_defs => 'beefcake:pb_defs'
+
+namespace :beefcake do
+  task :pb_defs => 'lib/riak/client/beefcake/messages.rb'
+
+  PROTO_FILES = %w{riak_kv riak_search riak_yokozuna riak_dt}
+  PROTO_TMP = PROTO_FILES.map{|f| "tmp/#{f}.pb.rb"}
+  
+  task :clean do
+    sh "rm -rf tmp/riak_pb"
+    sh "rm -rf #{PROTO_TMP.join ' '}"
+  end
+
+  
+  file 'lib/riak/client/beefcake/messages.rb' => PROTO_TMP do |t|
+    sh "cat lib/riak/client/beefcake/header tmp/riak.pb.rb #{t.prerequisites.join ' '} lib/riak/client/beefcake/footer > #{t.name}"
+  end
+
+  PROTO_FILES.each do |f|
+    file "tmp/#{f}.pb.rb" => 'tmp/riak_pb' do |t|
+      sh "protoc --beefcake_out tmp -I tmp/riak_pb/src tmp/riak_pb/src/#{f}.proto"
+    end
+  end
+
+  directory 'tmp/riak_pb' do
+    cd 'tmp' do
+      sh "git clone -b develop https://github.com/basho/riak_pb.git"
+    end
+  end
+end

data/lib/riak.rb

@@ -3,7 +3,7 @@ require 'riak/core_ext'
 require 'riak/client'
 require 'riak/map_reduce'
 require 'riak/util/translation'
-require 'riak/instrumentation'
+require 'riak/crdt'
 
 # The Riak module contains all aspects of the client interface to
 # Riak.

data/lib/riak/bucket.rb

@@ -2,7 +2,7 @@ require 'riak/util/translation'
 require 'riak/client'
 require 'riak/robject'
 require 'riak/counter'
-require 'riak/failed_request'
+require 'riak/errors/failed_request'
 
 module Riak
   # Represents and encapsulates operations on a Riak bucket.  You may retrieve a bucket
@@ -101,7 +101,7 @@ module Riak
     alias :[] :get
 
     # Retrieve multiple keys from this bucket.
-    # @param [Array<String>] array of keys to fetch
+    # @param [Array<String>] keys array of keys to fetch
     # @return [Hash<String, Riak::RObject>] hash of keys to objects
     def get_many(keys)
       pairs = keys.map{|k| [self, k]}

data/lib/riak/client.rb

@@ -4,13 +4,11 @@ require 'innertube'
 require 'riak'
 require 'riak/util/translation'
 require 'riak/util/escape'
-require 'riak/failed_request'
+require 'riak/errors/failed_request'
 require 'riak/client/decaying'
 require 'riak/client/node'
 require 'riak/client/search'
-require 'riak/client/http_backend'
-require 'riak/client/net_http_backend'
-require 'riak/client/excon_backend'
+require 'riak/client/yokozuna'
 require 'riak/client/protobuffs_backend'
 require 'riak/client/beefcake_protobuffs_backend'
 require 'riak/bucket'
@@ -28,14 +26,11 @@ module Riak
     # When using integer client IDs, the exclusive upper-bound of valid values.
     MAX_CLIENT_ID = 4294967296
 
-    # Array of valid protocols
-    PROTOCOLS = %w[http https pbc]
-
     # Regexp for validating hostnames, lifted from uri.rb in Ruby 1.8.6
     HOST_REGEX = /^(?:(?:(?:[a-zA-Z\d](?:[-a-zA-Z\d]*[a-zA-Z\d])?)\.)*(?:[a-zA-Z](?:[-a-zA-Z\d]*[a-zA-Z\d])?)\.?|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|\[(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:(?:[a-fA-F\d]{1,4}:)*[a-fA-F\d]{1,4})?::(?:(?:[a-fA-F\d]{1,4}:)*(?:[a-fA-F\d]{1,4}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))?)\])$/n
 
     # Valid constructor options.
-    VALID_OPTIONS = [:protocol, :nodes, :client_id, :http_backend, :protobuffs_backend] | Node::VALID_OPTIONS
+    VALID_OPTIONS = [:nodes, :client_id, :protobuffs_backend, :authentication] | Node::VALID_OPTIONS
 
     # Network errors.
     NETWORK_ERRORS = [
@@ -52,21 +47,12 @@ module Riak
 
     Pool = ::Innertube::Pool
 
-    # @return [String] The protocol to use for the Riak endpoint
-    attr_reader :protocol
-
     # @return [Array] The set of Nodes this client can communicate with.
     attr_accessor :nodes
 
     # @return [String] The internal client ID used by Riak to route responses
     attr_reader :client_id
 
-    # @return [Symbol] The HTTP backend/client to use
-    attr_accessor :http_backend
-
-    # @return [Client::Pool] A pool of HTTP connections
-    attr_reader :http_pool
-
     # @return [Symbol] The Protocol Buffers backend/client to use
     attr_accessor :protobuffs_backend
 
@@ -76,23 +62,20 @@ module Riak
     # @return [Integer] The number of threads for multiget requests
     attr_reader :multiget_threads
 
+    # @return [Hash] The authentication information this client will use.
+    attr_reader :authentication
+
     # Creates a client connection to Riak
     # @param [Hash] options configuration options for the client
     # @option options [Array] :nodes A list of nodes this client connects to.
     #   Each element of the list is a hash which is passed to Node.new, e.g.
-    #   {host: '127.0.0.1', pb_port: 1234, ...}.
+    #   `{host: '127.0.0.1', pb_port: 1234, ...}`.
     #   If no nodes are given, a single node is constructed from the remaining
     #   options given to Client.new.
     # @option options [String] :host ('127.0.0.1') The host or IP address for the Riak endpoint
-    # @option options [String] :protocol ('http') The protocol to use for connecting to a node backend
-    # @option options [Fixnum] :http_port (8098) The port of the Riak HTTP endpoint
     # @option options [Fixnum] :pb_port (8087) The port of the Riak Protocol Buffers endpoint
-    # @option options [String] :prefix ('/riak/') The URL path prefix to the main HTTP endpoint
-    # @option options [String] :mapred ('/mapred') The path to the map-reduce HTTP endpoint
     # @option options [Fixnum, String] :client_id (rand(MAX_CLIENT_ID)) The internal client ID used by Riak to route responses
-    # @option options [String, Symbol] :http_backend (:NetHTTP) which  HTTP backend to use
     # @option options [String, Symbol] :protobuffs_backend (:Beefcake) which Protocol Buffers backend to use
-    # @option options [Boolean, Hash]  :ssl (nil) The SSL options to pass to each node or true for default options
     # @raise [ArgumentError] raised if any invalid options are given
     def initialize(options={})
       if options.include? :port
@@ -106,7 +89,7 @@ module Riak
       @nodes = (options[:nodes] || []).map do |n|
         Client::Node.new self, n
       end
-      if @nodes.empty? or options[:host] or options[:http_port] or options[:pb_port]
+      if @nodes.empty? or options[:host] or options[:pb_port]
         @nodes |= [Client::Node.new(self, options)]
       end
 
@@ -115,42 +98,21 @@ module Riak
                                   lambda { |b| b.teardown }
                                   )
 
-      @http_pool = Pool.new(
-                            method(:new_http_backend),
-                            lambda { |b| b.teardown }
-                            )
 
-      self.protocol           = options[:protocol]           || "http"
-      self.http_backend       = options[:http_backend]       || :NetHTTP
       self.protobuffs_backend = options[:protobuffs_backend] || :Beefcake
       self.client_id          = options[:client_id]          if options[:client_id]
-      self.ssl                = options[:ssl]                if options[:ssl]
       self.multiget_threads   = options[:multiget_threads]
+      @authentication         = options[:authentication] && options[:authentication].symbolize_keys
     end
 
-    # Yields a backend for operations that are protocol-independent.
-    # You can change which type of backend is used by setting the
-    # {#protocol}.
-    # @yield [HTTPBackend,ProtobuffsBackend] an appropriate client backend
-    def backend(&block)
-      case @protocol.to_s
-      when /https?/i
-        http &block
-      when /pbc/i
-        protobuffs &block
-      end
-    end
-
-    # Sets basic HTTP auth on all nodes.
-    def basic_auth=(auth)
-      @nodes.each do |node|
-        node.basic_auth = auth
-      end
-      auth
+    # Is security enabled?
+    # @return [Boolean] whether or not a secure connection is being used
+    def security?
+      !!authentication
     end
 
     # Retrieves a bucket from Riak.
-    # @param [String] bucket the bucket to retrieve
+    # @param [String] name the bucket to retrieve
     # @param [Hash] options options for retrieving the bucket
     # @option options [Boolean] :props (false) whether to retreive the bucket properties
     # @return [Bucket] the requested bucket
@@ -246,15 +208,6 @@ module Riak
       end
     end
 
-    # Deletes a file stored via the "Luwak" interface
-    # @param [String] filename the key/filename to delete
-    def delete_file(filename)
-      http do |h|
-        h.delete_file(filename)
-      end
-      true
-    end
-
     # Delete an object. See Bucket#delete
     def delete_object(bucket, key, options = {})
       backend do |b|
@@ -262,37 +215,10 @@ module Riak
       end
     end
 
-    # Checks whether a file exists in "Luwak".
-    # @param [String] key the key to check
-    # @return [true, false] whether the key exists in "Luwak"
-    def file_exists?(key)
-      http do |h|
-        h.file_exists?(key)
-      end
-    end
-    alias :file_exist? :file_exists?
-
     # Bucket properties. See Bucket#props
-    def get_bucket_props(bucket)
+    def get_bucket_props(bucket, options={  })
       backend do |b|
-        b.get_bucket_props bucket
-      end
-    end
-
-    # Retrieves a large file/IO object from Riak via the "Luwak"
-    # interface. Streams the data to a temporary file unless a block
-    # is given.
-    # @param [String] filename the key/filename for the object
-    # @return [IO, nil] the file (also having content_type and
-    #   original_filename accessors). The file will need to be
-    #   reopened to be read. nil will be returned if a block is given.
-    # @yield [chunk] stream contents of the file through the
-    #     block. Passing the block will result in nil being returned
-    #     from the method.
-    # @yieldparam [String] chunk a single chunk of the object's data
-    def get_file(filename, &block)
-      http do |h|
-        h.get_file(filename, &block)
+        b.get_bucket_props bucket, options
       end
     end
 
@@ -316,31 +242,11 @@ module Riak
       end
     end
 
-    # Yields an HTTPBackend.
-    def http(&block)
-      recover_from @http_pool, &block
-    end
-
-    # Sets the desired HTTP backend
-    def http_backend=(value)
-      @http_backend = value
-      # Shut down existing connections using the old backend
-      @http_pool.clear
-      @http_backend
-    end
-
     # @return [String] A representation suitable for IRB and debugging output.
     def inspect
       "#<Riak::Client #{nodes.inspect}>"
     end
 
-    # Link-walk.
-    def link_walk(object, specs)
-      http do |h|
-        h.link_walk object, specs
-      end
-    end
-
     # Retrieves a list of keys in the given bucket. See Bucket#keys
     def list_keys(bucket, options={}, &block)
       if block_given?
@@ -361,23 +267,6 @@ module Riak
       end
     end
 
-    # Creates a new HTTP backend.
-    # @return [HTTPBackend] An HTTP backend for a given node.
-    def new_http_backend
-      klass = self.class.const_get("#{@http_backend}Backend")
-      if klass.configured?
-        node = choose_node(
-          @nodes.select do |n|
-            n.http?
-          end
-        )
-
-        klass.new(self, node)
-      else
-        raise t('http_configuration', :backend => @http_backend)
-      end
-    end
-
     # Creates a new protocol buffers backend.
     # @return [ProtobuffsBackend] the Protocol Buffers backend for
     #    a given node.
@@ -413,6 +302,7 @@ module Riak
     def protobuffs(&block)
       recover_from @protobuffs_pool, &block
     end
+    alias :backend :protobuffs
 
     # Sets the desired Protocol Buffers backend
     def protobuffs_backend=(value)
@@ -422,33 +312,6 @@ module Riak
       @protobuffs_backend
     end
 
-    # Set the protocol of the Riak endpoint.  Value must be in the
-    # Riak::Client::PROTOCOLS array.
-    # @raise [ArgumentError] if the protocol is not in PROTOCOLS
-    # @return [String] the protocol being assigned
-    def protocol=(value)
-      unless PROTOCOLS.include?(value.to_s)
-        raise ArgumentError, t("protocol_invalid", :invalid => value, :valid => PROTOCOLS.join(', '))
-      end
-
-      #TODO
-      @backend = nil
-      @protocol = value
-
-      case value
-      when 'https'
-        nodes.each do |node|
-          node.ssl = true unless node.ssl_enabled?
-        end
-      when 'http'
-        nodes.each do |node|
-          node.ssl = false
-        end
-      end
-
-      @protocol
-    end
-
     # Takes a pool. Acquires a backend from the pool and yields it with
     # node-specific error recovery.
     def recover_from(pool)
@@ -496,27 +359,17 @@ module Riak
     end
 
     # Sets the properties on a bucket. See Bucket#props=
-    def set_bucket_props(bucket, properties)
-      # A bug in Beefcake is still giving us trouble with default booleans.
-      # Until it is resolved, we'll use the HTTP backend.
-      http do |b|
-        b.set_bucket_props(bucket, properties)
+    def set_bucket_props(bucket, properties, type=nil)
+      backend do |b|
+        b.set_bucket_props(bucket, properties, type)
       end
     end
 
     # Clears the properties on a bucket. See Bucket#clear_props
     def clear_bucket_props(bucket)
-      http do |b|
-        b.clear_bucket_props(bucket)
-      end
-    end
-
-    # Enables or disables SSL on all nodes, for HTTP backends.
-    def ssl=(value)
-      @nodes.each do |node|
-        node.ssl = value
+      backend do |b|
+        b.reset_bucket_props(bucket)
       end
-      value
     end
 
     # Exposes a {Stamp} object for use in generating unique
@@ -527,22 +380,6 @@ module Riak
       @stamp ||= Riak::Stamp.new(self)
     end
 
-    # Stores a large file/IO-like object in Riak via the "Luwak" interface.
-    # @overload store_file(filename, content_type, data)
-    #   Stores the file at the given key/filename
-    #   @param [String] filename the key/filename for the object
-    #   @param [String] content_type the MIME Content-Type for the data
-    #   @param [IO, String] data the contents of the file
-    # @overload store_file(content_type, data)
-    #   Stores the file with a server-determined key/filename
-    #   @param [String] content_type the MIME Content-Type for the data
-    #   @param [String, #read] data the contents of the file
-    # @return [String] the key/filename where the object was stored
-    def store_file(*args)
-      http do |h|
-        h.store_file(*args)
-      end
-    end
 
     # Stores an object in Riak.
     def store_object(object, options = {})
@@ -568,15 +405,5 @@ module Riak
         n.ssl_disable
       end
     end
-
-    # @private
-    class LuwakFile < DelegateClass(Tempfile)
-      attr_accessor :original_filename, :content_type
-      alias :key :original_filename
-      def initialize(fn)
-        super(Tempfile.new(fn))
-        @original_filename = fn
-      end
-    end
   end
 end

data/lib/riak/client/beefcake/crdt_loader.rb

@@ -0,0 +1,127 @@
+module Riak
+  class Client
+    class BeefcakeProtobuffsBackend
+
+      # Returns a new {CrdtLoader} for deserializing a protobuffs response full
+      # of CRDTs.
+      # @api private
+      def crdt_loader
+        return CrdtLoader.new self
+      end
+      
+      # Loads, and deserializes CRDTs from protobuffs into Ruby hashes,
+      # sets, strings, and integers.
+      # @api private
+      class CrdtLoader
+        include Util::Translation
+
+        attr_reader :backend, :context
+
+        def initialize(backend)
+          @backend = backend
+        end
+
+        # Perform the protobuffs request and return a deserialized CRDT.
+        def load(bucket, key, bucket_type, options={})
+          bucket = bucket.name if bucket.is_a? ::Riak::Bucket
+          fetch_args = options.merge(
+                                     bucket: bucket,
+                                     key: key,
+                                     type: bucket_type
+                                     )
+          request = DtFetchReq.new fetch_args
+
+          response = backend.protocol do |p|
+            p.write :DtFetchReq, request
+            p.expect :DtFetchResp, DtFetchResp
+          end
+
+          @context = response.context
+          rubyfy response
+        end
+
+        private
+        # Convert the protobuffs response into low-level Ruby objects.
+        def rubyfy(response)
+          return nil_rubyfy(response.type) if response.value.nil?
+          case response.type
+          when DtFetchResp::DataType::COUNTER
+            response.value.counter_value
+          when DtFetchResp::DataType::SET
+            ::Set.new response.value.set_value
+          when DtFetchResp::DataType::MAP
+            rubyfy_map response.value.map_value
+          end
+        end
+
+        # Convert a top-level map into a Ruby {Hash} of hashes.
+        def rubyfy_map(map_value)
+          accum = {
+            counters: {},
+            flags: {},
+            maps: {},
+            registers: {},
+            sets: {}
+          }
+
+          rubyfy_map_contents map_value, accum
+        end
+
+        # Convert a map inside another map into a Ruby {Hash}.
+        def rubyfy_inner_map(accum, map_value)
+          destination = accum[:maps][map_value.field.name]
+          if destination.nil?
+            destination = accum[:maps][map_value.field.name] = {
+              counters: {},
+              flags: {},
+              maps: {},
+              registers: {},
+              sets: {}
+            }
+          end
+          
+          rubyfy_map_contents map_value.map_value, destination
+        end
+
+        # Load the contents of a map into Ruby hashes.
+        def rubyfy_map_contents(map_value, destination)
+          return destination if map_value.nil?
+          map_value.each do |inner_mv|
+            case inner_mv.field.type
+            when MapField::MapFieldType::COUNTER
+              destination[:counters][inner_mv.field.name] = inner_mv.counter_value
+            when MapField::MapFieldType::FLAG
+              destination[:flags][inner_mv.field.name] = inner_mv.flag_value
+            when MapField::MapFieldType::MAP
+              rubyfy_inner_map destination, inner_mv
+            when MapField::MapFieldType::REGISTER
+              destination[:registers][inner_mv.field.name] = inner_mv.register_value
+            when MapField::MapFieldType::SET
+              destination[:sets][inner_mv.field.name] = ::Set.new inner_mv.set_value
+            end
+          end
+
+          return destination
+        end
+
+        # Sometimes a CRDT is empty, provide a sane default.
+        def nil_rubyfy(type)
+          case type
+          when DtFetchResp::DataType::COUNTER
+            0
+          when DtFetchResp::DataType::SET
+            ::Set.new
+          when DtFetchResp::DataType::MAP
+            {
+              counters: {},
+              flags: {},
+              maps: {},
+              registers: {},
+              sets: {},
+            }
+          end
+        end
+      end
+    end
+  end
+end