riakpb 0.1.0

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.0.1 2010-05-21
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,30 @@
+History.txt
+Manifest.txt
+README.rdoc
+Rakefile
+lib/riak.rb
+lib/riak/client.rb
+lib/riak/client/rpc.rb
+lib/riak/client_pb.rb
+lib/riak/bucket.rb
+lib/riak/key.rb
+lib/riak/riak_content.rb
+lib/riak/i18n.rb
+lib/riak/locale/en.yml
+lib/riak/util/decode.rb
+lib/riak/util/encode.rb
+lib/riak/util/message_code.rb
+lib/riak/util/translation.rb
+lib/riak/failed_exchange.rb
+lib/riak/failed_request.rb
+lib/riak/sibling_error.rb
+script/console
+script/destroy
+script/generate
+spec/spec_helper.rb
+spec/riak/client_spec.rb
+spec/riak/rpc_spec.rb
+spec/riak/bucket_spec.rb
+spec/riak/key_spec.rb
+spec/riak/riak_content_spec.rb
+spec/riak/map_reduce_spec.rb

data/README.rdoc

@@ -0,0 +1,156 @@
+= riak Ruby Protocol Buffers Client
+
+* http://github.com/aitrus/riak-pbclient
+
+== DESCRIPTION:
+
+This is a Ruby client for Riak, using protocol buffers instead of REST.  It offers some benefit in terms of speed and it abstracts Buckets/Keys differently than does the REST client.  Different != Better.
+
+== FEATURES/PROBLEMS:
+
+* It can list buckets!  Huzzah!
+* You can destroy a bucket, which basically lists its keys and iterates over them--deleting each.
+* It works, which surely counts for something.
+* It's still rough around the edges, but certainly usable.  I'm attempting to polish it, though <b>suggestions welcome</b>.
+
+== SYNOPSIS:
+
+This is a work in progress and I am current improving the rspec and the documentation, but please have fun and give me some feedback.  The below examples give you an overview on how to use this package.
+
+=== Connect to riak and ping it:
+
+  >> client = Riak::Client.new # Riak::Client.new({:host => "ubuntu01", :port => 1234})
+  => #<Riak::Client:0x0000010328a040 @host="127.0.0.1", @port=8087, @w=nil, @dw=nil, @buckets=[], @bucket_cache={}>
+  >> client.ping?
+  => true
+
+=== List Buckets within riak:
+<i>Not suggested for use in production</i>
+
+  >> client = Riak::Client.new
+  => #<Riak::Client:0x00000101439a78 @host="127.0.0.1", @port=8087, @w=nil, @dw=nil, @buckets=[], @bucket_cache={}>
+  >> client.buckets
+  => ["hb", "goog"]
+
+=== Fetch a Bucket:
+
+  >> bucket = client["goog"]
+  => #<Riak::Bucket name=goog, props={n_val=>3, allow_mult=false}>
+
+=== Destroy a Bucket:
+
+  >> bucket.destroy!
+  => true
+
+=== List Keys within the Bucket:
+
+  >> bucket = client["goog"]
+  => #<Riak::Bucket name=goog, props={n_val=>3, allow_mult=false}>
+  >> bucket.keys
+  => ["2010-05-03", "2010-03-24", "2010-03-05", "2010-02-25", <b>SNIP 80~ Lines</b>, "2005-02-10", "2005-01-04"]
+
+=== Fetch a Key:
+
+  >> key = bucket["2010-04-12"]
+  => #<Riak::Key name="2010-04-12", vclock="k\xCEa```\xCC`\xCA\x05R,\xCC\xF3{*2\x98\x12\x19\xF3X\x19N\xCF\xEB8\xCA\x97\x05\x00", contents=[#<#Riak::RiakContent value={"Date"=>Mon, 12 Apr 2010, "Open"=>"567.35", "High"=>"574.00", "Low"=>"566.22", "Close"=>"572.73", "Volume"=>"2352400", "Adj Close"=>"572.73"}, content_type="application/json", vtag="3pbhcnmUtPZvXBSCmo5ESs", links=[], last_mod=1276388043, last_mod_usecs=65345, usermeta={}>]>
+
+=== Fetch a Key's Content:
+<i>Differs from the REST client.  Key+RiakContent = ROjbect</i>
+
+  >> rcon = key.content
+  => #<#Riak::RiakContent value={"Date"=>Mon, 12 Apr 2010, "Open"=>"567.35", "High"=>"574.00", "Low"=>"566.22", "Close"=>"572.73", "Volume"=>"2352400", "Adj Close"=>"572.73"}, content_type="application/json", vtag="3pbhcnmUtPZvXBSCmo5ESs", links=[], last_mod=1276388043, last_mod_usecs=65345, usermeta={}>
+
+=== Play with the RiakContent's data and save it:
+
+  >> rcon.value
+  => {"Date"=>Mon, 12 Apr 2010, "Open"=>"567.35", "High"=>"574.00", "Low"=>"566.22", "Close"=>"572.73", "Volume"=>"2352400", "Adj Close"=>"572.73"}
+  >> rcon.value.class
+  => Hash
+  >> rcon.value["Date"]
+  => Mon, 12 Apr 2010
+  >> rcon.value["Date"] += 1.day
+  => Tue, 13 Apr 2010
+  >> rcon
+  => #<#Riak::RiakContent value={"Date"=>Tue, 13 Apr 2010, "Open"=>"567.35", "High"=>"574.00", "Low"=>"566.22", "Close"=>"572.73", "Volume"=>"2352400", "Adj Close"=>"572.73"}, content_type="application/json", vtag="3pbhcnmUtPZvXBSCmo5ESs", links=[], last_mod=1276388043, last_mod_usecs=65345, usermeta={}>
+  >> rcon.save
+  => true
+  >> key = bucket["2010-04-12"]
+  => #<Riak::Key name="2010-04-12", vclock="k\xCEa```\xCA`\xCA\x05R,\xCC\xF3{*2\x98\x12\x19\xF3X\x19N\xCF\xEB8\xCA\a\x13\xFE\x9A\xF3\x05*|x)P8\v\x00", contents=[#<#Riak::RiakContent value={"Date"=>Tue, 13 Apr 2010, "Open"=>"567.35", "High"=>"574.00", "Low"=>"566.22", "Close"=>"572.73", "Volume"=>"2352400", "Adj Close"=>"572.73"}, content_type="application/json", vtag="2wCEI09OVZEUcHozcm9XCX", links=[], last_mod=1276389827, last_mod_usecs=762316, usermeta={}>]>
+  >> rcon = key.content
+  => #<#Riak::RiakContent value={"Date"=>Tue, 13 Apr 2010, "Open"=>"567.35", "High"=>"574.00", "Low"=>"566.22", "Close"=>"572.73", "Volume"=>"2352400", "Adj Close"=>"572.73"}, content_type="application/json", vtag="2wCEI09OVZEUcHozcm9XCX", links=[], last_mod=1276389827, last_mod_usecs=762316, usermeta={}>
+  >> rcon.value["Date"] -= 1.day
+  => Mon, 12 Apr 2010
+  >> rcon.save
+  => true
+  >> key = bucket["2010-04-12"]
+  => #<Riak::Key name="2010-04-12", vclock="k\xCEa```\xCA`\xCA\x05R,\xCC\xF3{*2\x98\x12\x19\xF3X\x19N\xCF\xEB8\xCA\a\x13\xFE\x9A\xF3\x05(\xCC\x04\x14~\xBC\x14(\x9C\x05\x00", contents=[#<#Riak::RiakContent value={"Date"=>Mon, 12 Apr 2010, "Open"=>"567.35", "High"=>"574.00", "Low"=>"566.22", "Close"=>"572.73", "Volume"=>"2352400", "Adj Close"=>"572.73"}, content_type="application/json", vtag="264RmLHrCK89zvhtq4n6Fj", links=[], last_mod=1276389859, last_mod_usecs=810556, usermeta={}>]>
+  >> rcon = key.content
+  => #<#Riak::RiakContent value={"Date"=>Mon, 12 Apr 2010, "Open"=>"567.35", "High"=>"574.00", "Low"=>"566.22", "Close"=>"572.73", "Volume"=>"2352400", "Adj Close"=>"572.73"}, content_type="application/json", vtag="264RmLHrCK89zvhtq4n6Fj", links=[], last_mod=1276389859, last_mod_usecs=810556, usermeta={}>
+  >> 
+
+=== MapReduce:
+
+I'm working on ways to simplify M/R requests.  It works the same as in ripple.  Largely because it's ripped off from it, with very few changes (so far).
+
+=== Load the GOOG stocks from the riak Fast Track:
+
+  require 'riak'
+  require 'csv'
+
+  client = Riak::Client.new
+  bucket = client["goog"]
+
+  CSV.foreach('goog.csv', :headers => true) do |row|
+    puts row.first[1].to_s
+    key = bucket[row.first[1].to_s]
+    key.content.value = Hash[row.to_a]
+    key.save
+  end
+
+== REQUIREMENTS:
+
+* Rspec >= 2.0.0.beta.9
+* ActiveSupport >= 3.0.0.beta3
+* [Rakefile] hoe >= 2.1.0
+* [Rakefile] newgem
+
+== INSTALL:
+
+  git clone git://github.com/aitrus/riak-pbclient.git
+  cd riak-pbclient
+  rake gem
+  [sudo] gem install -l riak-X.Y.Z.gem
+
+== LINKS:
+
+* Riak Fast Track: https://wiki.basho.com/display/RIAK/The+Riak+Fast+Track
+* Riak Fast Track GOOG data: https://wiki.basho.com/display/RIAK/Loading+Data+and+Running+MapReduce+Queries
+* Ruby Client for Riak's REST Interface: http://github.com/seancribbs/ripple
+
+== THANKS:
+
+This is my first public open source whatever, and I appreciate the people in #riak who tolerated me.
+
+== LICENSE:
+
+Copyright 2010:
+  Scott Gonyea
+  Sean Cribbs, Sonian Inc.
+  Basho Technologies, Inc.
+
+The riak-pbclient was pieced together by Scott Gonyea.  Big chunks
+of code were lifted / refactored from the riak-client project, by
+Sean Cribbs.  Lots of pieces are my own work, but it doesn't really
+matter to me.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,43 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/riak'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'riakpb' do
+  self.developer              'Scott Gonyea', 'me@sgonyea.com'
+  self.rubyforge_name       = self.name
+  self.summary              = 'riak-pbclient is a protocol buffer client for Riak--the distributed database by Basho.'
+  self.url                  = 'http://github.com/aitrus/riak-pbclient'
+end
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+desc "Run Unit Specs Only"
+Rspec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = "spec/riak/**/*_spec.rb"
+end
+
+namespace :spec do
+  desc "Run Integration Specs Only"
+  Rspec::Core::RakeTask.new(:integration) do |spec|
+    spec.pattern = "spec/integration/**/*_spec.rb"
+  end
+
+  desc "Run All Specs"
+  Rspec::Core::RakeTask.new(:all) do |spec|
+    spec.pattern = Rake::FileList["spec/**/*_spec.rb"]
+  end
+end
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features]
+

data/lib/riak/bucket.rb

@@ -0,0 +1,177 @@
+require 'riak'
+
+module Riak
+  # Represents and encapsulates operations on a Riak bucket.  You may retrieve a bucket
+  # using {Client#bucket}, or create it manually and retrieve its meta-information later.
+  class Bucket
+    include Util::Translation
+    include Util::MessageCode
+
+    # @return [Riak::Client] the associated client
+    attr_reader :client
+
+    # @return [String] the bucket name
+    attr_reader :name
+
+    # @return [Fixnum] the number of replicas for objects in this bucket
+    attr_reader :n_val
+
+    # @return [TrueClass/FalseClass] whether or not a key's siblings are to be retrieved
+    attr_reader :allow_mult
+
+    attr_reader :key_cache
+
+    # Create a Riak bucket manually.
+    # @param [Client] client the {Riak::Client} for this bucket
+    # @param [String] name the name of the bucket
+    def initialize(client, name, options={})
+      options.assert_valid_keys(:n_val, :allow_mult)
+      raise ArgumentError, t("client_type", :client => client.inspect)  unless client.is_a?(Client)
+      raise ArgumentError, t("string_type", :string => name.inspect)    unless name.is_a?(String)
+
+      @client     = client
+      @name       = name
+      @n_val      = options[:n_val]       || nil
+      @allow_mult = options[:allow_mult]  || nil
+      @key_cache  = Hash.new{|k,v| k[v] = Riak::Key.new(self, v)}
+    end
+
+    # Load information for the bucket from a response given by the {Riak::Client::HTTPBackend}.
+    # Used mostly internally - use {Riak::Client#bucket} to get a {Bucket} instance.
+    # @param [RpbHash] response a response from {Riak::Client::HTTPBackend}
+    # @return [Bucket] self
+    # @see Client#bucket
+    def load(response)
+      raise ArgumentError, t("response_type") unless response.is_a?(Riak::RpbGetBucketResp)
+
+      self.n_val      = response.props.n_val
+      self.allow_mult = response.props.allow_mult
+
+      return(self)
+    end
+
+    # Accesses or retrieves a list of keys in this bucket.  Needs to have expiration / cacheing, though not now.
+    # @return [Array<String>] Keys in this bucket
+    def keys
+      @keys ||= @client.keys_in @name
+    end
+
+    # Accesses or retrieves a list of keys in this bucket.  Needs to have expiration / cacheing, though not now.
+    # @return [Array<String>] Keys in this bucket
+    def keys!
+      @keys = @client.keys_in @name
+    end
+
+    # Retrieve an object from within the bucket.
+    # @param [String] key the key of the object to retrieve
+    # @param [Fixnum] r - the read quorum for the request - how many nodes should concur on the read
+    # @return [Riak::Key] the object
+    def key(key, options={})
+      raise ArgumentError, t("fixnum_invalid", :num => options[:r]) unless options[:r].is_a?(NilClass) or options[:r].is_a?(Fixnum)
+      raise ArgumentError, t("string_invalid", :string => key)      unless key.is_a?(String)
+
+      if options[:safely] == true and not @key_cache[key].empty?
+        return(@key_cache[key])
+      end
+
+      response = @client.get_request @name, key, options[:r]
+
+      @key_cache[key].load(response)
+    end
+    alias :[] :key
+
+    # Retrieve an object from within the bucket.  Will raise an error message if key does not exist.
+    # @param [String] key the key of the object to retrieve
+    # @param [Fixnum] quorum - the read quorum for the request - how many nodes should concur on the read
+    # @return [Riak::Key] the object
+    def key!(key, r=nil)
+      raise ArgumentError, t("string_invalid", :string  => key) unless key.is_a?(String)
+      raise ArgumentError, t("fixnum_invalid", :num     => r)   unless r.is_a?(Fixnum)
+
+      response = @client.get_request @name, key, r
+
+      Riak::Key.new(self, key).load!(response)
+    end
+
+    # Retrieves a Key from the given Bucket. Originally written for link retrieval.
+    # @param [String] bucket the name of the bucket, in which the Key is contained
+    # @param [String] key the name of the key to retrieve
+    # @option options [Fixnum] :quorum read quorum- num of replicas need to agree when retrieving the object
+    # @return [Riak::Key] the object
+    def get_linked(bucket, key, options=nil)
+      @client[bucket].key(key, options)
+    end
+
+    # Retrieves a Key from the given Bucket. Originally written for link retrieval.
+    # Inserts a key in this bucket's namespace, into riak.
+    # @option options [Fixnum] :w (write quorum) how many replicas to write to before returning a successful response.
+    # @option options [Fixnum :dw how many replicas to commit to durable storage before returning a successful response.
+    # @option options [Boolean] :return_body whether to return the contents of the stored object.
+    # @return [RpbPutResp] the response confirming Key storage and (optionally) the Key's updated/new data.
+    def store(options)
+      options[:bucket] = @name
+      @client.put_request(options)
+    end
+
+    # Deletes a key from the bucket
+    # @param [String] key the key to delete
+    # @param [Hash] options quorum options
+    # @option options [Fixnum] :rw - the read/write quorum for the delete
+    def delete(key, rw=nil)
+      key = key.name if key.is_a?(Riak::Key)
+
+      @client.del_request(@name, key, rw)
+    end
+
+    # Wipes out all keys stored in the bucket, as of execution
+    # @param [String] key the key to delete
+    # @param [Hash] options quorum options
+    # @option options [Fixnum] :rw - the read/write quorum for the delete
+    def destroy!(rw=nil)
+      keys!
+      
+      @keys.each do |key|
+        @client.del_request(@name, key, rw)
+      end
+#      super.destroy
+    end
+
+    # @return [true, false] whether the bucket allows divergent siblings
+    def allow_mult
+      @allow_mult
+    end
+
+    # Set the allow_mult property.  *NOTE* This will result in a PUT request to Riak.
+    # @param [true, false] value whether the bucket should allow siblings
+    def allow_mult=(value)
+      return(@allow_mult = value) if value.is_a?(TrueClass) or value.is_a?(FalseClass)
+
+      raise ArgumentError, t("boolean_type")
+    end
+
+    # @return [Fixnum] the N value, or number of replicas for this bucket
+    def n_val
+      @n_val
+    end
+
+    # Set the N value (number of replicas).
+    # Saving this value after the bucket has objects stored in it may have unpredictable results.
+    # @param [Fixnum] value the number of replicas the bucket should keep of each object
+    def n_val=(value)
+      raise ArgumentError, t("fixnum_type", :value => value) unless value.is_a?(Fixnum)
+
+      @n_val = value
+    end
+
+    # @return [String] a representation suitable for IRB and debugging output
+    def inspect
+      "#<Riak::Bucket name=#{@name}, props={n_val=>#{@n_val}, allow_mult=#{@allow_mult}}>"
+    end
+
+    # @return [String] a representation suitable for IRB and debugging output, including keys within this bucket
+    def inspect!
+      "#<Riak::Bucket name=#{@name}, props={n_val=>#{@n_val}, allow_mult=#{@allow_mult}}, keys=#{keys.inspect}>"
+    end
+
+  end
+end

data/lib/riak/client/rpc.rb

@@ -0,0 +1,127 @@
+# Special thanks to:
+#
+# - Radar in #ruby-lang
+# - xnotdotorg / arilerner[at]gmail.com - 
+#     http://blog.xnot.org/2008/11/16/communicating-from-ruby-to-erlang/
+#
+require 'socket'
+
+module Riak
+  class Client
+    class Rpc
+      include Riak::Util::MessageCode
+      include Riak::Util::Translation
+      include Riak::Util::Encode
+      include Riak::Util::Decode
+
+      RECV_LIMIT=1638400
+
+      attr_reader :req_message, :response, :resp_message_codes, :resp_message, :status
+
+      # Establishes a Client ID with the Riak node, for the life of the RPC connection.
+      # @param [Client] the Riak::Client object in which this Rpc instance lives
+      def initialize(client)
+        @status             = false
+        @client             = client
+        @client_id          = request(Util::MessageCode::GET_CLIENT_ID_REQUEST).client_id
+        @set_client_id      = Riak::RpbSetClientIdReq.new(:client_id => @client_id)
+
+        # Request / Response Data
+        @resp_message_codes = -1
+        @resp_message       = ''
+        @req_message_code   = -1
+        @req_message        = ''
+        @response           = ''
+      end
+
+      # Clears the request / response data, in preparation for a new request
+      def clear
+        @resp_message_codes = -1
+        @resp_message       = ''
+        @req_message_code   = -1
+        @req_message        = ''
+        @response           = ''
+        @status             = false
+      end
+
+      # Opens a TCPSocket connection with the riak host/node
+      # @yield [TCPSocket] hands off the socket connection
+      # @return [TCPSocket] data that was exchanged with the host/node
+      def with_socket(&block)
+        socket              = TCPSocket.open(@client.host, @client.port)
+        set_client_id(socket) if @set_client_id
+
+        out                 = yield(socket)
+        socket.close
+
+        return(out)
+      end
+
+      # Sets the Client ID for the TCPSocket session
+      # @param [TCPSocket] socket connection for which the Client ID will be set
+      # @return [True/False] whether or not the set client id request succeeded
+      def set_client_id(socket)
+        @set_c_id_req     ||= assemble_request( Util::MessageCode::SET_CLIENT_ID_REQUEST,
+                                                @set_client_id.serialize_to_string)
+
+        socket.send(@set_c_id_req, 0)
+        set_c_id_resp       = socket.recv(RECV_LIMIT)
+
+        resp_code, resp_msg = decode_message(set_c_id_resp)
+
+        return(resp_code == Util::MessageCode::SET_CLIENT_ID_RESPONSE)
+      end
+      
+      # Sends the request to the riak node
+      # @param [Fixnum] mc The message code that identifies the request
+      # @param [Protobuf::Message] pb_msg The protobuf message, if applicable, for the message code
+      # @return [True/False] whether or not the set client id request succeeded
+      def request(mc, pb_msg=nil)
+        clear
+
+        @req_message_code = mc
+        @response         = RESPONSE_CLASS_FOR[mc].new unless RESPONSE_CLASS_FOR[mc].nil?
+
+        with_socket do |socket|
+          begin
+            @req_message  = assemble_request mc, (pb_msg.serialize_to_string rescue '')
+
+            socket.send(@req_message, 0)
+            self.response = socket.recv(RECV_LIMIT)
+
+          end while(false == (@response.done rescue true))
+        end # with_socket
+
+        return(@response)
+      end # stream_request
+
+      # Handles the response from the Riak node
+      # @param [String] value The message returned from the Riak node over the TCP Socket
+      # @return [Protobuf::Message] @response the processed response (if any) from the Riak node
+      def response=(value)
+        @resp_message = value
+
+        response_chunk, @resp_message_codes = decode_message(value)
+
+        @resp_message_codes.each do |resp_mc|
+          if resp_mc.equal?(ERROR_RESPONSE)
+            raise FailedRequest.new(MC_RESPONSE_FOR[@req_message_code], @resp_message_codes, response_chunk)
+          end
+
+          # The below should never really happen
+          if resp_mc != MC_RESPONSE_FOR[@req_message_code]
+            raise FailedExchange.new(MC_RESPONSE_FOR[@req_message_code], @resp_message_codes, response_chunk, "failed_request")
+          end
+        end
+        
+        if response_chunk.size > 0
+          @response.parse_from_string response_chunk
+        end
+
+        @status = true
+        return(@response)
+      end
+
+    end # class Client
+  end # module Rpc
+end # module RiakPbclient