riak-client 2.3.2 → 2.4.0.pre1

data/lib/riak/client/beefcake/time_series_list_operator.rb

@@ -2,11 +2,16 @@ require_relative './ts_cell_codec'
 require_relative './operator'
 
 class Riak::Client::BeefcakeProtobuffsBackend
-  def time_series_list_operator
-    TimeSeriesListOperator.new(self)
+  def time_series_list_operator(convert_timestamp)
+    TimeSeriesListOperator.new(self, convert_timestamp)
   end
 
   class TimeSeriesListOperator < Operator
+    def initialize(backend, convert_timestamp)
+      super(backend)
+      @convert_timestamp = convert_timestamp
+    end
+
     def list(table_name, block, options = {  })
       request = TsListKeysReq.new options.merge(table: table_name)
 
@@ -25,7 +30,7 @@ class Riak::Client::BeefcakeProtobuffsBackend
       backend.protocol do |p|
         p.write :TsListKeysReq, request
 
-        codec = TsCellCodec.new
+        codec = TsCellCodec.new(@convert_timestamp)
 
         while resp = p.expect(:TsListKeysResp, TsListKeysResp)
           break if resp.done

data/lib/riak/client/beefcake/time_series_query_operator.rb

@@ -2,11 +2,16 @@ require_relative './ts_cell_codec'
 require_relative './operator'
 
 class Riak::Client::BeefcakeProtobuffsBackend
-  def time_series_query_operator
-    TimeSeriesQueryOperator.new(self)
+  def time_series_query_operator(convert_timestamp)
+    TimeSeriesQueryOperator.new(self, convert_timestamp)
   end
 
   class TimeSeriesQueryOperator < Operator
+    def initialize(backend, convert_timestamp)
+      super(backend)
+      @convert_timestamp = convert_timestamp
+    end
+
     def query(base, interpolations = {  })
       interpolator = TsInterpolation.new base: base
       interpolator.interpolations = pairs_for interpolations
@@ -20,7 +25,7 @@ class Riak::Client::BeefcakeProtobuffsBackend
 
       return [] if :empty == result
 
-      codec = TsCellCodec.new
+      codec = TsCellCodec.new(@convert_timestamp)
 
       collection = Riak::TimeSeries::Collection.
                    new(result.rows.map do |row|

data/lib/riak/client/beefcake/ts_cell_codec.rb

@@ -2,6 +2,12 @@ require 'bigdecimal'
 
 class Riak::Client::BeefcakeProtobuffsBackend
   class TsCellCodec
+    attr_accessor :convert_timestamp
+
+    def initialize(convert_timestamp = false)
+      @convert_timestamp = convert_timestamp
+    end
+
     def cells_for(measures)
       measures.map{ |m| cell_for m }
     end
@@ -57,7 +63,11 @@ class Riak::Client::BeefcakeProtobuffsBackend
 
     def timestamp(cell)
       return false unless cell.timestamp_value.is_a? Integer
-      Time.at(cell.timestamp_value.to_f / 1000)
+      return cell.timestamp_value unless @convert_timestamp
+      tsv = cell.timestamp_value
+      secs = tsv / 1000
+      msec = tsv % 1000
+      Time.at(secs, msec * 1000)
     end
   end
 end

data/lib/riak/client/beefcake_protobuffs_backend.rb

@@ -31,7 +31,11 @@ module Riak
       end
 
       def protocol
-        p = Protocol.new socket
+        p = Protocol.new(
+          socket,
+          read_timeout: client.read_timeout,
+          write_timeout: client.write_timeout
+        )
         in_request = false
         result = nil
         begin
@@ -45,7 +49,12 @@ module Riak
       end
 
       def new_socket
-        BeefcakeSocket.new @node.host, @node.pb_port, authentication: client.authentication
+        BeefcakeSocket.new(
+          @node.host,
+          @node.pb_port,
+          authentication: client.authentication,
+          connect_timeout: client.connect_timeout
+        )
       end
 
       def ping
@@ -333,18 +342,18 @@ module Riak
       def get_index(bucket, index, query, query_options = {}, &block)
         return super unless pb_indexes?
         bucket = bucket.name if Bucket === bucket
-        if Range === query
-          options = {
-            :qtype => RpbIndexReq::IndexQueryType::RANGE,
-            :range_min => query.begin.to_s,
-            :range_max => query.end.to_s
-          }
-        else
-          options = {
-            :qtype => RpbIndexReq::IndexQueryType::EQ,
-            :key => query.to_s
-          }
-        end
+        options = if Range === query
+                    {
+                      :qtype => RpbIndexReq::IndexQueryType::RANGE,
+                      :range_min => query.begin.to_s,
+                      :range_max => query.end.to_s
+                    }
+                  else
+                    {
+                      :qtype => RpbIndexReq::IndexQueryType::EQ,
+                      :key => query.to_s
+                    }
+                  end
 
         options.merge!(:bucket => bucket, :index => index.to_s)
         options.merge!(query_options)

data/lib/riak/client/protobuffs_backend.rb

@@ -32,6 +32,7 @@ module Riak
 
       attr_accessor :client
       attr_accessor :node
+
       def initialize(client, node)
         @client = client
         @node = node

data/lib/riak/counter.rb

@@ -14,8 +14,8 @@ module Riak
     # @param [Bucket] bucket the {Riak::Bucket} for this counter
     # @param [String] key the name of the counter
     def initialize(bucket, key)
-      raise ArgumentError, t("bucket_type", bucket: bucket.inspect) unless bucket.is_a? Bucket
-      raise ArgumentError, t("string_type", string: key.inspect) unless key.is_a? String
+      raise ArgumentError, t('bucket_type', bucket: bucket.inspect) unless bucket.is_a? Bucket
+      raise ArgumentError, t('string_type', string: key.inspect) unless key.is_a? String
       @bucket, @key = bucket, key
       @client = bucket.client
 

data/lib/riak/rcontent.rb

@@ -2,6 +2,7 @@ require 'set'
 require 'time'
 require 'yaml'
 require 'forwardable'
+require 'riak/util/gzip'
 require 'riak/util/translation'
 require 'riak/serializers'
 
@@ -17,6 +18,9 @@ module Riak
     # @return [String] the MIME content type of the value
     attr_accessor :content_type
 
+    # @return [String] the content encoding of the object, e.g. "gzip"
+    attr_accessor :content_encoding
+
     # @return [Set<Link>] a Set of {Riak::Link} objects for relationships between this object and other resources
     attr_accessor :links
 
@@ -63,7 +67,7 @@ module Riak
     def data
       if @raw_data && !@data
         raw = @raw_data.respond_to?(:read) ? @raw_data.read : @raw_data
-        @data = deserialize(raw)
+        @data = deserialize(decompress(raw))
         @raw_data = nil
       end
       @data
@@ -86,7 +90,7 @@ module Riak
     # @return [String] raw data stored in riak for this object's key
     def raw_data
       if @data && !@raw_data
-        @raw_data = serialize(@data)
+        @raw_data = compress(serialize(@data))
         @data = nil
       end
       @raw_data
@@ -125,6 +129,26 @@ module Riak
       Serializers.deserialize(@content_type, body)
     end
 
+    # Compresses the given string using gzip if {#content_encoding} is set to "gzip".
+    # Otherwise the given string is returned as-is.
+    # This method is called internally when storing the object.
+    # @param [String] data
+    # @return [String]
+    def compress(data)
+      return data unless content_encoding == "gzip"
+      Util::Gzip.compress(data)
+    end
+
+    # Decompresses the given string using gzip if {#content_encoding} is set to "gzip".
+    # Otherwise the given string is returned as-is.
+    # This method is called internally when loading the object.
+    # @param [String] data
+    # @return [String]
+    def decompress(data)
+      return data unless content_encoding == "gzip"
+      Util::Gzip.decompress(data)
+    end
+
     # @return [String] A representation suitable for IRB and debugging output
     def inspect
       body = if @data || Serializers[content_type]
@@ -159,11 +183,10 @@ module Riak
 
     private
     def extract_if_present(hash, key, attribute = nil)
-      if hash[key].present?
-        attribute ||= key
-        value = block_given? ? yield(hash[key]) : hash[key]
-        send("#{attribute}=", value)
-      end
+      return unless hash[key].present?
+      attribute ||= key
+      value = block_given? ? yield(hash[key]) : hash[key]
+      send("#{attribute}=", value)
     end
 
     def new_index_hash

data/lib/riak/robject.rb

@@ -56,6 +56,7 @@ module Riak
     end
 
     def_delegators :content, :content_type, :content_type=,
+              :content_encoding, :content_encoding=,
               :links, :links=,
               :etag, :etag=,
               :last_modified, :last_modified=,
@@ -63,7 +64,8 @@ module Riak
               :indexes, :indexes=,
               :data, :data=,
               :raw_data, :raw_data=,
-              :deserialize, :serialize
+              :deserialize, :serialize,
+              :decompress, :compress
 
     # Attempts to resolve conflict using the registered conflict callbacks.
     #
@@ -139,8 +141,12 @@ module Riak
     # @raise [Conflict] if the object has siblings
     def store(options = {})
       fail Conflict, self if conflict?
-      fail ArgumentError, t('content_type_undefined') unless content_type.present?
-      fail ArgumentError, t('zero_length_key') if key == ''
+      raise ArgumentError, t('content_type_undefined') unless content_type.present?
+      raise ArgumentError, t('zero_length_key') if key == ''
+      # NB: key can be nil to indicate that Riak should generate one
+      unless key.nil? || key.is_a?(String)
+        raise ArgumentError, t('string_type', :string => key)
+      end
       @bucket.client.store_object(self, default(options))
       self
     end

data/lib/riak/time_series/list.rb

@@ -44,9 +44,8 @@ module Riak::TimeSeries
       potential_results = nil
 
       client.backend do |be|
-        potential_results = be.time_series_list_operator.list(table_name,
-                                                              block,
-                                                              options)
+        op = be.time_series_list_operator(client.convert_timestamp)
+        potential_results = op.list(table_name, block, options)
       end
 
       return @results = potential_results unless block_given?

data/lib/riak/time_series/query.rb

@@ -36,7 +36,8 @@ module Riak::TimeSeries
     # attribute
     def issue!
       @results = client.backend do |be|
-        be.time_series_query_operator.query(query_text, interpolations)
+        op = be.time_series_query_operator(client.convert_timestamp)
+        op.query(query_text, interpolations)
       end
     end
   end

data/lib/riak/time_series/read.rb

@@ -11,7 +11,8 @@ module Riak::TimeSeries
 
     def read!
       client.backend do |be|
-        be.time_series_get_operator.get(table_name, key)
+        op = be.time_series_get_operator(client.convert_timestamp)
+        op.get(table_name, key)
       end
     end
   end

data/lib/riak/util/gzip.rb

@@ -0,0 +1,44 @@
+require 'zlib'
+require 'stringio'
+
+module Riak
+  module Util
+    # Borrowed from ActiveSupport
+    # https://github.com/rails/rails/blob/master/activesupport/lib/active_support/gzip.rb
+    #
+    # A convenient wrapper for the zlib standard library that allows
+    # compression/decompression of strings with gzip.
+    #
+    #   gzip = Riak::Util::Gzip.compress('compress me!')
+    #   # => "\x1F\x8B\b\x00o\x8D\xCDO\x00\x03K\xCE\xCF-(J-.V\xC8MU\x04\x00R>n\x83\f\x00\x00\x00"
+    #
+    #   Riak::Util::Gzip.decompress(gzip)
+    #   # => "compress me!"
+    module Gzip
+      class Stream < StringIO
+        def initialize(*)
+          super
+          set_encoding "BINARY"
+        end
+
+        def close
+          rewind
+        end
+      end
+
+      # Decompresses a gzipped string.
+      def self.decompress(source)
+        Zlib::GzipReader.new(StringIO.new(source)).read
+      end
+
+      # Compresses a string using gzip.
+      def self.compress(source, level = Zlib::DEFAULT_COMPRESSION, strategy = Zlib::DEFAULT_STRATEGY)
+        output = Stream.new
+        gz = Zlib::GzipWriter.new(output, level, strategy)
+        gz.write(source)
+        gz.close
+        output.string
+      end
+    end
+  end
+end

data/lib/riak/version.rb

@@ -1,3 +1,3 @@
 module Riak
-  VERSION = "2.3.2"
+  VERSION = "2.4.0.pre1"
 end

data/riak-client.gemspec

@@ -10,23 +10,23 @@ Gem::Specification.new do |gem|
   gem.email = ['bryce@basho.com']
   gem.homepage = "http://github.com/basho/riak-ruby-client"
   gem.authors = ['Bryce Kerley']
-  gem.license = 'Apache 2.0'
+  gem.license = 'Apache-2.0'
 
   gem.required_ruby_version = '>= 1.9.3'
 
   # Deps
-  gem.add_development_dependency "rspec", "~> 3.0"
-  gem.add_development_dependency 'rake', '~> 10.1.1'
-  gem.add_development_dependency 'yard', '~> 0.8.7'
+  gem.add_development_dependency 'rspec', '~> 3.0'
+  gem.add_development_dependency 'rake', '~> 10.1'
+  gem.add_development_dependency 'yard', '~> 0.8'
   gem.add_development_dependency 'kramdown', '~> 1.4'
-  gem.add_development_dependency 'simplecov', '~> 0.10.0'
-  gem.add_development_dependency "instrumentable", "~> 1.1.0"
-  gem.add_development_dependency "rubocop", "~> 0.28.0"
+  gem.add_development_dependency 'simplecov', '~> 0.10'
+  gem.add_development_dependency 'instrumentable', '~> 1.1'
+  gem.add_development_dependency 'rubocop', '~> 0.40'
 
-  gem.add_runtime_dependency "i18n", ">=0.6.8"
-  gem.add_runtime_dependency "beefcake", "~> 1.1"
-  gem.add_runtime_dependency "multi_json", "~>1.0"
-  gem.add_runtime_dependency "innertube", "~>1.0.2"
+  gem.add_runtime_dependency 'i18n', '~> 0.6'
+  gem.add_runtime_dependency 'beefcake', '~> 1.1'
+  gem.add_runtime_dependency 'multi_json', '~> 1.0'
+  gem.add_runtime_dependency 'innertube', '~> 1.0'
   gem.add_runtime_dependency 'cert_validator', '~> 0.0.1'
 
   # Files

data/spec/integration/riak/encodings/kv_spec.rb

@@ -77,5 +77,11 @@ describe 'Encoding and Riak KV', integration: true, test_client: true do
 
       expect(binary_bucket.keys).to include random_binary_string
     end
+
+    it 'throws an exception when non-string used as key' do
+      expect do
+        binary_bucket.get(1234)
+      end.to raise_error ArgumentError, /is not a String/
+    end
   end
 end

data/spec/integration/riak/protobuffs/timeouts_spec.rb

@@ -0,0 +1,174 @@
+require 'socket'
+require 'spec_helper'
+
+require 'riak/client/beefcake/messages'
+require 'riak/client/beefcake/protocol'
+
+describe 'Protocol Buffers', test_client: true, integration: true do
+  describe 'timeouts' do
+    it 'raises error on connect timeout' do
+      # unroutable TEST-NET (https://tools.ietf.org/html/rfc5737)
+      config = {}
+      config[:host] = '192.0.2.0'
+      config[:pb_port] = 65535
+
+      config[:connect_timeout] = 0.0001
+      client = Riak::Client.new(config)
+
+      expect do
+        client.ping
+      end.to raise_error RuntimeError, /timed out/
+    end
+
+    it 'raises error on read timeout' do
+      ok_to_continue = false
+      quitting = false
+      port = 0
+
+      server = nil
+      thr = Thread.new do
+        server = TCPServer.new port
+        port = server.addr[1]
+        ok_to_continue = true
+        loop do
+          begin
+            Thread.start(server.accept) do |s|
+              loop do
+                p = Riak::Client::BeefcakeProtobuffsBackend::Protocol.new s
+                begin
+                  msgname, _body = p.receive
+                rescue IOError
+                  break if quitting
+                  raise
+                end
+                case msgname
+                when :PingReq
+                  sleep 0.5
+                  p.write :PingResp
+                else
+                  $stderr.puts("unknown msgname: #{msgname}")
+                end
+              end
+            end
+          rescue IOError
+            break if quitting
+            raise
+          end
+        end
+      end
+
+      loop do
+        break if ok_to_continue
+        sleep 0.1
+      end
+      ok_to_continue = false
+
+      config = {}
+      config[:pb_port] = port
+      config[:client_id] = port
+      config[:read_timeout] = 0.0001
+      client = Riak::Client.new(config)
+
+      max_ping_attempts = 16
+      ping_count = 0
+      loop do
+        begin
+          client.ping
+          ping_count += 1
+          break if ping_count > max_ping_attempts
+        rescue RuntimeError => e
+          break if e.message =~ /timed out/
+        end
+        sleep 0.5
+      end
+
+      quitting = true
+      server.close
+      thr.join
+
+      expect(ping_count).to be < max_ping_attempts
+    end
+
+    it 'raises error on write timeout' do
+      ok_to_continue = false
+      quitting = false
+      port = 0
+
+      server = nil
+      thr = Thread.new do
+        server = TCPServer.new port
+        port = server.addr[1]
+        ok_to_continue = true
+        loop do
+          begin
+            Thread.start(server.accept) do |s|
+              loop do
+                p = Riak::Client::BeefcakeProtobuffsBackend::Protocol.new s
+                begin
+                  msgname, _body = p.receive
+                rescue IOError
+                  break if quitting
+                  raise
+                end
+                case msgname
+                when :PingReq
+                  p.write :PingResp
+                when :GetServerInfoReq
+                  r = Riak::Client::BeefcakeProtobuffsBackend::RpbGetServerInfoResp.new
+                  r.node = 'dev1@127.0.0.1'.force_encoding('BINARY')
+                  r.server_version = '2.1.4'.force_encoding('BINARY')
+                  p.write :GetServerInfoResp, r
+                when :PutReq
+                  r = Riak::Client::BeefcakeProtobuffsBackend::RpbPutResp.new
+                  p.write :PutResp, r
+                else
+                  $stderr.puts("unknown msgname: #{msgname}")
+                end
+              end
+            end
+          rescue IOError
+            break if quitting
+            raise
+          end
+        end
+      end
+
+      loop do
+        break if ok_to_continue
+        sleep 0.1
+      end
+      ok_to_continue = false
+
+      config = {}
+      config[:pb_port] = port
+      config[:client_id] = port
+      config[:write_timeout] = 0.0001
+      client = Riak::Client.new(config)
+
+      bucket = client.bucket('timeouts')
+
+      max_store_attempts = 16
+      store_count = 0
+      loop do
+        begin
+          obj = bucket.new "obj-#{store_count}"
+          # write enough data to grow beyond socket buffer capacity
+          obj.data = SecureRandom.urlsafe_base64(10_000_000)
+          obj.content_type = 'text/plain'
+          obj.store
+          store_count += 1
+          break if store_count > max_store_attempts
+        rescue RuntimeError => e
+          break if e.message =~ /timed out/
+        end
+        sleep 0.5
+      end
+
+      quitting = true
+      server.close
+      thr.join
+
+      expect(store_count).to be < max_store_attempts
+    end
+  end
+end