tarantool 0.3.0.7 → 0.4.2.1

data/lib/tarantool/response.rb

@@ -3,16 +3,32 @@ require 'tarantool/exceptions'
 require 'tarantool/serializers'
 
 module Tarantool
-  class Response < Struct.new(:cb, :get_tuples, :fields, :translators)
+  module ParseIProto
+    include Util::Packer
+    def _parse_iproto(data)
+      if Exception === data || data == ''
+        data
+      elsif (ret = unpack_int32!(data)) == 0
+        data
+      else
+        data.gsub!("\x00", "")
+        CODE_TO_EXCEPTION[ret].new(ret, data)
+      end
+    end
+  end
+
+  class Response < Struct.new(:cb, :request_type, :body, :get_tuples, :fields, :translators)
     include Util::Packer
     include Util::TailGetter
     include Serializers
+    UTF8 = 'utf-8'.freeze
+
     def call(data)
       if Exception === data
         cb.call(data)
       else
         if (ret = return_code(data)) == 0
-          cb.call parse_response(data)
+          call_callback parse_response(data)
         else
           data.gsub!("\x00", "")
           cb.call CODE_TO_EXCEPTION[ret].new(ret, data)
@@ -24,7 +40,12 @@ module Tarantool
       super || (self.translators = [])
     end
 
+    def call_callback(result)
+      cb.call(Exception === result || get_tuples != :first ? result : result.first)
+    end
+
     def parse_response(data)
+      return data  if Exception === data
       unless get_tuples
         unpack_int32(data)
       else
@@ -34,7 +55,7 @@ module Tarantool
             tuples.map!{|tuple| trans.call(tuple)}
           }
         end
-        get_tuples == :first ? tuples.first : tuples
+        tuples
       end
     end
 
@@ -62,6 +83,52 @@ module Tarantool
           tuple << (field_size == 0 ? nil :
             case field
             when :int, :integer
+              if field_size != 4
+                raise ValueError, "Bad field size #{field_size} for integer field ##{i}"
+              end
+              unpack_int32!(tuple_str)
+            when :string, :str
+              str = tuple_str.slice!(0, field_size)
+              str[0,1] = EMPTY  if str < ONE
+              str.force_encoding(UTF8)
+            when :int64
+              if field_size != 8
+                raise ValueError, "Bad field size #{field_size} for 64bit integer field ##{i}"
+              end
+              unpack_int64!(tuple_str)
+            when :bytes
+              tuple_str.slice!(0, field_size)
+            when :int16
+              if field_size != 2
+                raise ValueError, "Bad field size #{field_size} for 16bit integer field ##{i}"
+              end
+              unpack_int16!(tuple_str)
+            when :int8
+              if field_size != 1
+                raise ValueError, "Bad field size #{field_size} for 8bit integer field ##{i}"
+              end
+              unpack_int8!(tuple_str)
+            when :sint
+              if field_size != 4
+                raise ValueError, "Bad field size #{field_size} for integer field ##{i}"
+              end
+              unpack_sint32!(tuple_str)
+            when :sint64
+              if field_size != 8
+                raise ValueError, "Bad field size #{field_size} for 64bit integer field ##{i}"
+              end
+              unpack_sint64!(tuple_str)
+            when :sint16
+              if field_size != 2
+                raise ValueError, "Bad field size #{field_size} for 16bit integer field ##{i}"
+              end
+              unpack_sint16!(tuple_str)
+            when :sint8
+              if field_size != 1
+                raise ValueError, "Bad field size #{field_size} for 8bit integer field ##{i}"
+              end
+              unpack_sint8!(tuple_str)
+            when :varint
               case field_size
               when 8
                 unpack_int64!(tuple_str)
@@ -72,10 +139,6 @@ module Tarantool
               else
                 raise ValueError, "Bad field size #{field_size} for integer field ##{i}"
               end
-            when :str, :string
-              tuple_str.slice!(0, field_size).force_encoding('utf-8')
-            when :bytes
-              tuple_str.slice!(0, field_size)
             when :auto
               str = tuple_str.slice!(0, field_size).force_encoding('utf-8')
               case field_size

data/lib/tarantool/serializers.rb

@@ -4,22 +4,43 @@ module Tarantool
 
     def check_type(type)
       type = type.to_sym  if String === type
-      return :int  if Integer == type
-      return :str  if String  == type
+      return :varint  if Integer == type
+      return :string  if String  == type
 
       case type
-      when :int, :integer, :str, :string, :bytes, :auto
-        # pass
+      when :int, :integer, :int32, :integer32
+        :int
+      when :int64, :integer64
+        :int64
+      when :int16, :integer16
+        :int16
+      when :int8, :integer8
+        :int8
+      when :sint, :sinteger, :sint32, :sinteger32
+        :sint
+      when :sint16, :sinteger16
+        :sint16
+      when :sint64, :sinteger64
+        :sint64
+      when :sint8, :sinteger8
+        :sint8
+      when :varint
+        :varint
+      when :str, :string
+        :string
+      when :bytes, :auto
+        type
       when Symbol
         unless MAP.include?(type)
           raise ArgumentError, "Unknown type name #{type.inspect}"
         end
+        type
       else
         unless type.respond_to?(:encode) && type.respond_to?(:decode)
           raise ArgumentError, "Wrong serializer object #{type.inspect} (must respond to #encode and #decode)"
         end
+        type
       end
-      type
     end
 
     def get_serializer(type)

data/lib/tarantool/serializers/ber_array.rb

@@ -0,0 +1,14 @@
+module Tarantool
+  module Serializers
+    class BerArray
+      Serializers::MAP[:ber_array] = self
+      def self.encode(value)
+        value.pack('w*')
+      end
+
+      def self.decode(value)
+        value.unpack('w*')
+      end
+    end
+  end
+end

data/lib/tarantool/shards_support.rb

@@ -0,0 +1,204 @@
+require 'sumbur'
+require 'murmurhash3'
+module Tarantool
+  module Request
+    class SumburMurmur32
+      include Sumbur
+      include MurmurHash3::V32
+    end
+
+    class SumburMurmurFmix < SumburMurmur32
+      def call(shard_values, shards_count, this)
+        shard_values[0] &&
+          sumbur(murmur3_32_fmix(shard_values[0]), shards_count)
+      end
+    end
+
+    class SumburMurmurInt64 < SumburMurmur32
+      def call(shard_values, shards_count, this)
+        shard_values[0] &&
+          sumbur(murmur3_32_int64_hash(shard_values[0]), shards_count)
+      end
+    end
+
+    class SumburMurmurStr < SumburMurmur32
+      def call(shard_values, shards_count, this)
+        shard_values[0] &&
+          sumbur(murmur3_32_str_hash(shard_values[0]), shards_count)
+      end
+    end
+
+    class DefaultShardProc < SumburMurmur32
+      def call(shard_values, shards_count, this)
+        if shard_values.all?
+          hash = i = 0
+          size = shard_values.size
+          while i < size
+            hash = case value = shard_values[i]
+              when Integer
+                murmur3_32_int64_hash(value, hash)
+              when String
+                murmur3_32_str_hash(value, hash)
+              else
+                raise ValueError, "Default sharding proc could deal only with strings and integers"
+              end
+            i += 1
+          end
+          sumbur(hash, shards_count)
+        end
+      end
+    end
+
+    class ModuloShardProc
+      def call(shard_values, shards_count, this)
+        shard_values[0] && shard_values[0] % shards_count
+      end
+    end
+
+    attr_reader :shard_proc, :shards_count, :previous_shards_count
+    attr_reader :insert_with_shards_count
+
+    def _init_shard_vars(shard_proc, init_shard_for_index = true)
+      if init_shard_for_index
+        @shard_by_index = @index_fields.index{|index| index == @shard_fields}
+        @shard_for_index = @index_fields.map{|index|
+            if (pos = @shard_fields.map{|name| index.index(name)}).any?
+              pos.map{|i| i || 2**30}
+            end
+        }
+        @shard_proc = case shard_proc
+                      when nil, :default
+                        DefaultShardProc.new
+                      when :sumbur_murmur_fmix
+                        SumburMurmurFmix.new
+                      when :sumbur_murmur_int64
+                        SumburMurmurInt64.new
+                      when :sumbur_murmur_str
+                        SumburMurmurStr.new
+                      when :modulo, :module
+                        ModuloShardProc.new
+                      else
+                        unless shard_proc.respond_to?(:call)
+                          raise ArgumentError, "Wrong sharding proc object #{shard_proc.inspect}"
+                        end
+                        shard_proc
+                      end
+      end
+
+      @shards_count = @tarantool.shards_count
+      @previous_shards_count = @tarantool.previous_shards_count
+      @insert_with_shards_count = @tarantool.insert_with_shards_count
+      @default_shard = 0
+    end
+
+    def _detect_shards_for_keys(keys, index_no)
+      return _detect_shards_for_key(keys, index_no)  unless Array === keys
+      if index_no == @shard_by_index && (
+            @index_fields.size == 1 ||
+            keys.all?{|key| Array === key && key.size == @index_fields.size}
+          )
+        _flat_uniq keys.map{|key| _detect_shard(key)}
+      elsif pos = @shard_for_index[index_no]
+        _flat_uniq keys.map{|key| _detect_shard([*key].values_at(*pos)) }
+      else
+        _all_shards
+      end
+    end
+
+    def _detect_shards_for_key(key, index_no)
+      if index_no == @shard_by_index
+        _detect_shard(key)
+      elsif pos = @shard_for_index[index_no]
+        _detect_shard(key.values_at(*pos))
+      else
+        _all_shards
+      end
+    end
+
+    def _detect_shards(keys)
+      _flat_uniq keys.map{|key| _detect_shard(key)}
+    end
+
+    def _detect_shards_for_insert(keys)
+      _flat_uniq keys.map{|key| _detect_shard_for_insert(key)}
+    end
+
+    def _flat_uniq(array)
+      hsh = {}
+      array.each do |v|
+        if v.respond_to?(:each)
+          v.each{|vv| hsh[vv] = true}
+        else
+          hsh[v] = true
+        end
+      end
+      hsh.keys
+    end
+
+    def _detect_shard_for_insert(shard_values)
+      shard_values = [shard_values]  unless Array === shard_values
+      @shard_proc.call(shard_values, @insert_with_shards_count, self) || _all_shards
+    end
+
+    def _detect_shard(shard_values)
+      shard_values = [shard_values]  unless Array === shard_values
+      shards = @shard_proc.call(shard_values, @shards_count, self) || _all_shards
+      if @previous_shards_count
+        prev_shards = @shard_proc.call(shard_values, @previous_shards_count, self) || _all_shards
+        shards = [*shards, *prev_shards].uniq
+      end
+      shards
+    end
+
+    def _get_shard_nums
+      @shards_count == 1 ? @default_shard : yield
+    end
+
+    def _all_shards
+      (0...@shards_count).to_a
+    end
+
+    def detect_shard(shard_values)
+      @shards_count == 1 ? @default_shard : _detect_shard(shard_values)
+    end
+
+    def detect_shard_for_insert(shard_values)
+      @shards_count == 1 ? @default_shard : _detect_shard_for_insert(shard_values)
+    end
+
+    def detect_shards(shard_values)
+      @shards_count == 1 ? @default_shard : _detect_shards(shard_values)
+    end
+
+    def detect_shards_for_insert(shard_values)
+      @shards_count == 1 ? @default_shard : _detect_shards_for_insert(shard_values)
+    end
+
+    def all_shards
+      @shards_count == 1 ? @default_shard : (0...@shards_count).to_a
+    end
+
+    def shard(shard_number)
+      case shard_number
+      when Integer
+        if shard_number >= @shards_count
+          raise ArgumentError, "There is no shard #{shard_number}, amount of shards is #{@shards_count}"
+        end
+      when Array
+        shard_number.each do|i|
+          if i >= @shards_count
+            raise ArgumentError, "There is no shard #{i}, amount of shards is #{@shards_count}"
+          end
+        end
+      else
+        raise ArgumentError, "Shard number should be integer or array of integers"
+      end
+      (@_fixed_shards ||= {})[shard_number] ||=
+          clone.instance_exec do
+            @shards_count = 1
+            @default_shard = shard_number
+            self
+          end
+    end
+  end
+end

data/lib/tarantool/space_array.rb

@@ -6,10 +6,11 @@ require 'tarantool/core-ext'
 
 module Tarantool
   class SpaceArray
+    include CommonSpace
     include Request
     include Util::Array
 
-    def initialize(tarantool, space_no, fields, indexes)
+    def initialize(tarantool, space_no, fields, indexes, shard_fields = nil, shard_proc = nil)
       @tarantool = tarantool
       @space_no = space_no
 
@@ -24,9 +25,18 @@ module Tarantool
         @index_fields = indexes
         @indexes = _map_indexes(indexes)
       else
-        @index_fields = nil
+        @index_fields = []
         @indexes = [TYPES_FALLBACK]
       end
+
+      shard_fields ||= @index_fields[0]
+      unless shard_fields || @tarantool.shards_count == 1
+        raise ArgumentError, "You could not use space without specifying primary key or shard fields when shards count is greater than 1"
+      else
+        @shard_fields = [*shard_fields]
+        @shard_positions = @shard_fields
+        _init_shard_vars(shard_proc)
+      end
     end
 
     def _map_indexes(indexes)
@@ -40,10 +50,6 @@ module Tarantool
       end.freeze
     end
 
-    def _send_request(type, body, cb)
-      @tarantool._send_request(type, body, cb)
-    end
-
     def all_by_pks_cb(keys, cb, opts={})
       all_by_keys_cb(0, keys, cb, opts)
     end
@@ -76,7 +82,7 @@ module Tarantool
 
     def select_cb(index_no, keys, offset, limit, cb)
       if Array === index_no
-        raise ArgumentError, "Has no defined indexes to search index #{index_no}"  unless @index_fields
+        raise ArgumentError, "Has no defined indexes to search index #{index_no}"  if @index_fields.empty?
         index_fields = index_no
         index_no = @index_fields.index{|fields| fields.take(index_fields.size) == index_fields}
         unless index_no || index_fields.size == 1
@@ -87,7 +93,7 @@ module Tarantool
           end
         end
       end
-      if @index_fields
+      unless @index_fields.empty?
         unless index_types = @indexes[index_no]
           raise ArgumentError, "No index ##{index_no}"
         end
@@ -95,27 +101,46 @@ module Tarantool
         index_types = _detect_types([*[*keys][0]])
       end
 
-      _select(@space_no, index_no, offset, limit, keys, cb, @fields, index_types)
+      shard_nums = _get_shard_nums { _detect_shards_for_keys(keys, index_no) }
+
+      _select(@space_no, index_no, offset, limit, keys, cb, @fields, index_types, shard_nums)
     end
 
     def insert_cb(tuple, cb, opts = {})
-      _insert(@space_no, BOX_ADD, tuple, @fields, cb, opts[:return_tuple])
+      shard_nums = detect_shard_for_insert(tuple.values_at(*@shard_positions))
+      _insert(@space_no, BOX_ADD, tuple, @fields, cb, opts[:return_tuple], shard_nums)
     end
 
     def replace_cb(tuple, cb, opts = {})
-      _insert(@space_no, BOX_REPLACE, tuple, @fields, cb, opts[:return_tuple])
+      shard_nums = detect_shard(tuple.values_at(*@shard_positions))
+      _insert(@space_no, BOX_REPLACE, tuple, @fields,
+              cb, opts[:return_tuple], shard_nums, opts.fetch(:in_any_shard, true))
+    end
+
+    def store_cb(tuple, cb, opts = {})
+      shard_nums = _get_shard_nums{
+        if opts.fetch(:to_insert_shard, true)
+          _detect_shard_for_insert(tuple.values_at(*@shard_positions))
+        else
+          _detect_shard(tuple.values_at(*@shard_positions))
+        end
+      }
+      _insert(@space_no, 0, tuple, @fields, cb, opts[:return_tuple], shard_nums)
     end
 
     def update_cb(pk, operations, cb, opts = {})
+      shard_nums = _get_shard_nums{ _detect_shards_for_key(pk, 0) }
       _update(@space_no, pk, operations, @fields,
               @indexes[0] || _detect_types([*pk]),
-              cb, opts[:return_tuple])
+              cb, opts[:return_tuple],
+              shard_nums)
     end
 
     def delete_cb(pk, cb, opts = {})
+      shard_nums = _get_shard_nums{ _detect_shards_for_key(pk, 0) }
       _delete(@space_no, pk, @fields,
               @indexes[0] || _detect_types([*pk]),
-              cb, opts[:return_tuple])
+              cb, opts[:return_tuple], shard_nums)
     end
 
     def invoke_cb(func_name, values, cb, opts = {})