tarantool 0.3.0.7 → 0.4.2.1

data/lib/tarantool/space_hash.rb

@@ -5,9 +5,10 @@ require 'tarantool/core-ext'
 
 module Tarantool
   class SpaceHash
+    include CommonSpace
     include Request
 
-    def initialize(tarantool, space_no, fields_def, indexes)
+    def initialize(tarantool, space_no, fields_def, indexes, shard_fields = nil, shard_proc = nil)
       @tarantool = tarantool
       @space_no = space_no
 
@@ -46,16 +47,17 @@ module Tarantool
       @index_fields = [*indexes].map{|ind| [*ind].map{|fld| fld.to_sym}.freeze}.freeze
       @indexes = _map_indexes(@index_fields)
       @translators = [TranslateToHash.new(@field_names - [:_tail], @tail_size)].freeze
-    end
 
-    def _add_translator(v)
-      @translators += [v]
+      @shard_fields = [*(shard_fields || @index_fields[0])]
+      @shard_positions = @shard_fields.map{|name| @field_to_pos[name]}
+      _init_shard_vars(shard_proc)
     end
 
     def with_translator(cb = nil, &block)
-      copy = dup
-      copy._add_translator(cb || block)
-      copy
+      clone.instance_exec do
+        @translators += [cb || block]
+        self
+      end
     end
 
     def _map_indexes(indexes)
@@ -66,10 +68,6 @@ module Tarantool
       end.freeze
     end
 
-    def _send_request(type, body, cb)
-      @tarantool._send_request(type, body, cb)
-    end
-
     def select_cb(keys, offset, limit, cb)
       index_names = Hash === keys ? keys.keys : keys.first.keys
       index_no = @index_fields.index{|fields|
@@ -102,7 +100,9 @@ module Tarantool
         end
       end
 
-      _select(@space_no, index_no, offset, limit, keys, cb, @field_types, index_types, @translators)
+      shard_nums = _get_shard_nums { _detect_shards_for_keys(keys, index_no) }
+
+      _select(@space_no, index_no, offset, limit, keys, cb, @field_types, index_types, shard_nums, @translators)
     end
 
     def all_cb(keys, cb, opts = {})
@@ -114,15 +114,17 @@ module Tarantool
     end
 
     def all_by_pks_cb(keys, cb, opts={})
-      keys = [*keys].map{|key| _prepare_pk(key)}
+      keys = (Hash === keys ? [keys] : [*keys]).map{|key| _prepare_pk(key)}
+      shard_nums = _get_shard_nums{ _detect_shards_for_keys(keys, 0) }
       _select(@space_no, 0, 
               opts[:offset] || 0, opts[:limit] || -1,
-              keys, cb, @field_types, @indexes[0], @translators)
+              keys, cb, @field_types, @indexes[0], shard_nums, @translators)
     end
 
-    def by_pk_cb(key_array, cb)
-      key_array = _prepare_pk(key_array)
-      _select(@space_no, 0, 0, :first, [key_array], cb, @field_types, @indexes[0], @translators)
+    def by_pk_cb(pk, cb)
+      pk = _prepare_pk(pk)
+      shard_nums = _get_shard_nums{ _detect_shards_for_key(pk, 0) }
+      _select(@space_no, 0, 0, :first, [pk], cb, @field_types, @indexes[0], shard_nums, @translators)
     end
 
     def _prepare_tuple(tuple)
@@ -142,13 +144,30 @@ module Tarantool
     end
 
     def insert_cb(tuple, cb, opts = {})
-      _insert(@space_no, BOX_ADD, _prepare_tuple(tuple),
-              @field_types, cb, opts[:return_tuple], @translators)
+      tuple = _prepare_tuple(tuple)
+      shard_nums = detect_shard_for_insert(tuple.values_at(*@shard_positions))
+      _insert(@space_no, BOX_ADD, tuple, @field_types, cb, opts[:return_tuple],
+              shard_nums, nil, @translators)
     end
 
     def replace_cb(tuple, cb, opts = {})
-      _insert(@space_no, BOX_REPLACE, _prepare_tuple(tuple),
-              @field_types, cb, opts[:return_tuple], @translators)
+      tuple = _prepare_tuple(tuple)
+      shard_nums = detect_shard(tuple.values_at(*@shard_positions))
+      _insert(@space_no, BOX_REPLACE, tuple, @field_types, cb, opts[:return_tuple],
+              shard_nums, opts.fetch(:in_any_shard, true), @translators)
+    end
+
+    def store_cb(tuple, cb, opts = {})
+      tuple = _prepare_tuple(tuple)
+      shard_nums = _get_shard_nums{
+        if opts.fetch(:to_insert_shard, true)
+          _detect_shard_for_insert(tuple.values_at(*@shard_positions))
+        else
+          _detect_shard(tuple.values_at(*@shard_positions))
+        end
+      }
+      _insert(@space_no, 0, tuple, @field_types, cb, opts[:return_tuple],
+              shard_nums, nil, @translators)
     end
 
     def _prepare_pk(pk)
@@ -209,13 +228,16 @@ module Tarantool
           )
         end
       }
-      _update(@space_no, pk, opers, @field_types,
-              @indexes[0], cb, opts[:return_tuple], @translators)
+      shard_nums = _get_shard_nums{ _detect_shards_for_key(pk, 0) }
+      _update(@space_no, pk, opers, @field_types, @indexes[0], cb, opts[:return_tuple],
+              shard_nums, @translators)
     end
 
     def delete_cb(pk, cb, opts = {})
-      _delete(@space_no, _prepare_pk(pk), @field_types,
-              @indexes[0], cb, opts[:return_tuple], @translators)
+      pk = _prepare_pk(pk)
+      shard_nums = _get_shard_nums{ _detect_shards_for_key(pk, 0) }
+      _delete(@space_no, pk, @field_types, @indexes[0], cb, opts[:return_tuple],
+              shard_nums, @translators)
     end
 
     def invoke_cb(func_name, values, cb, opts = {})
@@ -252,10 +274,10 @@ module Tarantool
     end
 
     def first_blk(key, &block)
-      first_blk(key, block)
+      first_cb(key, block)
     end
 
-    def select_blk(keys, offset=0, limit=1, &block)
+    def select_blk(keys, offset=0, limit=-1, &block)
       select_cb(keys, offset, limit, block)
     end
   end

data/lib/tarantool/util.rb

@@ -1,8 +1,36 @@
 module Tarantool
   module Util
     module Packer
+      INT8  = 'C'.freeze
+      INT16 = 'v'.freeze
+      INT32 = 'V'.freeze
+      INT64 = 'Q<'.freeze
+      SINT8  = 'c'.freeze
+      SINT16 = 's<'.freeze
+      SINT32 = 'l<'.freeze
+      SINT64 = 'q<'.freeze
+      MIN_INT   = 0
+      MAX_INT64 = 2**64 - 1
+      MAX_INT32 = 2**32 - 1
+      MAX_INT16 = 2**16 - 1
+      MAX_INT8 = 2**8 - 1
+      MAX_SINT64 = 2**63 - 1
+      MAX_SINT32 = 2**31 - 1
+      MAX_SINT16 = 2**15 - 1
+      MAX_SINT8 = 2**7 - 1
+      MIN_SINT64 = -(2**63)
+      MIN_SINT32 = -(2**31)
+      MIN_SINT16 = -(2**15)
+      MIN_SINT8  = -(2**7)
     private
       EMPTY = ''.freeze
+      ONE = "\x01".freeze
+      def unpack_int8!(data)
+        int = data.getbyte(0)
+        data[0, 1] = EMPTY
+        int
+      end
+
       def unpack_int16(data)
         data.getbyte(0) + data.getbyte(1) * 256
       end
@@ -10,7 +38,7 @@ module Tarantool
       def unpack_int16!(data)
         int = data.getbyte(0) + data.getbyte(1) * 256
         data[0, 2] = EMPTY
-        data
+        int
       end
 
       def unpack_int32(int)
@@ -26,20 +54,33 @@ module Tarantool
       end
 
       def unpack_int64!(data)
-        int = (data.getbyte(0) + data.getbyte(1) * 256 +
-               data.getbyte(2) * 65536 + data.getbyte(3) * 16777216 +
-               data.getbyte(4) << 32 + data.getbyte(5) << 40 +
-               data.getbyte(6) << 48 + data.getbyte(7) << 56
-              )
+        int = data.unpack(INT64)[0]
         data[0, 8] = EMPTY
         int
       end
 
       def unpack_int64(data)
-        data.getbyte(0) + data.getbyte(1) * 256 +
-        data.getbyte(2) * 65536 + data.getbyte(3) * 16777216 +
-        data.getbyte(4) << 32 + data.getbyte(5) << 40 +
-        data.getbyte(6) << 48 + data.getbyte(7) << 56
+        data.unpack(INT64)[0]
+      end
+
+      def unpack_sint8!(data)
+        i = unpack_int8!(data)
+        i - ((i & 128) << 1)
+      end
+
+      def unpack_sint16!(data)
+        i = unpack_int16!(data)
+        i - ((i & 32768) << 1)
+      end
+
+      def unpack_sint32!(data)
+        i = unpack_int32!(data)
+        i - ((i >> 31) << 32)
+      end
+
+      def unpack_sint64!(data)
+        i = unpack_int64!(data)
+        i - ((i >> 63) << 64)
       end
 
       def ber_size(int)
@@ -64,6 +105,51 @@ module Tarantool
         data[0, pos+1] = EMPTY
         res
       end
+
+      def append_int8!(str, int)
+        str << (int & 255)
+      end
+
+      def append_int16!(str, int)
+        str << (int & 255) << ((int>>8) & 255)
+      end
+
+      def append_int32!(str, int)
+        str << (int & 255) << ((int>>8) & 255) <<
+               ((int>>16) & 255) << ((int>>24) & 255)
+      end
+
+      def append_int64!(str, int)
+        str << [int].pack(INT64)
+      end
+
+      alias append_sint8! append_int8!
+      alias append_sint16! append_int16!
+      alias append_sint32! append_int32!
+      alias append_sint64! append_int64!
+
+      def append_ber_int8!(str, int)
+        str << 1 << (int & 255)
+      end
+
+      def append_ber_int16!(str, int)
+        str << 2 << (int & 255) << ((int>>8) & 255)
+      end
+
+      def append_ber_int32!(str, int)
+        str << 4 <<
+                (int & 255) << ((int>>8) & 255) <<
+               ((int>>16) & 255) << ((int>>24) & 255)
+      end
+
+      def append_ber_int64!(str, int)
+        str << 8 << [int].pack(INT64)
+      end
+
+      alias append_ber_sint8! append_ber_int8!
+      alias append_ber_sint16! append_ber_int16!
+      alias append_ber_sint32! append_ber_int32!
+      alias append_ber_sint64! append_ber_int64!
     end
 
     module TailGetter

data/lib/tarantool/version.rb

@@ -1,3 +1,4 @@
 module Tarantool
-  VERSION = "0.3.0.7"
+  VERSION = "0.4.2.1"
+  RECORD_VERSION = "0.4.1.1"
 end

data/test/helper.rb

@@ -1,6 +1,6 @@
 require 'minitest/spec'
-require 'minitest/autorun'
 require 'rr'
+require 'fileutils'
 
 require 'tarantool'
 
@@ -14,6 +14,138 @@ class ArrayPackSerializer
   end
 end
 
+module TConf
+  extend FileUtils
+  CONF = {
+    master1: {port: 33013, replica: :imaster},
+    slave1:  {port: 34013, replica: :master1},
+    master2: {port: 35013, replica: :imaster},
+    slave2:  {port: 36013, replica: :master2},
+  }
+  DIR = File.expand_path('..', __FILE__)
+  def self.fjoin(*args)
+    File.join(*args.map(&:to_s))
+  end
+  def self.dir(name)
+    fjoin(DIR, "tarantool_#{name}")
+  end
+
+  def self._to_master(cfg, conf)
+    cfg.sub!(/^replication_source.*$/, '#\0')
+  end
+
+  def self._to_slave(cfg, master)
+    replica_source = "127.0.0.1:#{CONF.fetch(master)[:port]+3}"
+    cfg.sub!(/^#?replication_source.*$/, "replication_source = \"#{replica_source}\"")
+  end
+
+  def self.prepare(name)
+    conf = CONF.fetch(name)
+    return if conf[:dir]
+    clear(name)
+    dir = dir(name)
+    FileUtils.rm_rf dir
+    mkdir_p(dir)
+    cp fjoin(DIR, 'init.lua'), dir
+    cfg = File.read(fjoin(DIR, 'tarantool.cfg'))
+    cfg.sub!(/(pid_file\s*=\s*)"[^"]*"/, "\\1\"#{fjoin(dir, 'box.pid')}\"")
+    cfg.sub!(/(work_dir\s*=\s*)"[^"]*"/, "\\1\"#{dir}\"")
+    cfg.sub!(/(primary_port\s*=\s*)\d+/, "\\1#{conf[:port]}")
+    cfg.sub!(/(secondary_port\s*=\s*)\d+/, "\\1#{conf[:port]+1}")
+    cfg.sub!(/(admin_port\s*=\s*)\d+/, "\\1#{conf[:port]+2}")
+    cfg.sub!(/(replication_port\s*=\s*)\d+/, "\\1#{conf[:port]+3}")
+    if conf[:replica] == :imaster
+      _to_master(cfg, conf)
+    else
+      _to_slave(cfg, conf[:replica])
+    end
+    File.open(fjoin(dir, 'tarantool.cfg'), 'w'){|f| f.write(cfg)}
+    Dir.chdir(dir) do
+      `tarantool_box --init-storage 2>&1`
+    end
+    conf[:dir] = dir
+  end
+
+  def self.run(name)
+    conf = CONF.fetch(name)
+    return  if conf[:pid]
+    prepare(name)
+    Dir.chdir(conf[:dir]) do
+      conf[:pid] = spawn('tarantool_box')
+    end
+  end
+
+  def self.stop(name)
+    conf = CONF.fetch(name)
+    return  unless conf[:pid]
+    Process.kill('INT', conf[:pid])
+    Process.wait2(conf[:pid])
+    conf.delete :pid
+  end
+
+  def self.clear(name)
+    conf = CONF.fetch(name)
+    return  unless conf[:dir]
+    stop(name)
+    rm_rf(conf[:dir])
+    conf.delete :dir
+  end
+
+  def self.clear_all
+    CONF.keys.each{|name| clear(name) }
+  end
+
+  def self.reset_and_up_all
+    clear_all
+    CONF.keys.each{|name| run(name) }
+  end
+
+  def self.reset_and_up_masters
+    clear_masters
+    run(:master1)
+    run(:master2)
+    sleep(0.01)
+  end
+
+  def self.clear_masters
+    clear(:master1)
+    clear(:master2)
+  end
+
+  def self.conf(name)
+    {host: '127.0.0.1', port: CONF.fetch(name)[:port]}
+  end
+
+  def self.promote_to_master(name)
+    conf = CONF.fetch(name)
+    raise "#{name} not running"  unless conf[:pid]
+    fcfg = fjoin(dir(name), 'tarantool.cfg')
+    cfg = File.read(fcfg)
+    _to_master(cfg, conf)
+    File.open(fcfg, 'w'){|f| f.write(cfg)}
+    sock = TCPSocket.new('127.0.0.1', conf[:port]+2)
+    sock.write("reload configuration\n")
+    3.times{ sock.gets }
+  end
+
+  def self.promote_to_slave(slave, master)
+    clear(slave)
+    prepare(slave)
+    fcfg = fjoin(dir(slave), 'tarantool.cfg')
+    cfg = File.read(fcfg)
+    _to_slave(cfg, master)
+    File.open(fcfg, 'w'){|f| f.write(cfg)}
+    run(slave)
+  end
+
+  at_exit do
+    CONF.keys.each{|name| clear(name)}
+  end
+end
+require 'minitest/autorun'
+TConf.run(:master1)
+
+
 TCONFIG = { host: '127.0.0.1', port: 33013, admin: 33015 }
 
 SPACE0 = {
@@ -53,7 +185,7 @@ HSPACE3 = {
 module Helper
   def tarantool_pipe
     $tarantool_pipe ||= begin
-        cnf = TCONFIG
+        cnf = {port: 33013, admin: 33015} #TCONFIG
         tarant = %W{tarantool -p #{cnf[:port]} -m #{cnf[:admin]}}
         tarant = [{}, *tarant, :err => [:child, :out]]
         IO.popen(tarant, 'w+').tap{|p| p.sync = true}
@@ -115,17 +247,35 @@ module Helper
     end
   end
 
+  def setp(i)
+    @results ||= []
+    proc{|v| @results[i]=v; emstop}
+  end
+
+  def results
+    @results
+  end
+
+  class TimeOut < Exception
+  end
   def fibrun
     res = nil
     EM.run {
+      t = nil
       f = Fiber.new{
         begin
           res = yield
         ensure
-          EM.next_tick{ EM.stop }
+          EM.next_tick{
+            EM.cancel_timer t
+            EM.stop
+          }
         end
       }
       EM.next_tick{ f.resume }
+      t = EM.add_timer(1) {
+        f.resume TimeOut.new
+      }
     }
     res
   end
@@ -156,7 +306,7 @@ module MiniTest::Spec::SharedExamples
   end
 
   def it_behaves_like(desc)
-    class_eval &MiniTest::Spec.shared_examples.fetch(desc)
+    class_eval(&MiniTest::Spec.shared_examples.fetch(desc))
   end
 end