tarantool 0.3.0.7 → 0.4.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. data/Gemfile +2 -3
  2. data/README.md +90 -30
  3. data/Rakefile +6 -1
  4. data/lib/tarantool.rb +93 -18
  5. data/lib/tarantool/base_record.rb +97 -10
  6. data/lib/tarantool/block_db.rb +104 -6
  7. data/lib/tarantool/callback_db.rb +7 -0
  8. data/lib/tarantool/core-ext.rb +24 -8
  9. data/lib/tarantool/em_db.rb +189 -20
  10. data/lib/tarantool/exceptions.rb +4 -0
  11. data/lib/tarantool/fiber_db.rb +15 -1
  12. data/lib/tarantool/light_record.rb +17 -0
  13. data/lib/tarantool/query.rb +15 -9
  14. data/lib/tarantool/record/select.rb +21 -3
  15. data/lib/tarantool/request.rb +130 -43
  16. data/lib/tarantool/response.rb +70 -7
  17. data/lib/tarantool/serializers.rb +26 -5
  18. data/lib/tarantool/serializers/ber_array.rb +14 -0
  19. data/lib/tarantool/shards_support.rb +204 -0
  20. data/lib/tarantool/space_array.rb +38 -13
  21. data/lib/tarantool/space_hash.rb +49 -27
  22. data/lib/tarantool/util.rb +96 -10
  23. data/lib/tarantool/version.rb +2 -1
  24. data/test/helper.rb +154 -4
  25. data/test/{tarant/init.lua → init.lua} +0 -0
  26. data/test/run_all.rb +2 -2
  27. data/test/shared_record.rb +59 -0
  28. data/test/shared_replicated_shard.rb +1018 -0
  29. data/test/shared_reshard.rb +380 -0
  30. data/test/tarantool.cfg +2 -0
  31. data/test/test_light_record.rb +2 -0
  32. data/test/test_light_record_callback.rb +92 -0
  33. data/test/test_query_block.rb +1 -0
  34. data/test/test_query_fiber.rb +1 -0
  35. data/test/test_reshard_block.rb +7 -0
  36. data/test/test_reshard_fiber.rb +11 -0
  37. data/test/test_shard_replication_block.rb +7 -0
  38. data/test/test_shard_replication_fiber.rb +11 -0
  39. data/test/test_space_array_block.rb +1 -0
  40. data/test/test_space_array_callback.rb +50 -121
  41. data/test/test_space_array_callback_nodef.rb +39 -96
  42. data/test/test_space_array_fiber.rb +1 -0
  43. data/test/test_space_hash_block.rb +1 -0
  44. data/test/test_space_hash_fiber.rb +1 -0
  45. metadata +54 -17
  46. data/lib/tarantool/record.rb +0 -164
  47. data/test/box.pid +0 -1
  48. data/test/tarantool.log +0 -6
  49. data/test/tarantool_repl.cfg +0 -53
  50. data/test/test_record.rb +0 -88
  51. data/test/test_record_composite_pk.rb +0 -77
data/Gemfile CHANGED
@@ -6,11 +6,10 @@ end
6
6
 
7
7
  group :test do
8
8
  gem "rr"
9
- gem "activesupport"
10
9
  gem "activemodel"
11
10
  gem "yajl-ruby"
12
11
  gem "bson"
13
12
  gem "bson_ext"
14
13
  end
15
- # Specify your gem's dependencies in em-tarantool.gemspec
16
- gemspec
14
+ # Specify your gem's dependencies in tarantool.gemspec
15
+ gemspec name: 'tarantool'
data/README.md CHANGED
@@ -14,12 +14,36 @@ gem install tarantool
14
14
  require 'tarantool'
15
15
  ```
16
16
 
17
- To be able to send requests to the server, you must
18
- initialize Tarantool and Tarantool space:
17
+ To be able to send requests to the server, you must initialize Tarantool
18
+ and Tarantool space. Space could be initialized with definition of fields
19
+ types (and names) or without (which is not recommended).
20
+
21
+ Available field types:
22
+ - `:int`, `:integer` - nonnegative 32 bit integer
23
+ - `:int64`, `:integer64` - nonnegative 64 bit integer
24
+ - `:varint` - 32bit or 64bit integer, depending on value
25
+ - `:str`, `:string` - UTF-8 string (attention: empty string is stored as "\x00", which converted back to "" on load)
26
+ - `:bytes` - ASCII8-bit
27
+ - `:auto` - do not use it (used for space without definition)
28
+ - any object with #encode and #decode methods
29
+
30
+ Declaration of indexes is optional for array spaces and required for hash spaces.
31
+ When there is no indexes defined for space array, their behaviour is not fixes, so that
32
+ is up to you to specify right amount of values for that indexes.
19
33
 
20
34
  ```ruby
21
35
  DB = Tarantool.new host: 'locahost', port: 33013
22
- space = DB.space 0
36
+ space_array_without_definition = DB.space 0
37
+
38
+ space_array = DB.space 1, [:int, :str, :int], keys: [0, [1,2]]
39
+
40
+ # last integer specifies tuples tail pattern
41
+ # note, that two indexes are defined here
42
+ space_array_with_tail = DB.space 2, [:int, :str, :str, :int, 2], keys: [0, 1]
43
+
44
+ # space, which returns hashes
45
+ space_hash = DB.space 1, {id: int, name: :str, score: :int}, keys: [:id, [:name, :score]]
46
+ space_hash_with_tail = DB.space 2, {id: int, name: :str, _tail: [:str, :int]}, keys: [:id, :name]
23
47
  ```
24
48
 
25
49
  The driver internals can work in three modes:
@@ -28,39 +52,61 @@ The driver internals can work in three modes:
28
52
  - EM::Synchrony like via EventMachine and fibers, so that control flow is visually
29
53
  blocked, but eventloop is not (see EM::Synchrony)
30
54
 
31
- By default it uses block mode.
55
+ ```ruby
56
+ DB_SYNC = Tarantool.new host: 'localhost', port: 33013, type: :block
57
+ DB_CALLBACK = Tarantool.new host: 'localhost', port: 33013, type: :em_callback || :em_cb
58
+ DB_FIBER = Tarantool.new host: 'localhost', port: 33013, type: :em_fiber || :em
59
+ ```
32
60
 
61
+ Blocking and Fibered interfaces look similar:
33
62
 
34
63
  ```ruby
35
- space.insert 'prepor', 'Andrew', 'ceo@prepor.ru'
36
- res = space.select 'prepor'
37
- puts "Name: #{res.tuple[1].to_s}; Email: #{res.tuple[2].to_s}"
38
- space.delete 'prepor'
64
+ space = (DB_SYNC || DB_FIBER).space 0, [:str, :str, :str], keys: 0
65
+ # EM.synchrony do
66
+ space.insert ['prepor', 'Andrew', 'ceo@prepor.ru']
67
+ res = space.by_pk 'prepor' # || ['prepor']
68
+ res = space.first_by_key 0, 'prepor' # || ['prepor']
69
+ res = space.select 0, ['prepor']
70
+ puts "Name: #{res[1]}; Email: #{res[2]}"
71
+ space.delete 'prepor'
72
+ # EM.stop
73
+ # end
39
74
  ```
40
75
 
41
- **Notice** `Tarantool` instances (connections actually) are not threadsafe. So, you should create `Tarantool` instance per thread.
42
-
43
- To use EventMachine pass type: em in options:
76
+ Callback interface is a bit different:
44
77
 
45
78
  ```ruby
46
- require 'em-synchrony'
47
- DB = Tarantool.new host: 'locahost', port: 33013, type: :em
48
- EM.synchrony do
49
- space = DB.space 0
50
- space.insert 'prepor', 'Andrew', 'ceo@prepor.ru'
51
- res = space.select 'prepor'
52
- puts "Name: #{res.tuple[1].to_s}; Email: #{res.tuple[2].to_s}"
53
- space.delete 'prepor'
54
- EM.stop
79
+ space = DB_CALLBACK.space 0, [:str, :str, :str], keys: 0
80
+ EM.schedule do
81
+ space.insert ['prepor', 'Andrew', 'ceo@prepor.ru'] do |res|
82
+ if Exception === res
83
+ catch_error
84
+ else
85
+ space.by_pk 'prepor' do |res|
86
+ if Exception === res
87
+ catch_error
88
+ else
89
+ puts "Name: #{res[1]}; Email: #{res[2]}"
90
+ space.delete 'prepor' do |res|
91
+ catch_error if Exception === res
92
+ EM.stop
93
+ end
94
+ end
95
+ end
96
+ end
97
+ end
55
98
  end
56
99
  ```
57
100
 
58
- The driver itself provides ActiveModel API: Callbacks, Validations, Serialization, Dirty.
59
- Type casting is automatic, based on the index type chosen to process the query.
60
- For example:
101
+ **Notice** Blocking `Tarantool` connections are not threadsafe. So, you should create `Tarantool` instance per thread.
102
+
103
+ ## LightRecord
104
+
105
+ `LightRecord` is a light model with callbacks ala Sequel. It is not aware about ActiveModel goodness.
106
+ For ActiveModel avare record look for `tarantool-record` gem
61
107
 
62
108
  ```ruby
63
- require 'tarantool/record'
109
+ require 'tarantool/light_record'
64
110
  require 'tarantool/serializers/bson'
65
111
  class User < Tarantool::Record
66
112
  field :login, :string
@@ -70,10 +116,14 @@ class User < Tarantool::Record
70
116
  field :info, :bson
71
117
  index :name, :email
72
118
 
73
- validates_length_of(:login, minimum: 3)
119
+ def after_init
120
+ super
121
+ # some work
122
+ end
74
123
 
75
- after_create do
76
- # after work!
124
+ def before_create
125
+ # validation could occure here
126
+ super # call super if all is allright, return false otherwise
77
127
  end
78
128
  end
79
129
 
@@ -82,11 +132,19 @@ User.create login: 'prepor', email: 'ceo@prepor.ru', name: 'Andrew'
82
132
  User.create login: 'ruden', name: 'Andrew', email: 'rudenkoco@gmail.com'
83
133
 
84
134
  # find by primary key login
135
+ User.by_pk 'prepor'
136
+ User.first 'prepor'
137
+ User.first login: 'prepor'
85
138
  User.find 'prepor'
86
139
  # first 2 users with name Andrew
140
+ User.all({name: 'Andrew'}, limit: 2)
141
+ User.select({name: 'Andrew'}, limit: 2)
87
142
  User.where(name: 'Andrew').limit(2).all
88
143
  # second user with name Andrew
89
- User.where(name: 'Andrew').offset(1).limit(1).all
144
+ User.all({name: 'Andrew'}, offset: 1, limit: 1)[0]
145
+ User.select({name: 'Andrew'}, offset: 1, limit: 1)[0]
146
+ User.where(name: 'Andrew').offset(1).limit(1).all[0]
147
+ User.where(name: 'Andrew').offset(1).first
90
148
  # user with name Andrew and email ceo@prepor.ru
91
149
  User.where(name: 'Andrew', email: 'ceo@prepor.ru').first
92
150
  # raise exception, becouse we can't select query started from not first part of index
@@ -97,10 +155,12 @@ end
97
155
  # increment field apples_count by one. Its atomic operation via native Tarantool interface
98
156
  User.find('prepor').increment :apples_count
99
157
 
100
- # update only dirty attributes
158
+ # update all attributes (see tarantool-record gem for record, which updates only dirty attributes)
101
159
  user = User.find('prepor')
102
160
  user.name = "Petr"
103
161
  user.save
162
+ user.update_attributes email: "petr@inter.com" # calls callbacks as well as `save`
163
+ user.update email: "petr@inter.com" # do not calls callbacks, and reloads all fields
104
164
 
105
165
  # field serialization to bson
106
166
  user.info = { 'bio' => "hi!", 'age' => 23, 'hobbies' => ['mufa', 'tuka'] }
@@ -123,4 +183,4 @@ in the tuple stored by Tarantool. By default, the primary key is field 0.
123
183
  * admin-socket protocol
124
184
  * safe to add fields to exist model
125
185
  * Hash, Array and lambdas as default values
126
- * timers to response, reconnect strategies
186
+ * timers to response
data/Rakefile CHANGED
@@ -1,5 +1,10 @@
1
1
  #!/usr/bin/env rake
2
- require "bundler/gem_tasks"
2
+ require "bundler/gem_helper"
3
+ Bundler::GemHelper.install_tasks name: 'tarantool'
4
+ namespace :record do
5
+ Bundler::GemHelper.install_tasks name: 'tarantool-record'
6
+ end
7
+
3
8
  require 'rake/testtask'
4
9
  Rake::TestTask.new do |i|
5
10
  i.options = '-v'
data/lib/tarantool.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  require 'eventmachine'
2
2
  require "iproto"
3
3
  require "tarantool/version"
4
+ require "tarantool/exceptions"
4
5
  require "tarantool/request"
5
6
  require "tarantool/response"
6
7
  require "tarantool/space_array.rb"
@@ -11,31 +12,74 @@ require "tarantool/serializers.rb"
11
12
  module Tarantool
12
13
  #autoload :Record, 'tarantool/record'
13
14
  #autoload :LightRecord, 'tarantool/light_record'
15
+ DEFAULT_PORT = 33013
14
16
 
15
17
  class << self
16
18
  def new(conf)
19
+ if conf[:host]
20
+ shards = [ [ _fix_connection(conf) ] ]
21
+ else
22
+ shards = conf[:servers]
23
+ unless shards.is_a? Array
24
+ shards = [ shards ]
25
+ end
26
+ unless shards.first.is_a? Array
27
+ shards = [ shards ]
28
+ end
29
+ shards = shards.map{|shard| shard.map{|server| _fix_connection(server)}}
30
+ end
31
+
32
+ replica_strategy = conf[:replica_strategy] || :round_robin
33
+ if %w{round_robin master_first}.include?(replica_strategy)
34
+ replica_strategy = replica_strategy.to_sym
35
+ end
36
+ unless [:round_robin, :master_first].include?(replica_strategy)
37
+ raise ArgumentError, "Shard strategy could be :round_robin or :master_first, got #{replica_strategy.inspect}"
38
+ end
39
+
40
+ previous_shards_count = conf[:previous_shards_count]
41
+ insert_to_previous_shard = conf[:insert_to_previous_shard]
42
+
17
43
  case conf[:type] || :block
18
44
  when :em, :em_fiber
19
45
  require 'tarantool/fiber_db'
20
- FiberDB.new(conf[:host], conf[:port])
46
+ FiberDB.new(shards, replica_strategy, previous_shards_count, insert_to_previous_shard)
21
47
  when :em_cb, :em_callback
22
48
  require 'tarantool/callback_db'
23
- CallbackDB.new(conf[:host], conf[:port])
49
+ CallbackDB.new(shards, replica_strategy, previous_shards_count, insert_to_previous_shard)
24
50
  when :block
25
51
  require 'tarantool/block_db'
26
- BlockDB.new(conf[:host], conf[:port])
52
+ BlockDB.new(shards, replica_strategy, previous_shards_count, insert_to_previous_shard)
53
+ else
54
+ raise "Unknown Tarantool connection type #{conf[:type]}"
55
+ end
56
+ end
57
+
58
+ private
59
+ def _fix_connection(conn)
60
+ if conn.is_a? Hash
61
+ conn = [conn[:host], conn[:port]].compact.join(':')
62
+ end
63
+ if conn.is_a? String
64
+ host, port = conn.split(':')
65
+ port ||= DEFAULT_PORT
66
+ conn = [host, port.to_i]
27
67
  end
68
+ raise ArgumentError, "Wrong connection declaration #{conn}" unless conn.is_a? Array
69
+ conn
28
70
  end
29
71
  end
30
72
 
31
73
  class DB
32
- attr_reader :closed, :connection
74
+ attr_reader :closed, :connections
33
75
  alias closed? closed
34
- def initialize(host, port)
35
- @host = host
36
- @port = port
76
+ def initialize(shards, replica_strategy, previous_shards_count, insert_to_previous_shard)
77
+ @shards = shards
78
+ @replica_strategy = replica_strategy
79
+ @previous_shards_count = previous_shards_count
80
+ @insert_to_previous_shard = insert_to_previous_shard
81
+ @connections = {}
37
82
  @closed = false
38
- establish_connection
39
83
  end
40
84
 
41
85
  # returns regular space, where fields are named by position
@@ -43,14 +87,29 @@ module Tarantool
43
87
  # tarantool.space_block(0, [:int, :str, :int, :str], keys: [[0], [1,2]])
44
88
  def space_array(space_no, field_types = [], opts = {})
45
89
  indexes = opts[:keys] || opts[:indexes]
46
- self.class::SpaceArray.new(self, space_no, field_types, indexes)
90
+ shard_fields = opts[:shard_fields]
91
+ shard_proc = opts[:shard_proc]
92
+ self.class::SpaceArray.new(self, space_no, field_types, indexes,
93
+ shard_fields, shard_proc)
94
+ end
95
+
96
+ def space(space_no, fields = [], opts = {})
97
+ case fields
98
+ when Array
99
+ space_array(space_no, fields, opts)
100
+ when Hash
101
+ space_hash(space_no, fields, opts)
102
+ else
103
+ raise "You should specify fields as an array or hash (got #{fields.inspect})"
104
+ end
47
105
  end
48
- # alias space_array to space for backward compatibility
49
- alias space space_array
50
106
 
51
107
  def space_hash(space_no, fields, opts = {})
52
108
  indexes = opts[:keys] || opts[:indexes]
53
- self.class::SpaceHash.new(self, space_no, fields, indexes)
109
+ shard_fields = opts[:shard_fields]
110
+ shard_proc = opts[:shard_proc]
111
+ self.class::SpaceHash.new(self, space_no, fields, indexes,
112
+ shard_fields, shard_proc)
54
113
  end
55
114
 
56
115
  def query
@@ -70,17 +129,33 @@ module Tarantool
70
129
  close_connection
71
130
  end
72
131
 
73
- def establish_connection
74
- raise NoMethodError, "#establish_connection should be redefined"
132
+ def shards_count
133
+ @shards.count
75
134
  end
76
135
 
77
- def close_connection
78
- raise NoMethodError, "#close_connection should be redefined"
136
+ attr_reader :previous_shards_count
137
+
138
+ def insert_with_shards_count
139
+ @insert_to_previous_shard && @previous_shards_count || @shards.count
79
140
  end
80
141
 
81
- def _send_request(request_type, body, cb)
82
- raise NoMethodError, "#_send_request should be redefined"
142
+ def _shard(number)
143
+ @connections[number] ||= begin
144
+ @shards[number].map do |host, port|
145
+ IProto.get_connection(host, port, self.class::IPROTO_CONNECTION_TYPE)
146
+ end
147
+ end
148
+ end
149
+
150
+ def close_connection
151
+ @connections.each do |number, replicas|
152
+ replicas.each(&:close)
153
+ end
154
+ @connections.clear
83
155
  end
84
156
 
157
+ def primary_interface
158
+ raise NoMethodError, "#primary_interface should by overriden"
159
+ end
85
160
  end
86
161
  end
@@ -1,30 +1,66 @@
1
1
  require 'tarantool'
2
2
  require 'tarantool/record/select'
3
- require 'active_support/core_ext/class/attribute'
3
+ require 'tarantool/core-ext'
4
4
 
5
5
  module Tarantool
6
6
  class RecordError < StandardError; end
7
7
  class UpdateNewRecord < RecordError; end
8
8
 
9
9
  class BaseRecord
10
- class_attribute :fields, instance_reader: false, instance_writer: false
10
+ extend ::Tarantool::ClassAttribute
11
+ t_class_attribute :fields
11
12
  self.fields = {}.freeze
12
13
 
13
- class_attribute :default_values, instance_reader: false, instance_writer: false
14
+ t_class_attribute :default_values
14
15
  self.default_values = {}.freeze
15
16
 
16
- class_attribute :indexes, instance_reader: false, instance_writer: false
17
+ t_class_attribute :indexes
17
18
  self.indexes = [].freeze
18
19
 
19
- class_attribute :space_no, instance_reader: false, instance_writer: false
20
- class_attribute :tarantool, instance_reader: false, instance_writer: false
20
+ t_class_attribute :_space_no
21
+ t_class_attribute :_tarantool
22
+
23
+ t_class_attribute :_shard_proc
24
+ t_class_attribute :_shard_fields
25
+ self._shard_proc = nil
26
+ self._shard_fields = nil
21
27
 
22
28
  class << self
23
- alias set_space_no space_no=
24
- alias set_tarantool tarantool=
29
+ alias set_shard_proc _shard_proc=
25
30
  end
26
31
 
27
32
  module ClassMethods
33
+ def tarantool(v=nil)
34
+ unless v
35
+ _tarantool
36
+ else
37
+ self.tarantool = v
38
+ end
39
+ end
40
+
41
+ def tarantool=(v)
42
+ reset_space!
43
+ unless ::Tarantool::DB === v && v.primary_interface == :synchronous
44
+ raise ArgumentError, "you may assing to record's tarantool only instances of Tarantool::BlockDB or Tarantool::FiberDB"
45
+ end
46
+ self._tarantool= v
47
+ end
48
+ alias set_tarantool tarantool=
49
+
50
+ def space_no(v=nil)
51
+ unless v
52
+ _space_no
53
+ else
54
+ self.space_no = v
55
+ end
56
+ end
57
+
58
+ def space_no=(v)
59
+ reset_space!
60
+ self._space_no = v
61
+ end
62
+ alias set_space_no space_no=
63
+
28
64
  def field(name, type, params = {})
29
65
  type = Serializers.check_type(type)
30
66
 
@@ -57,13 +93,50 @@ module Tarantool
57
93
  end
58
94
  end
59
95
 
96
+ def shard_proc(cb = nil, &block)
97
+ if cb ||= block
98
+ self._shard_proc = cb
99
+ else
100
+ _shard_proc
101
+ end
102
+ end
103
+
104
+ def shard_fields(*args)
105
+ if args.empty?
106
+ _shard_fields
107
+ else
108
+ self._shard_fields = args
109
+ end
110
+ end
111
+ alias set_shard_fields shard_fields
112
+
60
113
  def primary_index
61
114
  indexes[0]
62
115
  end
63
116
 
64
117
  def space
65
118
  @space ||= begin
66
- tarantool.space_hash(space_no, fields.dup, keys: indexes)
119
+ shard_fields = _shard_fields || primary_index
120
+ shard_proc = _shard_proc ||
121
+ if shard_fields.size == 1
122
+ case fields[shard_fields[0]]
123
+ when :int, :int16, :int8
124
+ :sumbur_murmur_fmix
125
+ when :int64
126
+ :sumbur_murmur_int64
127
+ when :string
128
+ :sumbur_murmur_str
129
+ else
130
+ :default
131
+ end
132
+ else
133
+ :default
134
+ end
135
+ _tarantool.space_hash(_space_no, fields.dup,
136
+ keys: indexes,
137
+ shard_fields: shard_fields,
138
+ shard_proc: shard_proc
139
+ )
67
140
  end
68
141
  end
69
142
 
@@ -75,6 +148,11 @@ module Tarantool
75
148
  end
76
149
  end
77
150
 
151
+ def reset_space!
152
+ @space = nil
153
+ @auto_space = nil
154
+ end
155
+
78
156
  def by_pk(pk)
79
157
  if Hash === (res = space.by_pk(pk))
80
158
  from_fetched(res)
@@ -171,6 +249,15 @@ module Tarantool
171
249
  end
172
250
  end
173
251
 
252
+ def store(hash, ret_tuple = false)
253
+ hash = default_values.merge(hash)
254
+ if ret_tuple
255
+ from_fetched space.store(hash, return_tuple: true)
256
+ else
257
+ space.store(hash)
258
+ end
259
+ end
260
+
174
261
  def update(pk, ops, ret_tuple=false)
175
262
  if ret_tuple
176
263
  from_fetched space.update(pk, ops, return_tuple: true)
@@ -187,7 +274,7 @@ module Tarantool
187
274
  end
188
275
  end
189
276
 
190
- %w{where limit offset}.each do |meth|
277
+ %w{where limit offset shard}.each do |meth|
191
278
  class_eval <<-"EOF", __FILE__, __LINE__
192
279
  def #{meth}(arg)
193
280
  select.#{meth}(arg)