elastictastic 0.5.0 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. data/LICENSE +1 -1
  2. data/README.md +161 -10
  3. data/lib/elastictastic/adapter.rb +84 -0
  4. data/lib/elastictastic/association.rb +6 -0
  5. data/lib/elastictastic/basic_document.rb +213 -0
  6. data/lib/elastictastic/bulk_persistence_strategy.rb +64 -19
  7. data/lib/elastictastic/callbacks.rb +18 -12
  8. data/lib/elastictastic/child_collection_proxy.rb +15 -11
  9. data/lib/elastictastic/client.rb +47 -24
  10. data/lib/elastictastic/configuration.rb +59 -4
  11. data/lib/elastictastic/dirty.rb +43 -28
  12. data/lib/elastictastic/discrete_persistence_strategy.rb +48 -23
  13. data/lib/elastictastic/document.rb +1 -85
  14. data/lib/elastictastic/embedded_document.rb +34 -0
  15. data/lib/elastictastic/errors.rb +17 -5
  16. data/lib/elastictastic/field.rb +3 -0
  17. data/lib/elastictastic/mass_assignment_security.rb +2 -4
  18. data/lib/elastictastic/middleware.rb +66 -84
  19. data/lib/elastictastic/multi_get.rb +30 -0
  20. data/lib/elastictastic/multi_search.rb +70 -0
  21. data/lib/elastictastic/nested_document.rb +3 -27
  22. data/lib/elastictastic/new_relic_instrumentation.rb +8 -8
  23. data/lib/elastictastic/observing.rb +8 -6
  24. data/lib/elastictastic/optimistic_locking.rb +57 -0
  25. data/lib/elastictastic/parent_child.rb +56 -54
  26. data/lib/elastictastic/persistence.rb +16 -16
  27. data/lib/elastictastic/properties.rb +136 -96
  28. data/lib/elastictastic/railtie.rb +1 -1
  29. data/lib/elastictastic/rotor.rb +105 -0
  30. data/lib/elastictastic/scope.rb +186 -56
  31. data/lib/elastictastic/server_error.rb +20 -1
  32. data/lib/elastictastic/test_helpers.rb +152 -97
  33. data/lib/elastictastic/thrift/constants.rb +12 -0
  34. data/lib/elastictastic/thrift/rest.rb +83 -0
  35. data/lib/elastictastic/thrift/types.rb +124 -0
  36. data/lib/elastictastic/thrift_adapter.rb +61 -0
  37. data/lib/elastictastic/transport_methods.rb +27 -0
  38. data/lib/elastictastic/validations.rb +11 -13
  39. data/lib/elastictastic/version.rb +1 -1
  40. data/lib/elastictastic.rb +148 -27
  41. data/spec/environment.rb +1 -1
  42. data/spec/examples/bulk_persistence_strategy_spec.rb +151 -23
  43. data/spec/examples/callbacks_spec.rb +65 -34
  44. data/spec/examples/dirty_spec.rb +160 -1
  45. data/spec/examples/document_spec.rb +168 -106
  46. data/spec/examples/middleware_spec.rb +1 -61
  47. data/spec/examples/multi_get_spec.rb +127 -0
  48. data/spec/examples/multi_search_spec.rb +113 -0
  49. data/spec/examples/observing_spec.rb +24 -3
  50. data/spec/examples/optimistic_locking_spec.rb +417 -0
  51. data/spec/examples/parent_child_spec.rb +73 -33
  52. data/spec/examples/properties_spec.rb +53 -0
  53. data/spec/examples/rotor_spec.rb +132 -0
  54. data/spec/examples/scope_spec.rb +78 -18
  55. data/spec/examples/search_spec.rb +26 -0
  56. data/spec/examples/validation_spec.rb +7 -1
  57. data/spec/models/author.rb +1 -1
  58. data/spec/models/blog.rb +2 -0
  59. data/spec/models/comment.rb +1 -1
  60. data/spec/models/photo.rb +9 -0
  61. data/spec/models/post.rb +3 -0
  62. metadata +97 -78
  63. data/lib/elastictastic/resource.rb +0 -4
  64. data/spec/examples/active_model_lint_spec.rb +0 -20
@@ -32,23 +32,23 @@ module Elastictastic
32
32
  end
33
33
 
34
34
  def all_fields
35
- @all_fields ||= {}.tap do |fields|
35
+ @_all_fields ||= {}.tap do |fields|
36
36
  each_field { |field, properties| fields[field] = properties }
37
37
  end
38
38
  end
39
39
 
40
40
  def field_properties
41
- @field_properties ||= {}
41
+ @_field_properties ||= {}
42
42
  end
43
43
 
44
44
  def properties
45
- return @properties if defined? @properties
46
- @properties = {}
47
- @properties.merge!(field_properties)
45
+ return @_properties if defined? @_properties
46
+ @_properties = {}
47
+ @_properties.merge!(field_properties)
48
48
  embeds.each_pair do |name, embed|
49
- @properties[name] = { 'properties' => embed.clazz.properties }
49
+ @_properties[name] = { 'properties' => embed.clazz.properties }
50
50
  end
51
- @properties
51
+ @_properties
52
52
  end
53
53
 
54
54
  def properties_for_field(field_name)
@@ -56,7 +56,7 @@ module Elastictastic
56
56
  end
57
57
 
58
58
  def embeds
59
- @embeds ||= {}
59
+ @_embeds ||= {}
60
60
  end
61
61
 
62
62
  def field(*field_names, &block)
@@ -66,6 +66,27 @@ module Elastictastic
66
66
  end
67
67
  end
68
68
 
69
+ def boost(field, options = {})
70
+ @_boost = { 'name' => field.to_s, 'null_value' => 1.0 }.merge(options.stringify_keys)
71
+ end
72
+
73
+ def route_with(field, options = {})
74
+ @_routing_field = field
75
+ @_routing_required = !!options[:required]
76
+ end
77
+
78
+ def routing_required?
79
+ !!@_routing_required
80
+ end
81
+
82
+ def route(instance)
83
+ if @_routing_field
84
+ @_routing_field.to_s.split('.').inject(instance) do |obj, attr|
85
+ Util.call_or_map(obj) { |el| el.__send__(attr) } if obj
86
+ end
87
+ end
88
+ end
89
+
69
90
  def define_field(field_name, options, &block)
70
91
  field_name = field_name.to_s
71
92
 
@@ -114,122 +135,141 @@ module Elastictastic
114
135
  end
115
136
  end
116
137
 
117
- module InstanceMethods
118
- def initialize(attributes = {})
119
- super
120
- @attributes = {}
121
- @embeds = {}
122
- self.attributes = attributes
123
- end
138
+ def initialize(attributes = {})
139
+ super()
140
+ @_attributes = {}
141
+ @_embeds = {}
142
+ self.attributes = attributes
143
+ end
124
144
 
125
- def attributes
126
- @attributes.with_indifferent_access
145
+ def attributes
146
+ super.merge(@_attributes).with_indifferent_access
147
+ end
148
+
149
+ def attributes=(attributes)
150
+ attributes.each_pair do |field, value|
151
+ __send__(:"#{field}=", value)
127
152
  end
153
+ end
128
154
 
129
- def attributes=(attributes)
130
- attributes.each_pair do |field, value|
131
- __send__(:"#{field}=", value)
132
- end
155
+ def inspect
156
+ inspected = "#<#{self.class.name}"
157
+ if attributes.any?
158
+ inspected << ' ' << attributes.each_pair.map do |attr, value|
159
+ "#{attr}: #{value.inspect}"
160
+ end.join(', ')
133
161
  end
162
+ inspected << '>'
163
+ end
134
164
 
135
- def elasticsearch_doc
136
- {}.tap do |doc|
137
- @attributes.each_pair do |field, value|
138
- next if value.nil?
139
- doc[field] = Util.call_or_map(value) do |item|
140
- serialize_value(field, item)
141
- end
165
+ def elasticsearch_doc
166
+ {}.tap do |doc|
167
+ @_attributes.each_pair do |field, value|
168
+ next if value.nil?
169
+ doc[field] = Util.call_or_map(value) do |item|
170
+ serialize_value(field, item)
142
171
  end
143
- @embeds.each_pair do |field, embedded|
144
- next if embedded.nil?
145
- doc[field] = Util.call_or_map(embedded) do |item|
146
- item.elasticsearch_doc
147
- end
172
+ end
173
+ @_embeds.each_pair do |field, embedded|
174
+ next if embedded.nil?
175
+ doc[field] = Util.call_or_map(embedded) do |item|
176
+ item.elasticsearch_doc
148
177
  end
149
178
  end
150
179
  end
180
+ end
151
181
 
152
- def elasticsearch_doc=(doc)
153
- return if doc.nil?
154
- doc.each_pair do |field_name, value|
155
- if self.class.properties.has_key?(field_name)
156
- embed = self.class.embeds[field_name]
157
- if embed
158
- embedded = Util.call_or_map(value) do |item|
159
- embed.clazz.new.tap { |e| e.elasticsearch_doc = item }
160
- end
161
- write_embed(field_name, embedded)
162
- else
163
- deserialized = Util.call_or_map(value) do |item|
164
- deserialize_value(field_name, item)
165
- end
166
- write_attribute(field_name, deserialized)
182
+ def elasticsearch_doc=(doc)
183
+ return if doc.nil?
184
+ doc.each_pair do |field_name, value|
185
+ if self.class.properties.has_key?(field_name)
186
+ embed = self.class.embeds[field_name]
187
+ if embed
188
+ embedded = Util.call_or_map(value) do |item|
189
+ embed.clazz.new.tap { |e| e.elasticsearch_doc = item }
190
+ end
191
+ write_embed(field_name, embedded)
192
+ else
193
+ deserialized = Util.call_or_map(value) do |item|
194
+ deserialize_value(field_name, item)
167
195
  end
196
+ write_attribute(field_name, deserialized)
168
197
  end
169
198
  end
170
199
  end
200
+ end
171
201
 
172
- protected
202
+ def _routing
203
+ end
173
204
 
174
- def read_attribute(field)
175
- @attributes[field.to_s]
176
- end
205
+ protected
177
206
 
178
- def write_attribute(field, value)
179
- if value.nil?
180
- @attributes.delete(field.to_s)
181
- else
182
- @attributes[field.to_s] = value
183
- end
184
- end
207
+ def read_attribute(field)
208
+ @_attributes[field.to_s]
209
+ end
185
210
 
186
- def read_attributes
187
- @attributes
211
+ def write_attribute(field, value)
212
+ if value.nil?
213
+ @_attributes.delete(field.to_s)
214
+ else
215
+ @_attributes[field.to_s] = value
188
216
  end
217
+ end
189
218
 
190
- def write_attributes(attributes)
191
- @attributes = attributes
192
- end
219
+ def read_attributes
220
+ @_attributes
221
+ end
193
222
 
194
- def read_embed(field)
195
- @embeds[field.to_s]
196
- end
223
+ def read_embeds
224
+ @_embeds
225
+ end
197
226
 
198
- def write_embed(field, value)
199
- @embeds[field.to_s] = value
200
- end
227
+ def write_attributes(attributes)
228
+ @_attributes = attributes
229
+ end
201
230
 
202
- private
231
+ def write_embeds(embeds)
232
+ @_embeds = embeds
233
+ end
203
234
 
204
- def serialize_value(field_name, value)
205
- type = self.class.properties_for_field(field_name)['type'].to_s
206
- case type
207
- when 'date'
208
- time = value.to_time
209
- time.to_i * 1000 + time.usec / 1000
210
- when 'integer', 'byte', 'short', 'long'
211
- value.to_i
212
- when 'float', 'double'
213
- value.to_f
214
- when 'boolean'
215
- !!value
216
- else
217
- value
218
- end
235
+ def read_embed(field)
236
+ @_embeds[field.to_s]
237
+ end
238
+
239
+ def write_embed(field, value)
240
+ @_embeds[field.to_s] = value
241
+ end
242
+
243
+ private
244
+
245
+ def serialize_value(field_name, value)
246
+ type = self.class.properties_for_field(field_name)['type'].to_s
247
+ case type
248
+ when 'date'
249
+ time = value.to_time
250
+ time.to_i * 1000 + time.usec / 1000
251
+ when 'integer', 'byte', 'short', 'long'
252
+ value.to_i
253
+ when 'float', 'double'
254
+ value.to_f
255
+ when 'boolean'
256
+ !!value
257
+ else
258
+ value
219
259
  end
260
+ end
220
261
 
221
- def deserialize_value(field_name, value)
222
- return nil if value.nil?
223
- if self.class.properties_for_field(field_name)['type'].to_s == 'date'
224
- if value.is_a? Fixnum
225
- sec, usec = value / 1000, (value % 1000) * 1000
226
- Time.at(sec, usec).utc
227
- else
228
- Time.parse(value)
229
- end
262
+ def deserialize_value(field_name, value)
263
+ return nil if value.nil?
264
+ if self.class.properties_for_field(field_name)['type'].to_s == 'date'
265
+ if value.is_a? Fixnum
266
+ sec, usec = value / 1000, (value % 1000) * 1000
267
+ Time.at(sec, usec).utc
230
268
  else
231
- value
269
+ Time.parse(value)
232
270
  end
271
+ else
272
+ value
233
273
  end
234
274
  end
235
275
  end
@@ -27,7 +27,7 @@ module Elastictastic
27
27
  ::Elastictastic::Observing.instantiate_observers
28
28
 
29
29
  ActionDispatch::Callbacks.to_prepare do
30
- ::Elastictastic.instantiate_observers
30
+ ::Elastictastic::Observing.instantiate_observers
31
31
  end
32
32
  end
33
33
  end
@@ -0,0 +1,105 @@
1
+ require 'elastictastic/transport_methods'
2
+
3
+ module Elastictastic
4
+
5
+ class Rotor
6
+
7
+ include TransportMethods
8
+
9
+ NodeUnavailable = Class.new(StandardError)
10
+
11
+ def initialize(hosts, options)
12
+ node_options = {}
13
+ [:backoff_threshold, :backoff_start, :backoff_max].each do |key|
14
+ node_options[key] = options.delete(key)
15
+ end
16
+ adapter = Adapter[options.delete(:adapter)]
17
+ @connections = hosts.map do |host|
18
+ Node.new(adapter.new(host, options), node_options)
19
+ end
20
+ @head_index = 0
21
+ end
22
+
23
+ def request(method, path, body = nil)
24
+ try_rotate { |node| node.request(method, path, body) }
25
+ end
26
+
27
+ private
28
+
29
+ def peek
30
+ @connections[@head_index]
31
+ end
32
+
33
+ def shift
34
+ peek.tap { @head_index = (@head_index + 1) % @connections.length }
35
+ end
36
+
37
+ def try_rotate
38
+ last = peek
39
+ begin
40
+ yield shift
41
+ rescue ConnectionFailed, NodeUnavailable => e
42
+ raise NoServerAvailable, e if peek == last
43
+ retry
44
+ end
45
+ end
46
+
47
+ class Node
48
+ def initialize(connection, options)
49
+ @connection = connection
50
+ @failures = 0
51
+ @backoff_threshold = options[:backoff_threshold] || 0
52
+ @backoff_start = options[:backoff_start]
53
+ @backoff_max = options[:backoff_max]
54
+ end
55
+
56
+ def request(method, path, body = nil)
57
+ try_track { @connection.request(method, path, body).tap { succeeded! }}
58
+ end
59
+
60
+ private
61
+
62
+ def try_track
63
+ raise NodeUnavailable, "Won't retry this node until #{@back_off_until}" unless available?
64
+ begin
65
+ yield
66
+ rescue ConnectionFailed => e
67
+ failed!
68
+ raise e
69
+ end
70
+ end
71
+
72
+ def available?
73
+ !backoff_failures_reached? ||
74
+ !backing_off?
75
+ end
76
+
77
+ def backing_off?
78
+ @back_off_until &&
79
+ @back_off_until > Time.now
80
+ end
81
+
82
+ def backoff_failures_reached?
83
+ @failures >= @backoff_threshold
84
+ end
85
+
86
+ def succeeded!
87
+ @failures = 0
88
+ @back_off_until = nil
89
+ end
90
+
91
+ def failed!
92
+ @failures += 1
93
+ if @backoff_start && backoff_failures_reached?
94
+ backoff_count = @failures - @backoff_threshold
95
+ backoff_interval = @backoff_start * 2 ** backoff_count
96
+ backoff_interval = @backoff_max if @backoff_max &&
97
+ backoff_interval > @backoff_max
98
+ @back_off_until = Time.now + backoff_interval
99
+ end
100
+ end
101
+ end
102
+
103
+ end
104
+
105
+ end