elastictastic 0.5.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +1 -1
- data/README.md +161 -10
- data/lib/elastictastic/adapter.rb +84 -0
- data/lib/elastictastic/association.rb +6 -0
- data/lib/elastictastic/basic_document.rb +213 -0
- data/lib/elastictastic/bulk_persistence_strategy.rb +64 -19
- data/lib/elastictastic/callbacks.rb +18 -12
- data/lib/elastictastic/child_collection_proxy.rb +15 -11
- data/lib/elastictastic/client.rb +47 -24
- data/lib/elastictastic/configuration.rb +59 -4
- data/lib/elastictastic/dirty.rb +43 -28
- data/lib/elastictastic/discrete_persistence_strategy.rb +48 -23
- data/lib/elastictastic/document.rb +1 -85
- data/lib/elastictastic/embedded_document.rb +34 -0
- data/lib/elastictastic/errors.rb +17 -5
- data/lib/elastictastic/field.rb +3 -0
- data/lib/elastictastic/mass_assignment_security.rb +2 -4
- data/lib/elastictastic/middleware.rb +66 -84
- data/lib/elastictastic/multi_get.rb +30 -0
- data/lib/elastictastic/multi_search.rb +70 -0
- data/lib/elastictastic/nested_document.rb +3 -27
- data/lib/elastictastic/new_relic_instrumentation.rb +8 -8
- data/lib/elastictastic/observing.rb +8 -6
- data/lib/elastictastic/optimistic_locking.rb +57 -0
- data/lib/elastictastic/parent_child.rb +56 -54
- data/lib/elastictastic/persistence.rb +16 -16
- data/lib/elastictastic/properties.rb +136 -96
- data/lib/elastictastic/railtie.rb +1 -1
- data/lib/elastictastic/rotor.rb +105 -0
- data/lib/elastictastic/scope.rb +186 -56
- data/lib/elastictastic/server_error.rb +20 -1
- data/lib/elastictastic/test_helpers.rb +152 -97
- data/lib/elastictastic/thrift/constants.rb +12 -0
- data/lib/elastictastic/thrift/rest.rb +83 -0
- data/lib/elastictastic/thrift/types.rb +124 -0
- data/lib/elastictastic/thrift_adapter.rb +61 -0
- data/lib/elastictastic/transport_methods.rb +27 -0
- data/lib/elastictastic/validations.rb +11 -13
- data/lib/elastictastic/version.rb +1 -1
- data/lib/elastictastic.rb +148 -27
- data/spec/environment.rb +1 -1
- data/spec/examples/bulk_persistence_strategy_spec.rb +151 -23
- data/spec/examples/callbacks_spec.rb +65 -34
- data/spec/examples/dirty_spec.rb +160 -1
- data/spec/examples/document_spec.rb +168 -106
- data/spec/examples/middleware_spec.rb +1 -61
- data/spec/examples/multi_get_spec.rb +127 -0
- data/spec/examples/multi_search_spec.rb +113 -0
- data/spec/examples/observing_spec.rb +24 -3
- data/spec/examples/optimistic_locking_spec.rb +417 -0
- data/spec/examples/parent_child_spec.rb +73 -33
- data/spec/examples/properties_spec.rb +53 -0
- data/spec/examples/rotor_spec.rb +132 -0
- data/spec/examples/scope_spec.rb +78 -18
- data/spec/examples/search_spec.rb +26 -0
- data/spec/examples/validation_spec.rb +7 -1
- data/spec/models/author.rb +1 -1
- data/spec/models/blog.rb +2 -0
- data/spec/models/comment.rb +1 -1
- data/spec/models/photo.rb +9 -0
- data/spec/models/post.rb +3 -0
- metadata +97 -78
- data/lib/elastictastic/resource.rb +0 -4
- data/spec/examples/active_model_lint_spec.rb +0 -20
@@ -32,23 +32,23 @@ module Elastictastic
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def all_fields
|
35
|
-
@
|
35
|
+
@_all_fields ||= {}.tap do |fields|
|
36
36
|
each_field { |field, properties| fields[field] = properties }
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
40
|
def field_properties
|
41
|
-
@
|
41
|
+
@_field_properties ||= {}
|
42
42
|
end
|
43
43
|
|
44
44
|
def properties
|
45
|
-
return @
|
46
|
-
@
|
47
|
-
@
|
45
|
+
return @_properties if defined? @_properties
|
46
|
+
@_properties = {}
|
47
|
+
@_properties.merge!(field_properties)
|
48
48
|
embeds.each_pair do |name, embed|
|
49
|
-
@
|
49
|
+
@_properties[name] = { 'properties' => embed.clazz.properties }
|
50
50
|
end
|
51
|
-
@
|
51
|
+
@_properties
|
52
52
|
end
|
53
53
|
|
54
54
|
def properties_for_field(field_name)
|
@@ -56,7 +56,7 @@ module Elastictastic
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def embeds
|
59
|
-
@
|
59
|
+
@_embeds ||= {}
|
60
60
|
end
|
61
61
|
|
62
62
|
def field(*field_names, &block)
|
@@ -66,6 +66,27 @@ module Elastictastic
|
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
69
|
+
def boost(field, options = {})
|
70
|
+
@_boost = { 'name' => field.to_s, 'null_value' => 1.0 }.merge(options.stringify_keys)
|
71
|
+
end
|
72
|
+
|
73
|
+
def route_with(field, options = {})
|
74
|
+
@_routing_field = field
|
75
|
+
@_routing_required = !!options[:required]
|
76
|
+
end
|
77
|
+
|
78
|
+
def routing_required?
|
79
|
+
!!@_routing_required
|
80
|
+
end
|
81
|
+
|
82
|
+
def route(instance)
|
83
|
+
if @_routing_field
|
84
|
+
@_routing_field.to_s.split('.').inject(instance) do |obj, attr|
|
85
|
+
Util.call_or_map(obj) { |el| el.__send__(attr) } if obj
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
69
90
|
def define_field(field_name, options, &block)
|
70
91
|
field_name = field_name.to_s
|
71
92
|
|
@@ -114,122 +135,141 @@ module Elastictastic
|
|
114
135
|
end
|
115
136
|
end
|
116
137
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
end
|
138
|
+
def initialize(attributes = {})
|
139
|
+
super()
|
140
|
+
@_attributes = {}
|
141
|
+
@_embeds = {}
|
142
|
+
self.attributes = attributes
|
143
|
+
end
|
124
144
|
|
125
|
-
|
126
|
-
|
145
|
+
def attributes
|
146
|
+
super.merge(@_attributes).with_indifferent_access
|
147
|
+
end
|
148
|
+
|
149
|
+
def attributes=(attributes)
|
150
|
+
attributes.each_pair do |field, value|
|
151
|
+
__send__(:"#{field}=", value)
|
127
152
|
end
|
153
|
+
end
|
128
154
|
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
155
|
+
def inspect
|
156
|
+
inspected = "#<#{self.class.name}"
|
157
|
+
if attributes.any?
|
158
|
+
inspected << ' ' << attributes.each_pair.map do |attr, value|
|
159
|
+
"#{attr}: #{value.inspect}"
|
160
|
+
end.join(', ')
|
133
161
|
end
|
162
|
+
inspected << '>'
|
163
|
+
end
|
134
164
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
end
|
165
|
+
def elasticsearch_doc
|
166
|
+
{}.tap do |doc|
|
167
|
+
@_attributes.each_pair do |field, value|
|
168
|
+
next if value.nil?
|
169
|
+
doc[field] = Util.call_or_map(value) do |item|
|
170
|
+
serialize_value(field, item)
|
142
171
|
end
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
172
|
+
end
|
173
|
+
@_embeds.each_pair do |field, embedded|
|
174
|
+
next if embedded.nil?
|
175
|
+
doc[field] = Util.call_or_map(embedded) do |item|
|
176
|
+
item.elasticsearch_doc
|
148
177
|
end
|
149
178
|
end
|
150
179
|
end
|
180
|
+
end
|
151
181
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
end
|
166
|
-
write_attribute(field_name, deserialized)
|
182
|
+
def elasticsearch_doc=(doc)
|
183
|
+
return if doc.nil?
|
184
|
+
doc.each_pair do |field_name, value|
|
185
|
+
if self.class.properties.has_key?(field_name)
|
186
|
+
embed = self.class.embeds[field_name]
|
187
|
+
if embed
|
188
|
+
embedded = Util.call_or_map(value) do |item|
|
189
|
+
embed.clazz.new.tap { |e| e.elasticsearch_doc = item }
|
190
|
+
end
|
191
|
+
write_embed(field_name, embedded)
|
192
|
+
else
|
193
|
+
deserialized = Util.call_or_map(value) do |item|
|
194
|
+
deserialize_value(field_name, item)
|
167
195
|
end
|
196
|
+
write_attribute(field_name, deserialized)
|
168
197
|
end
|
169
198
|
end
|
170
199
|
end
|
200
|
+
end
|
171
201
|
|
172
|
-
|
202
|
+
def _routing
|
203
|
+
end
|
173
204
|
|
174
|
-
|
175
|
-
@attributes[field.to_s]
|
176
|
-
end
|
205
|
+
protected
|
177
206
|
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
else
|
182
|
-
@attributes[field.to_s] = value
|
183
|
-
end
|
184
|
-
end
|
207
|
+
def read_attribute(field)
|
208
|
+
@_attributes[field.to_s]
|
209
|
+
end
|
185
210
|
|
186
|
-
|
187
|
-
|
211
|
+
def write_attribute(field, value)
|
212
|
+
if value.nil?
|
213
|
+
@_attributes.delete(field.to_s)
|
214
|
+
else
|
215
|
+
@_attributes[field.to_s] = value
|
188
216
|
end
|
217
|
+
end
|
189
218
|
|
190
|
-
|
191
|
-
|
192
|
-
|
219
|
+
def read_attributes
|
220
|
+
@_attributes
|
221
|
+
end
|
193
222
|
|
194
|
-
|
195
|
-
|
196
|
-
|
223
|
+
def read_embeds
|
224
|
+
@_embeds
|
225
|
+
end
|
197
226
|
|
198
|
-
|
199
|
-
|
200
|
-
|
227
|
+
def write_attributes(attributes)
|
228
|
+
@_attributes = attributes
|
229
|
+
end
|
201
230
|
|
202
|
-
|
231
|
+
def write_embeds(embeds)
|
232
|
+
@_embeds = embeds
|
233
|
+
end
|
203
234
|
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
235
|
+
def read_embed(field)
|
236
|
+
@_embeds[field.to_s]
|
237
|
+
end
|
238
|
+
|
239
|
+
def write_embed(field, value)
|
240
|
+
@_embeds[field.to_s] = value
|
241
|
+
end
|
242
|
+
|
243
|
+
private
|
244
|
+
|
245
|
+
def serialize_value(field_name, value)
|
246
|
+
type = self.class.properties_for_field(field_name)['type'].to_s
|
247
|
+
case type
|
248
|
+
when 'date'
|
249
|
+
time = value.to_time
|
250
|
+
time.to_i * 1000 + time.usec / 1000
|
251
|
+
when 'integer', 'byte', 'short', 'long'
|
252
|
+
value.to_i
|
253
|
+
when 'float', 'double'
|
254
|
+
value.to_f
|
255
|
+
when 'boolean'
|
256
|
+
!!value
|
257
|
+
else
|
258
|
+
value
|
219
259
|
end
|
260
|
+
end
|
220
261
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
else
|
228
|
-
Time.parse(value)
|
229
|
-
end
|
262
|
+
def deserialize_value(field_name, value)
|
263
|
+
return nil if value.nil?
|
264
|
+
if self.class.properties_for_field(field_name)['type'].to_s == 'date'
|
265
|
+
if value.is_a? Fixnum
|
266
|
+
sec, usec = value / 1000, (value % 1000) * 1000
|
267
|
+
Time.at(sec, usec).utc
|
230
268
|
else
|
231
|
-
value
|
269
|
+
Time.parse(value)
|
232
270
|
end
|
271
|
+
else
|
272
|
+
value
|
233
273
|
end
|
234
274
|
end
|
235
275
|
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'elastictastic/transport_methods'
|
2
|
+
|
3
|
+
module Elastictastic
|
4
|
+
|
5
|
+
class Rotor
|
6
|
+
|
7
|
+
include TransportMethods
|
8
|
+
|
9
|
+
NodeUnavailable = Class.new(StandardError)
|
10
|
+
|
11
|
+
def initialize(hosts, options)
|
12
|
+
node_options = {}
|
13
|
+
[:backoff_threshold, :backoff_start, :backoff_max].each do |key|
|
14
|
+
node_options[key] = options.delete(key)
|
15
|
+
end
|
16
|
+
adapter = Adapter[options.delete(:adapter)]
|
17
|
+
@connections = hosts.map do |host|
|
18
|
+
Node.new(adapter.new(host, options), node_options)
|
19
|
+
end
|
20
|
+
@head_index = 0
|
21
|
+
end
|
22
|
+
|
23
|
+
def request(method, path, body = nil)
|
24
|
+
try_rotate { |node| node.request(method, path, body) }
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def peek
|
30
|
+
@connections[@head_index]
|
31
|
+
end
|
32
|
+
|
33
|
+
def shift
|
34
|
+
peek.tap { @head_index = (@head_index + 1) % @connections.length }
|
35
|
+
end
|
36
|
+
|
37
|
+
def try_rotate
|
38
|
+
last = peek
|
39
|
+
begin
|
40
|
+
yield shift
|
41
|
+
rescue ConnectionFailed, NodeUnavailable => e
|
42
|
+
raise NoServerAvailable, e if peek == last
|
43
|
+
retry
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class Node
|
48
|
+
def initialize(connection, options)
|
49
|
+
@connection = connection
|
50
|
+
@failures = 0
|
51
|
+
@backoff_threshold = options[:backoff_threshold] || 0
|
52
|
+
@backoff_start = options[:backoff_start]
|
53
|
+
@backoff_max = options[:backoff_max]
|
54
|
+
end
|
55
|
+
|
56
|
+
def request(method, path, body = nil)
|
57
|
+
try_track { @connection.request(method, path, body).tap { succeeded! }}
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def try_track
|
63
|
+
raise NodeUnavailable, "Won't retry this node until #{@back_off_until}" unless available?
|
64
|
+
begin
|
65
|
+
yield
|
66
|
+
rescue ConnectionFailed => e
|
67
|
+
failed!
|
68
|
+
raise e
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def available?
|
73
|
+
!backoff_failures_reached? ||
|
74
|
+
!backing_off?
|
75
|
+
end
|
76
|
+
|
77
|
+
def backing_off?
|
78
|
+
@back_off_until &&
|
79
|
+
@back_off_until > Time.now
|
80
|
+
end
|
81
|
+
|
82
|
+
def backoff_failures_reached?
|
83
|
+
@failures >= @backoff_threshold
|
84
|
+
end
|
85
|
+
|
86
|
+
def succeeded!
|
87
|
+
@failures = 0
|
88
|
+
@back_off_until = nil
|
89
|
+
end
|
90
|
+
|
91
|
+
def failed!
|
92
|
+
@failures += 1
|
93
|
+
if @backoff_start && backoff_failures_reached?
|
94
|
+
backoff_count = @failures - @backoff_threshold
|
95
|
+
backoff_interval = @backoff_start * 2 ** backoff_count
|
96
|
+
backoff_interval = @backoff_max if @backoff_max &&
|
97
|
+
backoff_interval > @backoff_max
|
98
|
+
@back_off_until = Time.now + backoff_interval
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
104
|
+
|
105
|
+
end
|