elastictastic 0.5.0 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. data/LICENSE +1 -1
  2. data/README.md +161 -10
  3. data/lib/elastictastic/adapter.rb +84 -0
  4. data/lib/elastictastic/association.rb +6 -0
  5. data/lib/elastictastic/basic_document.rb +213 -0
  6. data/lib/elastictastic/bulk_persistence_strategy.rb +64 -19
  7. data/lib/elastictastic/callbacks.rb +18 -12
  8. data/lib/elastictastic/child_collection_proxy.rb +15 -11
  9. data/lib/elastictastic/client.rb +47 -24
  10. data/lib/elastictastic/configuration.rb +59 -4
  11. data/lib/elastictastic/dirty.rb +43 -28
  12. data/lib/elastictastic/discrete_persistence_strategy.rb +48 -23
  13. data/lib/elastictastic/document.rb +1 -85
  14. data/lib/elastictastic/embedded_document.rb +34 -0
  15. data/lib/elastictastic/errors.rb +17 -5
  16. data/lib/elastictastic/field.rb +3 -0
  17. data/lib/elastictastic/mass_assignment_security.rb +2 -4
  18. data/lib/elastictastic/middleware.rb +66 -84
  19. data/lib/elastictastic/multi_get.rb +30 -0
  20. data/lib/elastictastic/multi_search.rb +70 -0
  21. data/lib/elastictastic/nested_document.rb +3 -27
  22. data/lib/elastictastic/new_relic_instrumentation.rb +8 -8
  23. data/lib/elastictastic/observing.rb +8 -6
  24. data/lib/elastictastic/optimistic_locking.rb +57 -0
  25. data/lib/elastictastic/parent_child.rb +56 -54
  26. data/lib/elastictastic/persistence.rb +16 -16
  27. data/lib/elastictastic/properties.rb +136 -96
  28. data/lib/elastictastic/railtie.rb +1 -1
  29. data/lib/elastictastic/rotor.rb +105 -0
  30. data/lib/elastictastic/scope.rb +186 -56
  31. data/lib/elastictastic/server_error.rb +20 -1
  32. data/lib/elastictastic/test_helpers.rb +152 -97
  33. data/lib/elastictastic/thrift/constants.rb +12 -0
  34. data/lib/elastictastic/thrift/rest.rb +83 -0
  35. data/lib/elastictastic/thrift/types.rb +124 -0
  36. data/lib/elastictastic/thrift_adapter.rb +61 -0
  37. data/lib/elastictastic/transport_methods.rb +27 -0
  38. data/lib/elastictastic/validations.rb +11 -13
  39. data/lib/elastictastic/version.rb +1 -1
  40. data/lib/elastictastic.rb +148 -27
  41. data/spec/environment.rb +1 -1
  42. data/spec/examples/bulk_persistence_strategy_spec.rb +151 -23
  43. data/spec/examples/callbacks_spec.rb +65 -34
  44. data/spec/examples/dirty_spec.rb +160 -1
  45. data/spec/examples/document_spec.rb +168 -106
  46. data/spec/examples/middleware_spec.rb +1 -61
  47. data/spec/examples/multi_get_spec.rb +127 -0
  48. data/spec/examples/multi_search_spec.rb +113 -0
  49. data/spec/examples/observing_spec.rb +24 -3
  50. data/spec/examples/optimistic_locking_spec.rb +417 -0
  51. data/spec/examples/parent_child_spec.rb +73 -33
  52. data/spec/examples/properties_spec.rb +53 -0
  53. data/spec/examples/rotor_spec.rb +132 -0
  54. data/spec/examples/scope_spec.rb +78 -18
  55. data/spec/examples/search_spec.rb +26 -0
  56. data/spec/examples/validation_spec.rb +7 -1
  57. data/spec/models/author.rb +1 -1
  58. data/spec/models/blog.rb +2 -0
  59. data/spec/models/comment.rb +1 -1
  60. data/spec/models/photo.rb +9 -0
  61. data/spec/models/post.rb +3 -0
  62. metadata +97 -78
  63. data/lib/elastictastic/resource.rb +0 -4
  64. data/spec/examples/active_model_lint_spec.rb +0 -20
@@ -1,37 +1,70 @@
1
- require 'faraday'
1
+ require 'elastictastic/transport_methods'
2
2
 
3
3
  module Elastictastic
4
+
4
5
  module Middleware
5
- class JsonEncodeBody < Faraday::Middleware
6
- def call(env)
7
- case env[:body]
6
+
7
+ class Base
8
+
9
+ include TransportMethods
10
+
11
+ def initialize(connection)
12
+ @connection = connection
13
+ end
14
+
15
+ def request(method, path, body = nil)
16
+ @connection.request(method, path, body)
17
+ end
18
+
19
+ end
20
+
21
+ class JsonEncodeBody < Base
22
+
23
+ def request(method, path, body = nil)
24
+ case body
8
25
  when String, nil
9
- # nothing
10
- else env[:body] = env[:body].to_json
26
+ super
27
+ else
28
+ @connection.request(
29
+ method, path,
30
+ Elastictastic.json_encode(body)
31
+ )
11
32
  end
12
- @app.call(env)
13
33
  end
34
+
14
35
  end
15
36
 
16
- class JsonDecodeResponse < Faraday::Middleware
17
- def call(env)
18
- @app.call(env).on_complete do
19
- env[:body] &&= JSON.parse(env[:body])
37
+ class JsonDecodeResponse < Base
38
+
39
+ def request(method, path, body = nil)
40
+ response = super
41
+ if response.body.present?
42
+ Adapter::Response.new(
43
+ response.status,
44
+ response.headers,
45
+ Elastictastic.json_decode(response.body)
46
+ )
47
+ else
48
+ response
20
49
  end
21
50
  end
51
+
22
52
  end
23
53
 
24
- class RaiseServerErrors < Faraday::Middleware
25
- ERROR_PATTERN = /^([A-Z][A-Za-z]*)(?::\s*)?(.*)$/
26
-
27
- def call(env)
28
- @app.call(env).on_complete do
29
- body = env[:body]
30
- if body['error']
31
- raise_error(body['error'], body['status'])
32
- elsif body['_shards'] && body['_shards']['failures']
33
- raise_error(
34
- body['_shards']['failures'].first['reason'], body['status'])
54
+ class RaiseServerErrors < Base
55
+
56
+ def request(method, path, body = nil)
57
+ super.tap do |response|
58
+ if method != :head
59
+ if response.body.nil?
60
+ raise Elastictastic::ServerError::ServerError,
61
+ "No body in ElasticSearch response with status #{env[:status]}"
62
+ elsif response.body['error']
63
+ raise_error(response.body['error'], response.body['status'])
64
+ elsif response.body['_shards'] && response.body['_shards']['failures']
65
+ raise_error(
66
+ response.body['_shards']['failures'].first['reason'], response.body['status'])
67
+ end
35
68
  end
36
69
  end
37
70
  end
@@ -39,81 +72,30 @@ module Elastictastic
39
72
  private
40
73
 
41
74
  def raise_error(server_message, status)
42
- match = ERROR_PATTERN.match(server_message)
43
- if match
44
- clazz = Elastictastic::ServerError.const_get(match[1])
45
- error = clazz.new(match[2])
46
- error.status = status
47
- Kernel.raise error
48
- else
49
- Kernel.raise Elastictastic::ServerError::ServerError, server_message
50
- end
75
+ ::Kernel.raise(Elastictastic::ServerError[server_message, status])
51
76
  end
77
+
52
78
  end
53
79
 
54
- class LogRequests < Faraday::Middleware
55
- def initialize(app, logger)
56
- super(app)
80
+ class LogRequests < Base
81
+
82
+ def initialize(connection, logger)
83
+ super(connection)
57
84
  @logger = logger
58
85
  end
59
86
 
60
- def call(env)
87
+ def request(method, path, body = nil)
61
88
  now = Time.now
62
- body = env[:body]
63
- @app.call(env).on_complete do
64
- method = env[:method].to_s.upcase
89
+ super.tap do
65
90
  time = ((Time.now - now) * 1000).to_i
66
- message = "ElasticSearch #{method} (#{time}ms) #{env[:url].path}"
91
+ message = "ElasticSearch #{method.to_s.upcase} (#{time}ms) #{path}"
67
92
  message << ' ' << body if body
68
93
  @logger.debug(message)
69
94
  end
70
95
  end
71
- end
72
-
73
- class Rotor < Faraday::Middleware
74
- def initialize(app, *hosts)
75
- first = nil
76
- hosts.each do |host|
77
- node = Node.new(app, host)
78
- first ||= node
79
- @head.next = node if @head
80
- @head = node
81
- end
82
- @head.next = first
83
- end
84
-
85
- def call(env)
86
- last = @head
87
- begin
88
- @head = @head.next
89
- @head.call(env)
90
- rescue Faraday::Error::ConnectionFailed => e
91
- raise NoServerAvailable if @head == last
92
- retry
93
- end
94
- end
95
-
96
- class Node < Faraday::Middleware
97
- attr_accessor :next
98
96
 
99
- def initialize(app, url)
100
- super(app)
101
- # Create a connection instance so we can use its #build_url method.
102
- # Kinda lame -- seems like it would make more sense for Faraday to
103
- # just implement a middleware for injecting a host/path prefix.
104
- @connection = Faraday::Connection.new(:url => url)
105
- end
106
-
107
- def call(env)
108
- original_url = env[:url]
109
- begin
110
- env[:url] = @connection.build_url(original_url)
111
- @app.call(env)
112
- ensure
113
- env[:url] = original_url
114
- end
115
- end
116
- end
117
97
  end
98
+
118
99
  end
100
+
119
101
  end
@@ -0,0 +1,30 @@
1
+ module Elastictastic
2
+
3
+ class MultiGet
4
+
5
+ include Enumerable
6
+
7
+ def initialize
8
+ @docspecs = []
9
+ @scopes = []
10
+ end
11
+
12
+ def add(scope, *ids)
13
+ scope = scope.all
14
+ params = scope.multi_get_params
15
+ ids.flatten.each do |id|
16
+ @docspecs << params.merge('_id' => id.to_s)
17
+ @scopes << scope
18
+ end
19
+ end
20
+
21
+ def each
22
+ return if @docspecs.empty?
23
+ Elastictastic.client.mget(@docspecs)['docs'].zip(@scopes) do |hit, scope|
24
+ yield scope.materialize_hit(hit) if hit['exists']
25
+ end
26
+ end
27
+
28
+ end
29
+
30
+ end
@@ -0,0 +1,70 @@
1
+ require 'stringio'
2
+
3
+ module Elastictastic
4
+ class MultiSearch
5
+ Component = Struct.new(:scope, :search_type)
6
+
7
+ def self.query(*scopes)
8
+ new.query(*scopes).run
9
+ end
10
+
11
+ def self.count(*scopes)
12
+ new.count(*scopes).run
13
+ end
14
+
15
+ def initialize
16
+ @components = []
17
+ end
18
+
19
+ def query(*scopes)
20
+ components = validate_scopes_for_query(scopes.flatten).map do |scope|
21
+ Component.new(scope, 'query_then_fetch')
22
+ end
23
+ @components.concat(components)
24
+ self
25
+ end
26
+
27
+ def count(*scopes)
28
+ components = scopes.flatten.map { |scope| Component.new(scope, 'count') }
29
+ @components.concat(components)
30
+ self
31
+ end
32
+
33
+ def run
34
+ if @components.any?
35
+ responses = Elastictastic.client.msearch(search_bodies)['responses']
36
+ responses.zip(@components) do |response, component|
37
+ raise ServerError[response['error']] if response['error']
38
+ scope, search_type = component.scope, component.search_type
39
+ case search_type
40
+ when 'query_then_fetch' then scope.response = response
41
+ when 'count' then scope.counts = response
42
+ end
43
+ end
44
+ end
45
+ self
46
+ end
47
+
48
+ private
49
+
50
+ def search_bodies
51
+ StringIO.new.tap do |io|
52
+ @components.each do |component|
53
+ scope, search_type = component.scope, component.search_type
54
+ headers = scope.multi_search_headers.
55
+ merge('search_type' => search_type)
56
+ io.puts(Elastictastic.json_encode(headers))
57
+ io.puts(Elastictastic.json_encode(scope.params))
58
+ end
59
+ end.string
60
+ end
61
+
62
+ def validate_scopes_for_query(scopes)
63
+ scopes.each do |scope|
64
+ if scope.params['size'].blank?
65
+ raise ArgumentError, "Multi-search scopes must have an explicit size"
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
@@ -1,29 +1,5 @@
1
- module Elastictastic
2
- module NestedDocument
3
- extend ActiveSupport::Concern
4
-
5
- included do
6
- include Properties
7
- include Dirty
8
- include Dirty::NestedDocumentMethods
9
- include MassAssignmentSecurity
10
- include Validations
11
- end
1
+ warn 'Elastictastic::NestedDocument is deprecated. Use Elastictastic::EmbeddedDocument instead.'
12
2
 
13
- module InstanceMethods
14
- def initialize_copy(original)
15
- self.write_attributes(original.read_attributes.dup)
16
- end
17
-
18
- def inspect
19
- inspected = "#<#{self.class.name}"
20
- if attributes.any?
21
- inspected << ' ' << attributes.each_pair.map do |attr, value|
22
- "#{attr}: #{value.inspect}"
23
- end.join(', ')
24
- end
25
- inspected << '>'
26
- end
27
- end
28
- end
3
+ module Elastictastic
4
+ NestedDocument = EmbeddedDocument
29
5
  end
@@ -11,14 +11,14 @@ module Elastictastic
11
11
  included do
12
12
  include NewRelic::Agent::MethodTracer
13
13
 
14
- add_method_tracer :create, 'ElasticSearch/#{args[1].classify}#create'
15
- add_method_tracer :delete, 'ElasticSearch/#{args[1].classify + "#" if args[1]}/delete'
16
- add_method_tracer :get, 'ElasticSearch/#{args[1].classify}#get'
17
- add_method_tracer :mget, 'ElasticSearch/mget'
18
- add_method_tracer :put_mapping, 'ElasticSearch/#{args[1].classify}#put_mapping'
19
- add_method_tracer :scroll, 'ElasticSearch/scroll'
20
- add_method_tracer :search, 'ElasticSearch/#{args[1].classify}#search'
21
- add_method_tracer :update, 'ElasticSearch/#{args[1].classify}#update'
14
+ add_method_tracer :create, 'Database/ElasticSearch/create'
15
+ add_method_tracer :delete, 'Database/ElasticSearch/delete'
16
+ add_method_tracer :get, 'Database/ElasticSearch/get'
17
+ add_method_tracer :mget, 'Database/ElasticSearch/mget'
18
+ add_method_tracer :put_mapping, 'Database/ElasticSearch/put_mapping'
19
+ add_method_tracer :scroll, 'Database/ElasticSearch/scroll'
20
+ add_method_tracer :search, 'Database/ElasticSearch/search'
21
+ add_method_tracer :update, 'Database/ElasticSearch/update'
22
22
  end
23
23
  end
24
24
  end
@@ -7,15 +7,17 @@ module Elastictastic
7
7
  include ActiveModel::Observing
8
8
  end
9
9
 
10
- module InstanceMethods
11
- Callbacks::HOOKS.each do |method|
12
- module_eval <<-RUBY, __FILE__, __LINE__ + 1
13
- def #{method}(*args)
10
+ Callbacks::HOOKS.each do |method|
11
+ module_eval <<-RUBY, __FILE__, __LINE__ + 1
12
+ def #{method}(options = {})
13
+ if options[:observers] == false
14
+ super
15
+ else
14
16
  notify_observers(:before_#{method})
15
17
  super.tap { notify_observers(:after_#{method}) }
16
18
  end
17
- RUBY
18
- end
19
+ end
20
+ RUBY
19
21
  end
20
22
  end
21
23
  end
@@ -0,0 +1,57 @@
1
+ module Elastictastic
2
+
3
+ module OptimisticLocking
4
+
5
+ extend ActiveSupport::Concern
6
+
7
+ module ClassMethods
8
+
9
+ def create_or_update(id, &block)
10
+ scope = current_scope
11
+ new.tap do |instance|
12
+ instance.id = id
13
+ yield instance
14
+ end.create do |e|
15
+ case e
16
+ when nil # chill
17
+ when Elastictastic::ServerError::DocumentAlreadyExistsEngineException,
18
+ Elastictastic::ServerError::DocumentAlreadyExistsException # 0.19+
19
+ scope.update(id, &block)
20
+ else
21
+ raise e
22
+ end
23
+ end
24
+ rescue Elastictastic::CancelSave
25
+ # Do Nothing
26
+ end
27
+
28
+ def update(id, &block)
29
+ instance = scoped({}).find_one(id, :preference => '_primary_first')
30
+ instance.try_update(current_scope, &block) if instance
31
+ end
32
+
33
+ def update_each(&block)
34
+ all.each { |instance| instance.try_update(current_scope, &block) }
35
+ end
36
+
37
+ end
38
+
39
+ def try_update(scope, &block) #:nodoc:
40
+ yield self
41
+ update do |e|
42
+ case e
43
+ when nil # chill
44
+ when Elastictastic::ServerError::VersionConflictEngineException,
45
+ Elastictastic::ServerError::VersionConflictException # 0.19
46
+ scope.update(id, &block)
47
+ else
48
+ raise e
49
+ end
50
+ end
51
+ rescue Elastictastic::CancelSave
52
+ # Do Nothing
53
+ end
54
+
55
+ end
56
+
57
+ end
@@ -31,7 +31,7 @@ module Elastictastic
31
31
  end
32
32
 
33
33
  def child_associations
34
- @child_associations ||= {}
34
+ @_child_associations ||= {}
35
35
  end
36
36
 
37
37
  def mapping
@@ -41,75 +41,77 @@ module Elastictastic
41
41
  end
42
42
  end
43
43
 
44
- module InstanceMethods
45
44
 
46
- def initialize(attributes = {})
47
- super
48
- @children = Hash.new do |hash, child_association_name|
49
- hash[child_association_name] = Elastictastic::ChildCollectionProxy.new(
50
- self.class.child_association(child_association_name.to_s),
51
- self
52
- )
53
- end
54
- end
55
-
56
- def elasticsearch_doc=(doc)
57
- @parent_id = doc.delete('_parent')
58
- super
45
+ def initialize(attributes = {})
46
+ super
47
+ @_children = Hash.new do |hash, child_association_name|
48
+ hash[child_association_name] = Elastictastic::ChildCollectionProxy.new(
49
+ self.class.child_association(child_association_name.to_s),
50
+ self
51
+ )
59
52
  end
53
+ end
60
54
 
61
- def _parent #:nodoc:
62
- return @parent if defined? @parent
63
- @parent =
64
- if @parent_id
65
- self.class.parent_association.clazz.find(@parent_id)
66
- end
67
- end
55
+ def elasticsearch_doc=(doc)
56
+ @_parent_id = doc.delete('_parent')
57
+ super
58
+ end
68
59
 
69
- def _parent_id #:nodoc:
70
- if @parent
71
- @parent_id = @parent.id
72
- elsif @parent_id
73
- @parent_id
60
+ def _parent #:nodoc:
61
+ return @_parent if defined? @_parent
62
+ @_parent =
63
+ if @_parent_id
64
+ self.class.parent_association.clazz.in_index(index).find(@_parent_id)
74
65
  end
66
+ #TODO - here's a piece of debugging to fix a problem where we get weird parents. remove after fixing
67
+ if @_parent && !@_parent.respond_to?(:id)
68
+ raise ArgumentError.new("Bad parent loaded from id #{@_parent_id} is a #{@_parent.class.name}.")
75
69
  end
70
+ @_parent
71
+ end
76
72
 
77
- def parent_collection=(parent_collection)
78
- if @parent_collection
79
- raise Elastictastic::IllegalModificationError,
80
- "Document is already a child of #{_parent}"
73
+ def _parent_id #:nodoc:
74
+ if @_parent_id
75
+ @_parent_id
76
+ elsif @_parent
77
+ unless @_parent.respond_to?(:id)
78
+ raise ArgumentError,
79
+ "@_parent is incorrectly set to #{Object.instance_method(:inspect).bind(@_parent).call}"
81
80
  end
82
- if persisted?
83
- raise Elastictastic::IllegalModificationError,
84
- "Can't change parent of persisted object"
85
- end
86
- @parent_collection = parent_collection
87
- @parent = parent_collection.parent
81
+ @_parent_id = @_parent.id
88
82
  end
83
+ end
89
84
 
90
- def save
91
- super
92
- self.class.child_associations.each_pair do |name, association|
93
- association.extract(self).transient_children.each do |child|
94
- child.save unless child.pending_save?
95
- end
96
- end
85
+ def parent=(parent)
86
+ if @_parent
87
+ raise Elastictastic::IllegalModificationError,
88
+ "Document is already a child of #{_parent}"
89
+ end
90
+ if persisted?
91
+ raise Elastictastic::IllegalModificationError,
92
+ "Can't change parent of persisted object"
97
93
  end
94
+ #TODO - here's a piece of debugging to fix a problem where we get weird parents. remove after fixing
95
+ if parent && !parent.respond_to?(:id)
96
+ raise ArgumentError.new("Bad parent loaded from id #{parent_id} is a #{parent.class.name}.")
97
+ end
98
+ @_parent = parent
99
+ end
98
100
 
99
- def persisted!
100
- was_persisted = @persisted
101
- super
102
- if @parent_collection && !was_persisted
103
- @parent_collection.persisted!(self)
101
+ def save(options = {})
102
+ super
103
+ self.class.child_associations.each_pair do |name, association|
104
+ association.extract(self).transient_children.each do |child|
105
+ child.save unless child.pending_save?
104
106
  end
105
107
  end
108
+ end
106
109
 
107
- protected
108
-
109
- def read_child(field_name)
110
- @children[field_name.to_s]
111
- end
110
+ protected
112
111
 
112
+ def read_child(field_name)
113
+ @_children[field_name.to_s]
113
114
  end
115
+
114
116
  end
115
117
  end
@@ -1,19 +1,19 @@
1
1
  module Elastictastic
2
2
  module Persistence
3
- def save
4
- persisted? ? update : create
3
+ def save(options = {}, &block)
4
+ persisted? ? update(options, &block) : create(options, &block)
5
5
  end
6
6
 
7
- def destroy
7
+ def destroy(options = {}, &block)
8
8
  if persisted?
9
- Elastictastic.persister.destroy(self)
9
+ Elastictastic.persister.destroy(self, &block)
10
10
  else
11
11
  raise OperationNotAllowed, "Cannot destroy transient document: #{inspect}"
12
12
  end
13
13
  end
14
14
 
15
15
  def persisted?
16
- !!@persisted
16
+ !!@_persisted
17
17
  end
18
18
 
19
19
  def transient?
@@ -21,38 +21,38 @@ module Elastictastic
21
21
  end
22
22
 
23
23
  def pending_save?
24
- !!@pending_save
24
+ !!@_pending_save
25
25
  end
26
26
 
27
27
  def pending_destroy?
28
- !!@pending_destroy
28
+ !!@_pending_destroy
29
29
  end
30
30
 
31
31
  def persisted!
32
- @persisted = true
33
- @pending_save = false
32
+ @_persisted = true
33
+ @_pending_save = false
34
34
  end
35
35
 
36
36
  def transient!
37
- @persisted = @pending_destroy = false
37
+ @_persisted = @_pending_destroy = false
38
38
  end
39
39
 
40
40
  def pending_save!
41
- @pending_save = true
41
+ @_pending_save = true
42
42
  end
43
43
 
44
44
  def pending_destroy!
45
- @pending_destroy = true
45
+ @_pending_destroy = true
46
46
  end
47
47
 
48
48
  protected
49
49
 
50
- def create
51
- Elastictastic.persister.create(self)
50
+ def create(options = {}, &block)
51
+ Elastictastic.persister.create(self, &block)
52
52
  end
53
53
 
54
- def update
55
- Elastictastic.persister.update(self)
54
+ def update(options = {}, &block)
55
+ Elastictastic.persister.update(self, &block)
56
56
  end
57
57
 
58
58
  private