elastictastic 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. data/LICENSE +19 -0
  2. data/README.md +326 -0
  3. data/lib/elastictastic/association.rb +21 -0
  4. data/lib/elastictastic/bulk_persistence_strategy.rb +70 -0
  5. data/lib/elastictastic/callbacks.rb +30 -0
  6. data/lib/elastictastic/child_collection_proxy.rb +56 -0
  7. data/lib/elastictastic/client.rb +101 -0
  8. data/lib/elastictastic/configuration.rb +35 -0
  9. data/lib/elastictastic/dirty.rb +130 -0
  10. data/lib/elastictastic/discrete_persistence_strategy.rb +52 -0
  11. data/lib/elastictastic/document.rb +98 -0
  12. data/lib/elastictastic/errors.rb +7 -0
  13. data/lib/elastictastic/field.rb +38 -0
  14. data/lib/elastictastic/index.rb +19 -0
  15. data/lib/elastictastic/mass_assignment_security.rb +15 -0
  16. data/lib/elastictastic/middleware.rb +119 -0
  17. data/lib/elastictastic/nested_document.rb +29 -0
  18. data/lib/elastictastic/new_relic_instrumentation.rb +26 -0
  19. data/lib/elastictastic/observer.rb +3 -0
  20. data/lib/elastictastic/observing.rb +21 -0
  21. data/lib/elastictastic/parent_child.rb +115 -0
  22. data/lib/elastictastic/persistence.rb +67 -0
  23. data/lib/elastictastic/properties.rb +236 -0
  24. data/lib/elastictastic/railtie.rb +35 -0
  25. data/lib/elastictastic/resource.rb +4 -0
  26. data/lib/elastictastic/scope.rb +283 -0
  27. data/lib/elastictastic/scope_builder.rb +32 -0
  28. data/lib/elastictastic/scoped.rb +20 -0
  29. data/lib/elastictastic/search.rb +180 -0
  30. data/lib/elastictastic/server_error.rb +15 -0
  31. data/lib/elastictastic/test_helpers.rb +172 -0
  32. data/lib/elastictastic/util.rb +63 -0
  33. data/lib/elastictastic/validations.rb +45 -0
  34. data/lib/elastictastic/version.rb +3 -0
  35. data/lib/elastictastic.rb +82 -0
  36. data/spec/environment.rb +6 -0
  37. data/spec/examples/active_model_lint_spec.rb +20 -0
  38. data/spec/examples/bulk_persistence_strategy_spec.rb +233 -0
  39. data/spec/examples/callbacks_spec.rb +96 -0
  40. data/spec/examples/dirty_spec.rb +238 -0
  41. data/spec/examples/document_spec.rb +600 -0
  42. data/spec/examples/mass_assignment_security_spec.rb +13 -0
  43. data/spec/examples/middleware_spec.rb +92 -0
  44. data/spec/examples/observing_spec.rb +141 -0
  45. data/spec/examples/parent_child_spec.rb +308 -0
  46. data/spec/examples/properties_spec.rb +92 -0
  47. data/spec/examples/scope_spec.rb +491 -0
  48. data/spec/examples/search_spec.rb +382 -0
  49. data/spec/examples/spec_helper.rb +15 -0
  50. data/spec/examples/validation_spec.rb +65 -0
  51. data/spec/models/author.rb +9 -0
  52. data/spec/models/blog.rb +5 -0
  53. data/spec/models/comment.rb +5 -0
  54. data/spec/models/post.rb +41 -0
  55. data/spec/models/post_observer.rb +11 -0
  56. data/spec/support/fakeweb_request_history.rb +13 -0
  57. metadata +227 -0
@@ -0,0 +1,180 @@
1
+ module Elastictastic
2
+ class Search
3
+ KEYS = %w(query filter from size sort highlight fields script_fields
4
+ preference facets)
5
+
6
+ attr_reader :sort, :from, :size, :fields, :script_fields, :preference, :facets
7
+ delegate :[], :to => :params
8
+
9
+ def initialize(params = {})
10
+ params = Util.deep_stringify(params) # this creates a copy
11
+ @queries, @query_filters = extract_queries_and_query_filters(params['query'])
12
+ @filters = extract_filters(params['filter'])
13
+ @from = params.delete('from')
14
+ @size = params.delete('size')
15
+ @sort = Util.ensure_array(params.delete('sort'))
16
+ highlight = params.delete('highlight')
17
+ if highlight
18
+ @highlight_fields = highlight.delete('fields')
19
+ @highlight_settings = highlight
20
+ end
21
+ @fields = Util.ensure_array(params.delete('fields'))
22
+ @script_fields = params.delete('script_fields')
23
+ @preference = params.delete('preference')
24
+ @facets = params.delete('facets')
25
+ end
26
+
27
+ def initialize_copy(other)
28
+ @queries = deep_copy(other.queries)
29
+ @query_filters = deep_copy(other.query_filters)
30
+ @filters = deep_copy(other.filters)
31
+ @sort = deep_copy(other.sort)
32
+ @highlight = deep_copy(other.highlight)
33
+ @fields = other.fields.dup if other.fields
34
+ @script_fields = deep_copy(other.script_fields)
35
+ @facets = deep_copy(other.facets)
36
+ end
37
+
38
+ def params
39
+ {}.tap do |params|
40
+ params['query'] = query
41
+ params['filter'] = filter
42
+ params['from'] = from
43
+ params['size'] = size
44
+ params['sort'] = maybe_array(sort)
45
+ params['highlight'] = highlight
46
+ params['fields'] = maybe_array(fields)
47
+ params['script_fields'] = script_fields
48
+ params['preference'] = preference
49
+ params['facets'] = facets
50
+ params.reject! { |k, v| v.blank? }
51
+ end
52
+ end
53
+
54
+ def query
55
+ query_query = maybe_array(queries) do
56
+ { 'bool' => { 'must' => queries }}
57
+ end
58
+ query_filter = maybe_array(query_filters) do
59
+ { 'and' => query_filters }
60
+ end
61
+ if query_query
62
+ if query_filter
63
+ { 'filtered' => { 'query' => query_query, 'filter' => query_filter }}
64
+ else
65
+ query_query
66
+ end
67
+ elsif query_filter
68
+ { 'constant_score' => { 'filter' => query_filter }}
69
+ end
70
+ end
71
+
72
+ def filter
73
+ maybe_array(filters) do
74
+ { 'and' => filters }
75
+ end
76
+ end
77
+
78
+ def highlight
79
+ if @highlight_fields
80
+ @highlight_settings.merge('fields' => @highlight_fields)
81
+ end
82
+ end
83
+
84
+ def merge(other)
85
+ dup.merge!(other)
86
+ end
87
+
88
+ def merge!(other)
89
+ @queries = combine(@queries, other.queries)
90
+ @query_filters = combine(@query_filters, other.query_filters)
91
+ @filters = combine(@filters, other.filters)
92
+ @from = other.from || @from
93
+ @size = other.size || @size
94
+ @sort = combine(@sort, other.sort)
95
+ if @highlight_fields && other.highlight_fields
96
+ @highlight_fields = combine(highlight_fields_with_settings, other.highlight_fields_with_settings)
97
+ @highlight_settings = {}
98
+ else
99
+ @highlight_settings = combine(@highlight_settings, other.highlight_settings)
100
+ @highlight_fields = combine(@highlight_fields, other.highlight_fields)
101
+ end
102
+ @fields = combine(@fields, other.fields)
103
+ @script_fields = combine(@script_fields, other.script_fields)
104
+ @preference = other.preference || @preference
105
+ @facets = combine(@facets, other.facets)
106
+ self
107
+ end
108
+
109
+ protected
110
+ attr_reader :queries, :query_filters, :filters, :highlight_fields,
111
+ :highlight_settings
112
+
113
+ def highlight_fields_with_settings
114
+ if @highlight_fields
115
+ {}.tap do |fields_with_settings|
116
+ @highlight_fields.each_pair do |field, settings|
117
+ fields_with_settings[field] = @highlight_settings.merge(settings)
118
+ end
119
+ end
120
+ end
121
+ end
122
+
123
+ private
124
+
125
+ def maybe_array(array)
126
+ case array.length
127
+ when 0 then nil
128
+ when 1 then array.first
129
+ else
130
+ if block_given? then yield
131
+ else array
132
+ end
133
+ end
134
+ end
135
+
136
+ def combine(object1, object2)
137
+ if object1.nil? then object2
138
+ elsif object2.nil? then object1
139
+ else
140
+ case object1
141
+ when Array then object1 + object2
142
+ when Hash then object1.merge(object2)
143
+ else raise ArgumentError, "Don't know how to combine #{object1.inspect} with #{object2.inspect}"
144
+ end
145
+ end
146
+ end
147
+
148
+ def extract_queries_and_query_filters(params)
149
+ if params.nil? then [[], []]
150
+ elsif params.keys == %w(filtered)
151
+ [extract_queries(params['filtered']['query']), extract_filters(params['filtered']['filter'])]
152
+ elsif params.keys == %w(constant_score) && params['constant_score'].keys == %w(filter)
153
+ [[], extract_filters(params['constant_score']['filter'])]
154
+ else
155
+ [extract_queries(params), []]
156
+ end
157
+ end
158
+
159
+ def extract_queries(params)
160
+ if params.nil? then []
161
+ elsif params.keys == %w(bool) && params['bool'].keys == %w(must)
162
+ params['bool']['must']
163
+ else [params]
164
+ end
165
+ end
166
+
167
+ def extract_filters(params)
168
+ if params.nil? then []
169
+ elsif params.keys == %w(and)
170
+ params['and']
171
+ else
172
+ [params]
173
+ end
174
+ end
175
+
176
+ def deep_copy(object)
177
+ Marshal.load(Marshal.dump(object)) unless object.nil?
178
+ end
179
+ end
180
+ end
@@ -0,0 +1,15 @@
1
+ module Elastictastic
2
+ module ServerError
3
+ class ServerError < StandardError
4
+ attr_accessor :status
5
+ end
6
+
7
+ class <<self
8
+ def const_missing(name)
9
+ Class.new(::Elastictastic::ServerError::ServerError).tap do |error|
10
+ const_set(name, error)
11
+ end
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,172 @@
1
+ require 'fakeweb'
2
+
3
+ module Elastictastic
4
+ module TestHelpers
5
+ ALPHANUM = ('0'..'9').to_a + ('A'..'Z').to_a + ('a'..'z').to_a
6
+
7
+ def stub_elasticsearch_create(index, type, *args)
8
+ options = args.extract_options!
9
+ id = args.pop
10
+ if id.nil?
11
+ id = ''
12
+ 22.times { id << ALPHANUM[rand(ALPHANUM.length)] }
13
+ path = "/#{index}/#{type}"
14
+ method = :post
15
+ else
16
+ path = "/#{index}/#{type}/#{id}/_create"
17
+ method = :put
18
+ end
19
+
20
+ FakeWeb.register_uri(
21
+ method,
22
+ /^#{Regexp.escape(TestHelpers.uri_for_path(path))}(\?.*)?$/,
23
+ options.reverse_merge(:body => {
24
+ 'ok' => 'true',
25
+ '_index' => index,
26
+ '_type' => type,
27
+ '_id' => id
28
+ }.to_json)
29
+ )
30
+ id
31
+ end
32
+
33
+ def stub_elasticsearch_update(index, type, id)
34
+ FakeWeb.register_uri(
35
+ :put,
36
+ /^#{TestHelpers.uri_for_path("/#{index}/#{type}/#{id}")}(\?.*)?$/,
37
+ :body => {
38
+ 'ok' => 'true',
39
+ '_index' => index,
40
+ '_type' => type,
41
+ '_id' => id
42
+ }.to_json
43
+ )
44
+ end
45
+
46
+ def stub_elasticsearch_get(index, type, id, doc = {})
47
+ FakeWeb.register_uri(
48
+ :get,
49
+ /^#{Regexp.escape(TestHelpers.uri_for_path("/#{index}/#{type}/#{id}").to_s)}(\?.*)?$/,
50
+ :body => {
51
+ 'ok' => true,
52
+ '_index' => index,
53
+ '_type' => type,
54
+ '_id' => id,
55
+ '_source' => doc,
56
+ 'exists' => !doc.nil?
57
+ }.to_json
58
+ )
59
+ end
60
+
61
+ def stub_elasticsearch_mget(index, type, *ids)
62
+ given_ids_with_docs = ids.extract_options!
63
+ ids_with_docs = {}
64
+ ids.each { |id| ids_with_docs[id] = {} }
65
+ ids_with_docs.merge!(given_ids_with_docs)
66
+ path = index ? "/#{index}/#{type}/_mget" : "/_mget"
67
+ docs = ids_with_docs.each_pair.map do |id, doc|
68
+ id, index = *id if Array === id
69
+ {
70
+ '_index' => index,
71
+ '_type' => type,
72
+ '_id' => id,
73
+ 'exists' => !!doc,
74
+ '_source' => doc
75
+ }
76
+ end
77
+
78
+ FakeWeb.register_uri(
79
+ :post,
80
+ TestHelpers.uri_for_path(path).to_s,
81
+ :body => {
82
+ 'docs' => docs
83
+ }.to_json
84
+ )
85
+ end
86
+
87
+ def stub_elasticsearch_destroy(index, type, id, options = {})
88
+ FakeWeb.register_uri(
89
+ :delete,
90
+ /^#{TestHelpers.uri_for_path("/#{index}/#{type}/#{id}")}(\?.*)?$/,
91
+ options.reverse_merge(:body => {
92
+ 'ok' => true,
93
+ 'found' => true,
94
+ '_index' => 'test',
95
+ '_type' => 'test',
96
+ '_id' => id,
97
+ '_version' => 1
98
+ }.to_json)
99
+ )
100
+ end
101
+
102
+ def stub_elasticsearch_destroy_all(index, type)
103
+ FakeWeb.register_uri(
104
+ :delete,
105
+ TestHelpers.uri_for_path("/#{index}/#{type}"),
106
+ :body => { 'ok' => true }.to_json
107
+ )
108
+ end
109
+
110
+ def stub_elasticsearch_bulk(*responses)
111
+ FakeWeb.register_uri(
112
+ :post,
113
+ TestHelpers.uri_for_path("/_bulk"),
114
+ :body => { 'took' => 1, 'items' => responses }.to_json
115
+ )
116
+ end
117
+
118
+ def stub_elasticsearch_put_mapping(index, type)
119
+ FakeWeb.register_uri(
120
+ :put,
121
+ TestHelpers.uri_for_path("/#{index}/#{type}/_mapping"),
122
+ :body => { 'ok' => true, 'acknowledged' => true }.to_json
123
+ )
124
+ end
125
+
126
+ def stub_elasticsearch_search(index, type, data)
127
+ if Array === data
128
+ response = data.map do |datum|
129
+ { :body => datum.to_json }
130
+ end
131
+ else
132
+ response = { :body => data.to_json }
133
+ end
134
+
135
+ uri = TestHelpers.uri_for_path("/#{index}/#{type}/_search").to_s
136
+ FakeWeb.register_uri(
137
+ :post,
138
+ /^#{Regexp.escape(uri)}/,
139
+ response
140
+ )
141
+ end
142
+
143
+ def stub_elasticsearch_scan(index, type, batch_size, *hits)
144
+ scan_uri = Regexp.escape(TestHelpers.uri_for_path("/#{index}/#{type}/_search").to_s)
145
+ scroll_ids = Array.new(batch_size + 1) { rand(10**100).to_s(36) }
146
+ FakeWeb.register_uri(
147
+ :post,
148
+ /^#{scan_uri}\?.*search_type=scan/,
149
+ :body => {
150
+ '_scroll_id' => scroll_ids.first,
151
+ 'hits' => { 'total' => hits.length, 'hits' => [] }
152
+ }.to_json
153
+ )
154
+
155
+ batches = hits.each_slice(batch_size).each_with_index.map do |hit_batch, i|
156
+ { :body => { '_scroll_id' => scroll_ids[i+1], 'hits' => { 'hits' => hit_batch }}.to_json }
157
+ end
158
+ batches << { :body => { 'hits' => { 'hits' => [] }}.to_json }
159
+ scroll_uri = Regexp.escape(TestHelpers.uri_for_path("/_search/scroll").to_s)
160
+ FakeWeb.register_uri(
161
+ :post,
162
+ /^#{scroll_uri}/,
163
+ batches
164
+ )
165
+ scroll_ids
166
+ end
167
+
168
+ def self.uri_for_path(path)
169
+ "#{Elastictastic.config.hosts.first}#{path}"
170
+ end
171
+ end
172
+ end
@@ -0,0 +1,63 @@
1
+ module Elastictastic
2
+ module Util
3
+ extend self
4
+
5
+ def deep_stringify(hash)
6
+ {}.tap do |stringified|
7
+ hash.each_pair do |key, value|
8
+ stringified[key.to_s] = Hash === value ? deep_stringify(value) : value
9
+ end
10
+ end
11
+ end
12
+
13
+ def deep_merge(l, r)
14
+ if l.nil? then r
15
+ elsif r.nil? then l
16
+ elsif Hash === l && Hash === r
17
+ {}.tap do |merged|
18
+ (l.keys | r.keys).each do |key|
19
+ merged[key] = deep_merge(l[key], r[key])
20
+ end
21
+ end
22
+ elsif Array === l && Array === r then l + r
23
+ elsif Array === l then l + [r]
24
+ elsif Array === r then [l] + r
25
+ else [l, r]
26
+ end
27
+ end
28
+
29
+ def ensure_array(object)
30
+ case object
31
+ when nil then []
32
+ when Array then object
33
+ else [object]
34
+ end
35
+ end
36
+
37
+ def call_or_each(object, &block)
38
+ if Array === object then object.each(&block)
39
+ else
40
+ block.call(object)
41
+ object
42
+ end
43
+ end
44
+
45
+ def call_or_map(object, &block)
46
+ if Array === object then object.map(&block)
47
+ else block.call(object)
48
+ end
49
+ end
50
+
51
+ def unflatten_hash(hash)
52
+ {}.tap do |unflattened|
53
+ hash.each_pair do |key, value|
54
+ namespace = key.split('.')
55
+ field_name = namespace.pop
56
+ namespace.inject(unflattened) do |current, component|
57
+ current[component] ||= {}
58
+ end[field_name] = value
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,45 @@
1
+ module Elastictastic
2
+ module Validations
3
+ extend ActiveSupport::Concern
4
+
5
+ included do
6
+ include ActiveModel::Validations
7
+ end
8
+
9
+ module ClassMethods
10
+ def embed(*embed_names)
11
+ super
12
+ embed_names.extract_options!
13
+ args = embed_names + [{ :nested => true }]
14
+ validates(*args)
15
+ end
16
+ end
17
+
18
+ module InstanceMethods
19
+ def save
20
+ if valid?
21
+ super
22
+ true
23
+ else
24
+ false
25
+ end
26
+ end
27
+
28
+ def save!
29
+ if !save
30
+ raise Elastictastic::RecordInvalid, errors.full_messages.to_sentence
31
+ end
32
+ self
33
+ end
34
+ end
35
+
36
+ class NestedValidator < ActiveModel::EachValidator
37
+ def validate_each(record, attribute, value)
38
+ value = [value].compact unless Array === value
39
+ unless value.all? { |el| el.valid? }
40
+ record.errors[:attribute] = :invalid
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,3 @@
1
+ module Elastictastic
2
+ VERSION = '0.5.0'
3
+ end
@@ -0,0 +1,82 @@
1
+ require 'active_support/core_ext'
2
+ require 'active_model'
3
+ require 'elastictastic/errors'
4
+
5
+ module Elastictastic
6
+ autoload :Association, 'elastictastic/association'
7
+ autoload :BulkPersistenceStrategy, 'elastictastic/bulk_persistence_strategy'
8
+ autoload :Callbacks, 'elastictastic/callbacks'
9
+ autoload :ChildCollectionProxy, 'elastictastic/child_collection_proxy'
10
+ autoload :Client, 'elastictastic/client'
11
+ autoload :Configuration, 'elastictastic/configuration'
12
+ autoload :Dirty, 'elastictastic/dirty'
13
+ autoload :DiscretePersistenceStrategy, 'elastictastic/discrete_persistence_strategy'
14
+ autoload :Document, 'elastictastic/document'
15
+ autoload :Field, 'elastictastic/field'
16
+ autoload :Index, 'elastictastic/index'
17
+ autoload :MassAssignmentSecurity, 'elastictastic/mass_assignment_security'
18
+ autoload :Middleware, 'elastictastic/middleware'
19
+ autoload :NestedCollectionProxy, 'elastictastic/nested_collection_proxy'
20
+ autoload :NestedDocument, 'elastictastic/nested_document'
21
+ autoload :Observer, 'elastictastic/observer'
22
+ autoload :Observing, 'elastictastic/observing'
23
+ autoload :ParentChild, 'elastictastic/parent_child'
24
+ autoload :Persistence, 'elastictastic/persistence'
25
+ autoload :Properties, 'elastictastic/properties'
26
+ autoload :Resource, 'elastictastic/resource'
27
+ autoload :Scope, 'elastictastic/scope'
28
+ autoload :ScopeBuilder, 'elastictastic/scope_builder'
29
+ autoload :Scoped, 'elastictastic/scoped'
30
+ autoload :Search, 'elastictastic/search'
31
+ autoload :ServerError, 'elastictastic/server_error'
32
+ autoload :TestHelpers, 'elastictastic/test_helpers'
33
+ autoload :Util, 'elastictastic/util'
34
+ autoload :Validations, 'elastictastic/validations'
35
+
36
+ class <<self
37
+ attr_writer :config
38
+
39
+ def config
40
+ @config ||= Configuration.new
41
+ end
42
+
43
+ def client
44
+ Thread.current['Elastictastic::client'] ||= Client.new(config)
45
+ end
46
+
47
+ def persister=(persister)
48
+ Thread.current['Elastictastic::persister'] = persister
49
+ end
50
+
51
+ def persister
52
+ Thread.current['Elastictastic::persister'] ||=
53
+ Elastictastic::DiscretePersistenceStrategy.instance
54
+ end
55
+
56
+ def bulk
57
+ original_persister = self.persister
58
+ begin
59
+ self.persister = Elastictastic::BulkPersistenceStrategy.new
60
+ yield
61
+ self.persister.flush
62
+ rescue Elastictastic::CancelBulkOperation
63
+ # Nothing to see here...
64
+ ensure
65
+ self.persister = original_persister
66
+ end
67
+ end
68
+
69
+ def Index(name_or_index)
70
+ Index === name_or_index ? name_or_index : Index.new(name_or_index)
71
+ end
72
+
73
+ private
74
+
75
+ def new_transport
76
+ transport_class = const_get("#{config.transport.camelize}Transport")
77
+ transport_class.new(config)
78
+ end
79
+ end
80
+ end
81
+
82
+ require 'elastictastic/railtie' if defined? Rails
@@ -0,0 +1,6 @@
1
+ require 'bundler'
2
+ Bundler.require(:default, :test, :development)
3
+
4
+ %w(author blog comment post post_observer).each do |model|
5
+ require File.expand_path("../models/#{model}", __FILE__)
6
+ end
@@ -0,0 +1,20 @@
1
+ require File.expand_path('../spec_helper', __FILE__)
2
+
3
+ describe 'ActiveModel compliance' do
4
+ include ActiveModel::Lint::Tests
5
+
6
+ ActiveModel::Lint::Tests.public_instance_methods.each do |method|
7
+ method = method.to_s
8
+ if method =~ /^test_/
9
+ example method.gsub('_', ' ') do
10
+ __send__(method)
11
+ end
12
+ end
13
+ end
14
+
15
+ private
16
+
17
+ def model
18
+ Post.new
19
+ end
20
+ end