elastic_record 0.11.1 → 0.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +2 -1
- data/README.rdoc +37 -9
- data/elastic_record.gemspec +2 -2
- data/lib/elastic_record/callbacks.rb +1 -1
- data/lib/elastic_record/connection.rb +36 -13
- data/lib/elastic_record/index.rb +33 -4
- data/lib/elastic_record/index/configurator.rb +14 -0
- data/lib/elastic_record/index/documents.rb +21 -13
- data/lib/elastic_record/index/manage.rb +4 -9
- data/lib/elastic_record/index/mapping.rb +12 -0
- data/lib/elastic_record/index/percolator.rb +1 -0
- data/lib/elastic_record/index/settings.rb +4 -0
- data/lib/elastic_record/lucene.rb +40 -22
- data/lib/elastic_record/model.rb +8 -5
- data/lib/elastic_record/relation/batches.rb +8 -1
- data/lib/elastic_record/relation/finder_methods.rb +2 -2
- data/lib/elastic_record/relation/none.rb +3 -3
- data/lib/elastic_record/relation/search_methods.rb +33 -3
- data/lib/elastic_record/relation/value_methods.rb +1 -1
- data/lib/elastic_record/searches_many.rb +4 -0
- data/lib/elastic_record/searches_many/association.rb +25 -14
- data/lib/elastic_record/searches_many/reflection.rb +1 -1
- data/lib/elastic_record/tasks/index.rake +5 -21
- data/test/elastic_record/callbacks_test.rb +9 -0
- data/test/elastic_record/connection_test.rb +19 -1
- data/test/elastic_record/index/configurator_test.rb +18 -0
- data/test/elastic_record/index/documents_test.rb +22 -3
- data/test/elastic_record/index/manage_test.rb +7 -0
- data/test/elastic_record/index/mapping_test.rb +11 -0
- data/test/elastic_record/index/percolator_test.rb +11 -9
- data/test/elastic_record/index_test.rb +23 -6
- data/test/elastic_record/lucene_test.rb +21 -13
- data/test/elastic_record/model_test.rb +9 -9
- data/test/elastic_record/relation/batches_test.rb +8 -0
- data/test/elastic_record/relation/finder_methods_test.rb +6 -7
- data/test/elastic_record/relation/none_test.rb +3 -0
- data/test/elastic_record/relation/search_methods_test.rb +17 -41
- data/test/elastic_record/relation_test.rb +2 -0
- data/test/elastic_record/searches_many/reflection_test.rb +7 -0
- metadata +15 -19
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 3a25b326c628b294aff17e44d2465d1b988ca2c3
|
4
|
+
data.tar.gz: f20002ac648ecf9f7df8f524bf639e6b1be55af2
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c3baffc104e19d488ae910b3265cd8dcb239ac5a1c05492cdbb1a7747bb2156f3e4085bd6de0defed0eea9dbd2b6b3478a17f759eeaf3a751e12d417f7912eef
|
7
|
+
data.tar.gz: 1bdeaefcffd51c750ad47d36ff5d8018d9040557f2f9a63d994715bb9206346c1439b1c2e8abcf09c997f127bf06fa6ccacccceef33798993f50548c4c5069db
|
data/Gemfile
CHANGED
data/README.rdoc
CHANGED
@@ -9,16 +9,38 @@ The usual Gemfile addition:
|
|
9
9
|
|
10
10
|
gem 'elastic_record'
|
11
11
|
|
12
|
-
Creating the index:
|
13
|
-
|
14
|
-
rake index:create
|
15
12
|
|
16
13
|
Include ElasticRecord into your model:
|
17
14
|
|
18
|
-
|
15
|
+
class Product < YourFavoriteOrm::Base
|
19
16
|
include ElasticRecord::Model
|
20
17
|
end
|
21
18
|
|
19
|
+
== Configuration
|
20
|
+
|
21
|
+
While elastic search automatically maps fields, you may wish to override the defaults:
|
22
|
+
|
23
|
+
class Product < YourFavoriteOrm::Base
|
24
|
+
elastic_index.configure do
|
25
|
+
property :status, type: "string", index: "not_analyzed"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
You can also directly access Product.elastic_index.mapping and Product.elastic_index.settings:
|
30
|
+
|
31
|
+
class Product
|
32
|
+
elastic_index.mapping = {
|
33
|
+
properties: {
|
34
|
+
name: {type: "string", index: "analyzed"}
|
35
|
+
status: {type: "string", index: "not_analyzed"}
|
36
|
+
}
|
37
|
+
}
|
38
|
+
end
|
39
|
+
|
40
|
+
Create the index:
|
41
|
+
|
42
|
+
rake index:create
|
43
|
+
|
22
44
|
== Searching
|
23
45
|
|
24
46
|
ElasticRecord adds the method 'elastic_search' to your models. It works similar to active_record scoping:
|
@@ -29,8 +51,9 @@ ElasticRecord adds the method 'elastic_search' to your models. It works similar
|
|
29
51
|
|
30
52
|
If a simple hash is passed into filter, a term or terms query is created:
|
31
53
|
|
32
|
-
search.filter(color: 'red') # Creates a term filter
|
33
|
-
search.filter(color: %w(red blue)) # Creates a terms filter
|
54
|
+
search.filter(color: 'red') # Creates a 'term' filter
|
55
|
+
search.filter(color: %w(red blue)) # Creates a 'terms' filter
|
56
|
+
search.filter(color: nil) # Creates a 'missing' filter
|
34
57
|
|
35
58
|
If a hash containing hashes is passed into filter, it is used directly as a filter DSL expression:
|
36
59
|
|
@@ -47,9 +70,6 @@ An Arelastic object can also be passed in, working similarily to Arel:
|
|
47
70
|
# Size is greater than 5
|
48
71
|
search.filter(Product.arelastic[:size].gt(5))
|
49
72
|
|
50
|
-
# Product has a nil name
|
51
|
-
search.filter(Product.arelastic[:name].missing)
|
52
|
-
|
53
73
|
# Name is 'hola' or name is missing
|
54
74
|
search.filter(Product.arelastic[:name].eq("hola").or(Product.arelastic[:name].missing))
|
55
75
|
|
@@ -115,3 +135,11 @@ Class methods can be executed within scopes:
|
|
115
135
|
|
116
136
|
# Increase the price of all red products by $10.
|
117
137
|
Product.filter(color: 'red').increase_prices
|
138
|
+
|
139
|
+
== Index Information
|
140
|
+
|
141
|
+
Core and Index APIs can be accessed with Product.elastic_index. Some examples include:
|
142
|
+
|
143
|
+
Production.elastic_index.refresh # Call the refresh API
|
144
|
+
Production.elastic_index.get_mapping # Get the index mapping defined by elastic search
|
145
|
+
Production.elastic_index.reset # Delete related indexes and deploy a new one
|
data/elastic_record.gemspec
CHANGED
@@ -15,7 +15,7 @@ module ElasticRecord
|
|
15
15
|
self.servers = servers.split(',')
|
16
16
|
end
|
17
17
|
|
18
|
-
self.current_server =
|
18
|
+
self.current_server = next_server
|
19
19
|
self.request_count = 0
|
20
20
|
self.max_request_count = 100
|
21
21
|
self.options = options
|
@@ -60,27 +60,35 @@ module ElasticRecord
|
|
60
60
|
}
|
61
61
|
|
62
62
|
def http_request(method, path, body = nil)
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
payload
|
69
|
-
|
70
|
-
|
63
|
+
with_retry do
|
64
|
+
request = METHODS[method].new(path)
|
65
|
+
request.body = body
|
66
|
+
http = new_http
|
67
|
+
|
68
|
+
ActiveSupport::Notifications.instrument("request.elastic_record") do |payload|
|
69
|
+
payload[:http] = http
|
70
|
+
payload[:request] = request
|
71
|
+
payload[:response] = http.request(request)
|
72
|
+
end
|
71
73
|
end
|
72
74
|
end
|
73
75
|
|
74
76
|
private
|
75
|
-
def
|
76
|
-
|
77
|
+
def next_server
|
78
|
+
if @shuffled_servers.nil?
|
79
|
+
@shuffled_servers = servers.shuffle
|
80
|
+
else
|
81
|
+
@shuffled_servers.push(@shuffled_servers.shift)
|
82
|
+
end
|
83
|
+
|
84
|
+
@shuffled_servers.first
|
77
85
|
end
|
78
86
|
|
79
87
|
def new_http
|
80
88
|
self.request_count += 1
|
81
89
|
|
82
|
-
if request_count > max_request_count
|
83
|
-
self.current_server =
|
90
|
+
if request_count > max_request_count
|
91
|
+
self.current_server = next_server
|
84
92
|
self.request_count = 0
|
85
93
|
end
|
86
94
|
|
@@ -92,5 +100,20 @@ module ElasticRecord
|
|
92
100
|
end
|
93
101
|
http
|
94
102
|
end
|
103
|
+
|
104
|
+
def with_retry
|
105
|
+
retry_count = 0
|
106
|
+
begin
|
107
|
+
yield
|
108
|
+
rescue StandardError
|
109
|
+
if retry_count < options[:retries].to_i
|
110
|
+
self.current_server = next_server
|
111
|
+
retry_count += 1
|
112
|
+
retry
|
113
|
+
else
|
114
|
+
raise
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
95
118
|
end
|
96
119
|
end
|
data/lib/elastic_record/index.rb
CHANGED
@@ -1,12 +1,31 @@
|
|
1
|
+
require 'elastic_record/index/configurator'
|
1
2
|
require 'elastic_record/index/deferred'
|
2
3
|
require 'elastic_record/index/documents'
|
3
4
|
require 'elastic_record/index/manage'
|
4
5
|
require 'elastic_record/index/mapping'
|
5
6
|
require 'elastic_record/index/percolator'
|
6
7
|
require 'elastic_record/index/settings'
|
7
|
-
|
8
|
+
|
9
|
+
require 'active_support/core_ext/hash/deep_dup'
|
8
10
|
|
9
11
|
module ElasticRecord
|
12
|
+
# ElasticRecord::Index provides access to elastic search's API. It is accessed with
|
13
|
+
# <tt>Widget.elastic_index</tt>. The methods provided are:
|
14
|
+
#
|
15
|
+
# [create]
|
16
|
+
# Create a new index that is not aliased
|
17
|
+
# [create_and_deploy]
|
18
|
+
# Create a new index and alias it
|
19
|
+
# [reset]
|
20
|
+
# Delete all aliased indexes and deploy a new one
|
21
|
+
# [refresh]
|
22
|
+
# Call the refresh API
|
23
|
+
# [exists?(index_name)]
|
24
|
+
# Returns if the index exists
|
25
|
+
# [get_mapping]
|
26
|
+
# Returns the mapping currently stored by elastic search.
|
27
|
+
# [put_mapping]
|
28
|
+
# Update elastic search's mapping
|
10
29
|
class Index
|
11
30
|
include Documents
|
12
31
|
include Manage
|
@@ -23,8 +42,10 @@ module ElasticRecord
|
|
23
42
|
@disabled = false
|
24
43
|
end
|
25
44
|
|
26
|
-
|
27
|
-
|
45
|
+
def initialize_copy(other)
|
46
|
+
@settings = settings.deep_dup
|
47
|
+
@mapping = mapping.deep_dup
|
48
|
+
end
|
28
49
|
|
29
50
|
def alias_name
|
30
51
|
@alias_name ||= model.base_class.model_name.collection
|
@@ -46,9 +67,17 @@ module ElasticRecord
|
|
46
67
|
model.elastic_connection
|
47
68
|
end
|
48
69
|
|
70
|
+
def configure(&block)
|
71
|
+
Configurator.new(self).instance_eval(&block)
|
72
|
+
end
|
73
|
+
|
74
|
+
def get(end_path, json = nil)
|
75
|
+
connection.json_get "/#{alias_name}/#{type}/#{end_path}", json
|
76
|
+
end
|
77
|
+
|
49
78
|
private
|
50
79
|
def new_index_name
|
51
|
-
"#{alias_name}_#{Time.now.to_i}"
|
80
|
+
"#{alias_name}_#{(Time.now.to_f * 100).to_i}"
|
52
81
|
end
|
53
82
|
end
|
54
83
|
end
|
@@ -8,9 +8,9 @@ module ElasticRecord
|
|
8
8
|
|
9
9
|
index_name ||= alias_name
|
10
10
|
|
11
|
-
if
|
12
|
-
|
13
|
-
|
11
|
+
if batch = current_batch
|
12
|
+
current_batch << { index: { _index: index_name, _type: type, _id: id } }
|
13
|
+
current_batch << document
|
14
14
|
else
|
15
15
|
connection.json_put "/#{index_name}/#{type}/#{id}", document
|
16
16
|
end
|
@@ -19,28 +19,28 @@ module ElasticRecord
|
|
19
19
|
def delete_document(id, index_name = nil)
|
20
20
|
index_name ||= alias_name
|
21
21
|
|
22
|
-
if
|
23
|
-
|
22
|
+
if batch = current_batch
|
23
|
+
batch << { delete: { _index: index_name, _type: type, _id: id } }
|
24
24
|
else
|
25
25
|
connection.json_delete "/#{index_name}/#{type}/#{id}"
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
29
|
def record_exists?(id)
|
30
|
-
|
30
|
+
get(id)['exists']
|
31
31
|
end
|
32
32
|
|
33
33
|
def search(elastic_query, options = {})
|
34
|
-
url = "
|
34
|
+
url = "_search"
|
35
35
|
if options.any?
|
36
36
|
url += "?#{options.to_query}"
|
37
37
|
end
|
38
38
|
|
39
|
-
|
39
|
+
get url, elastic_query
|
40
40
|
end
|
41
41
|
|
42
42
|
def explain(id, elastic_query)
|
43
|
-
|
43
|
+
get "_explain", elastic_query
|
44
44
|
end
|
45
45
|
|
46
46
|
def scroll(scroll_id, scroll_keep_alive)
|
@@ -49,14 +49,14 @@ module ElasticRecord
|
|
49
49
|
end
|
50
50
|
|
51
51
|
def bulk
|
52
|
-
@
|
52
|
+
@_batch = []
|
53
53
|
yield
|
54
|
-
if @
|
55
|
-
body = @
|
54
|
+
if @_batch.any?
|
55
|
+
body = @_batch.map { |action| "#{ActiveSupport::JSON.encode(action)}\n" }.join
|
56
56
|
connection.json_post "/_bulk", body
|
57
57
|
end
|
58
58
|
ensure
|
59
|
-
@
|
59
|
+
@_batch = nil
|
60
60
|
end
|
61
61
|
|
62
62
|
def bulk_add(batch, index_name = nil)
|
@@ -68,6 +68,14 @@ module ElasticRecord
|
|
68
68
|
end
|
69
69
|
end
|
70
70
|
end
|
71
|
+
|
72
|
+
def current_batch
|
73
|
+
if @_batch
|
74
|
+
@_batch
|
75
|
+
elsif model.superclass.respond_to?(:elastic_index)
|
76
|
+
model.superclass.elastic_index.current_batch
|
77
|
+
end
|
78
|
+
end
|
71
79
|
end
|
72
80
|
end
|
73
81
|
end
|
@@ -9,7 +9,6 @@ module ElasticRecord
|
|
9
9
|
|
10
10
|
def create(index_name = new_index_name)
|
11
11
|
connection.json_put "/#{index_name}", "settings" => settings
|
12
|
-
# update_settings(index_name)
|
13
12
|
update_mapping(index_name)
|
14
13
|
index_name
|
15
14
|
end
|
@@ -28,6 +27,10 @@ module ElasticRecord
|
|
28
27
|
connection.head("/#{index_name}") == '200'
|
29
28
|
end
|
30
29
|
|
30
|
+
def type_exists?(index_name = alias_name)
|
31
|
+
connection.head("/#{index_name}/#{type}") == '200'
|
32
|
+
end
|
33
|
+
|
31
34
|
def deploy(index_name)
|
32
35
|
actions = [
|
33
36
|
{
|
@@ -50,14 +53,6 @@ module ElasticRecord
|
|
50
53
|
connection.json_post '/_aliases', actions: actions
|
51
54
|
end
|
52
55
|
|
53
|
-
def update_mapping(index_name = alias_name)
|
54
|
-
connection.json_put "/#{index_name}/#{type}/_mapping", type => mapping
|
55
|
-
end
|
56
|
-
|
57
|
-
def update_settings(index_name = alias_name)
|
58
|
-
connection.json_put "/#{index_name}/_settings", settings
|
59
|
-
end
|
60
|
-
|
61
56
|
def refresh(index_name = alias_name)
|
62
57
|
connection.json_post "/#{index_name}/_refresh"
|
63
58
|
end
|
@@ -5,6 +5,18 @@ module ElasticRecord
|
|
5
5
|
mapping.deep_merge!(mapping)
|
6
6
|
end
|
7
7
|
|
8
|
+
def update_mapping(index_name = alias_name)
|
9
|
+
connection.json_put "/#{index_name}/#{type}/_mapping", type => mapping
|
10
|
+
end
|
11
|
+
|
12
|
+
def get_mapping(index_name = alias_name)
|
13
|
+
connection.json_get "/#{index_name}/#{type}/_mapping"
|
14
|
+
end
|
15
|
+
|
16
|
+
def delete_mapping(index_name = alias_name)
|
17
|
+
connection.json_delete "/#{index_name}/#{type}"
|
18
|
+
end
|
19
|
+
|
8
20
|
def mapping
|
9
21
|
@mapping ||= {
|
10
22
|
_source: {
|
@@ -4,47 +4,65 @@ module ElasticRecord
|
|
4
4
|
class Lucene
|
5
5
|
# Special characters:
|
6
6
|
# + - && || ! ( ) { } [ ] ^ " ~ * ? : \
|
7
|
-
ESCAPE_REGEX = /(\+|-|&&|\|\||!|\(|\)|{|}|\[|\]|`|"
|
7
|
+
ESCAPE_REGEX = /(\+|-|&&|\|\||!|\(|\)|{|}|\[|\]|`|"|~\*|\?|:|\\)/
|
8
8
|
|
9
9
|
class << self
|
10
10
|
def escape(query)
|
11
11
|
query.gsub(ESCAPE_REGEX, "\\\\\\1")
|
12
12
|
end
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
end
|
17
|
-
|
18
|
-
# Returns a lucene query that works like G
|
19
|
-
def smart_query(query, fields, &block)
|
14
|
+
# Returns a lucene query that works like GMail
|
15
|
+
def match_phrase(query, fields, &block)
|
20
16
|
return if query.blank?
|
21
17
|
|
22
|
-
words =
|
18
|
+
words = split_phrase_into_words(query)
|
23
19
|
|
24
20
|
words.map do |word|
|
25
21
|
if word =~ /^(\w+):(.+)$/ && fields.include?($1)
|
26
|
-
match_word $2, [$1]
|
22
|
+
match_word $2, [block_given? ? yield($1) : $1]
|
27
23
|
else
|
28
|
-
match_word word, fields
|
24
|
+
match_word word, (block_given? ? fields.map(&block) : fields)
|
29
25
|
end
|
30
26
|
end.join(' AND ')
|
31
27
|
end
|
32
28
|
|
33
|
-
|
29
|
+
# Performs a prefix match on the word:
|
30
|
+
#
|
31
|
+
# ElasticRecord::Lucene.match_word('blue', ['color', 'name'])
|
32
|
+
# => (color:blue* OR name:blue*)
|
33
|
+
#
|
34
|
+
# In the case that the word has special characters, it is wrapped in quotes:
|
35
|
+
#
|
36
|
+
# ElasticRecord::Lucene.match_word('A&M', ['name'])
|
37
|
+
# => (name:"A&M")
|
38
|
+
def match_word(word, fields)
|
39
|
+
if word =~ / / || word =~ ESCAPE_REGEX
|
40
|
+
word = "\"#{word.gsub('"', '')}\""
|
41
|
+
else
|
42
|
+
word = "#{word}*"
|
43
|
+
end
|
34
44
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
45
|
+
or_query = fields.map do |field|
|
46
|
+
"#{field}:#{word}"
|
47
|
+
end.join(' OR ')
|
48
|
+
|
49
|
+
"(#{or_query})"
|
50
|
+
end
|
41
51
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
52
|
+
private
|
53
|
+
# Converts a sentence into the words:
|
54
|
+
#
|
55
|
+
# split_phrase_into_words('his "blue fox"')
|
56
|
+
# => ['his', 'blue fox']
|
57
|
+
def split_phrase_into_words(phrase)
|
58
|
+
# If we have an odd number of double quotes,
|
59
|
+
# add a double quote to the end so that shellwords
|
60
|
+
# does not crap out.
|
61
|
+
if phrase.count('"') % 2 == 1
|
62
|
+
phrase = "#{phrase}\""
|
63
|
+
end
|
46
64
|
|
47
|
-
"
|
65
|
+
Shellwords::shellwords phrase.gsub("'", "\"'\"")
|
48
66
|
end
|
49
67
|
end
|
50
68
|
end
|