searchkick 2.5.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE.md +7 -0
- data/.travis.yml +2 -11
- data/CHANGELOG.md +22 -0
- data/CONTRIBUTING.md +1 -1
- data/Gemfile +3 -3
- data/LICENSE.txt +1 -1
- data/README.md +68 -141
- data/Rakefile +0 -4
- data/benchmark/Gemfile +3 -2
- data/benchmark/{benchmark.rb → index.rb} +33 -31
- data/benchmark/search.rb +48 -0
- data/docs/Searchkick-3-Upgrade.md +57 -0
- data/lib/searchkick.rb +50 -27
- data/lib/searchkick/bulk_indexer.rb +168 -0
- data/lib/searchkick/bulk_reindex_job.rb +1 -1
- data/lib/searchkick/index.rb +122 -348
- data/lib/searchkick/index_options.rb +29 -26
- data/lib/searchkick/logging.rb +8 -7
- data/lib/searchkick/model.rb +37 -90
- data/lib/searchkick/multi_search.rb +6 -7
- data/lib/searchkick/query.rb +169 -166
- data/lib/searchkick/record_data.rb +133 -0
- data/lib/searchkick/record_indexer.rb +55 -0
- data/lib/searchkick/reindex_queue.rb +1 -1
- data/lib/searchkick/reindex_v2_job.rb +10 -13
- data/lib/searchkick/results.rb +14 -25
- data/lib/searchkick/tasks.rb +0 -4
- data/lib/searchkick/version.rb +1 -1
- data/searchkick.gemspec +3 -3
- data/test/boost_test.rb +3 -9
- data/test/geo_shape_test.rb +0 -4
- data/test/highlight_test.rb +28 -12
- data/test/index_test.rb +9 -10
- data/test/language_test.rb +16 -0
- data/test/marshal_test.rb +6 -1
- data/test/match_test.rb +9 -4
- data/test/model_test.rb +3 -5
- data/test/multi_search_test.rb +0 -7
- data/test/order_test.rb +1 -7
- data/test/pagination_test.rb +1 -1
- data/test/reindex_v2_job_test.rb +6 -11
- data/test/routing_test.rb +1 -1
- data/test/similar_test.rb +2 -2
- data/test/sql_test.rb +0 -31
- data/test/test_helper.rb +37 -23
- metadata +19 -26
- data/test/gemfiles/activerecord31.gemfile +0 -7
- data/test/gemfiles/activerecord32.gemfile +0 -7
- data/test/gemfiles/activerecord40.gemfile +0 -8
- data/test/gemfiles/activerecord41.gemfile +0 -8
- data/test/gemfiles/mongoid2.gemfile +0 -7
- data/test/gemfiles/mongoid3.gemfile +0 -6
- data/test/gemfiles/mongoid4.gemfile +0 -7
- data/test/records_test.rb +0 -10
@@ -4,35 +4,23 @@ module Searchkick
|
|
4
4
|
options = @options
|
5
5
|
language = options[:language]
|
6
6
|
language = language.call if language.respond_to?(:call)
|
7
|
-
|
8
|
-
|
7
|
+
index_type = options[:_type]
|
8
|
+
index_type = index_type.call if index_type.respond_to?(:call)
|
9
9
|
|
10
10
|
if options[:mappings] && !options[:merge_mappings]
|
11
11
|
settings = options[:settings] || {}
|
12
12
|
mappings = options[:mappings]
|
13
13
|
else
|
14
|
-
below22 = Searchkick.server_below?("2.2.0")
|
15
|
-
below50 = Searchkick.server_below?("5.0.0-alpha1")
|
16
14
|
below60 = Searchkick.server_below?("6.0.0-alpha1")
|
17
|
-
default_type =
|
15
|
+
default_type = "text"
|
18
16
|
default_analyzer = :searchkick_index
|
19
|
-
keyword_mapping =
|
20
|
-
if below50
|
21
|
-
{
|
22
|
-
type: default_type,
|
23
|
-
index: "not_analyzed"
|
24
|
-
}
|
25
|
-
else
|
26
|
-
{
|
27
|
-
type: "keyword"
|
28
|
-
}
|
29
|
-
end
|
17
|
+
keyword_mapping = {type: "keyword"}
|
30
18
|
|
31
|
-
all = options.key?(:_all) ? options[:_all] :
|
32
|
-
index_true_value =
|
33
|
-
index_false_value =
|
19
|
+
all = options.key?(:_all) ? options[:_all] : false
|
20
|
+
index_true_value = true
|
21
|
+
index_false_value = false
|
34
22
|
|
35
|
-
keyword_mapping[:ignore_above] =
|
23
|
+
keyword_mapping[:ignore_above] = options[:ignore_above] || 30000
|
36
24
|
|
37
25
|
settings = {
|
38
26
|
analysis: {
|
@@ -40,7 +28,7 @@ module Searchkick
|
|
40
28
|
searchkick_keyword: {
|
41
29
|
type: "custom",
|
42
30
|
tokenizer: "keyword",
|
43
|
-
filter: ["lowercase"] + (options[:stem_conversions]
|
31
|
+
filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : [])
|
44
32
|
},
|
45
33
|
default_analyzer => {
|
46
34
|
type: "custom",
|
@@ -139,7 +127,6 @@ module Searchkick
|
|
139
127
|
},
|
140
128
|
searchkick_stemmer: {
|
141
129
|
# use stemmer if language is lowercase, snowball otherwise
|
142
|
-
# TODO deprecate language option in favor of stemmer
|
143
130
|
type: language == language.to_s.downcase ? "stemmer" : "snowball",
|
144
131
|
language: language || "English"
|
145
132
|
}
|
@@ -155,6 +142,22 @@ module Searchkick
|
|
155
142
|
}
|
156
143
|
}
|
157
144
|
|
145
|
+
if language == "chinese"
|
146
|
+
settings[:analysis][:analyzer].merge!(
|
147
|
+
default_analyzer => {
|
148
|
+
type: "ik_smart"
|
149
|
+
},
|
150
|
+
searchkick_search: {
|
151
|
+
type: "ik_smart"
|
152
|
+
},
|
153
|
+
searchkick_search2: {
|
154
|
+
type: "ik_max_word"
|
155
|
+
}
|
156
|
+
)
|
157
|
+
|
158
|
+
settings[:analysis][:filter].delete(:searchkick_stemmer)
|
159
|
+
end
|
160
|
+
|
158
161
|
if Searchkick.env == "test"
|
159
162
|
settings[:number_of_shards] = 1
|
160
163
|
settings[:number_of_replicas] = 0
|
@@ -175,7 +178,7 @@ module Searchkick
|
|
175
178
|
settings[:analysis][:filter][:searchkick_synonym] = {
|
176
179
|
type: "synonym",
|
177
180
|
# only remove a single space from synonyms so three-word synonyms will fail noisily instead of silently
|
178
|
-
synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |
|
181
|
+
synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s2| s2.sub(/\s+/, "") }.join(",") : s }.map(&:downcase)
|
179
182
|
}
|
180
183
|
# choosing a place for the synonym filter when stemming is not easy
|
181
184
|
# https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
|
@@ -210,7 +213,7 @@ module Searchkick
|
|
210
213
|
end
|
211
214
|
|
212
215
|
if options[:special_characters] == false
|
213
|
-
settings[:analysis][:analyzer].
|
216
|
+
settings[:analysis][:analyzer].each_value do |analyzer_settings|
|
214
217
|
analyzer_settings[:filter].reject! { |f| f == "asciifolding" }
|
215
218
|
end
|
216
219
|
end
|
@@ -320,7 +323,7 @@ module Searchkick
|
|
320
323
|
multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
|
321
324
|
|
322
325
|
mappings = {
|
323
|
-
|
326
|
+
index_type => {
|
324
327
|
properties: mapping,
|
325
328
|
_routing: routing,
|
326
329
|
# https://gist.github.com/kimchy/2898285
|
@@ -338,7 +341,7 @@ module Searchkick
|
|
338
341
|
|
339
342
|
if below60
|
340
343
|
all_enabled = all && (!options[:searchable] || options[:searchable].to_a.map(&:to_s).include?("_all"))
|
341
|
-
mappings[
|
344
|
+
mappings[index_type][:_all] = all_enabled ? analyzed_field_options : {enabled: false}
|
342
345
|
end
|
343
346
|
|
344
347
|
mappings = mappings.deep_merge(options[:mappings] || {})
|
data/lib/searchkick/logging.rb
CHANGED
@@ -129,7 +129,7 @@ module Searchkick
|
|
129
129
|
end
|
130
130
|
|
131
131
|
module SearchkickWithInstrumentation
|
132
|
-
def multi_search(searches
|
132
|
+
def multi_search(searches)
|
133
133
|
event = {
|
134
134
|
name: "Multi Search",
|
135
135
|
body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
|
@@ -167,7 +167,7 @@ module Searchkick
|
|
167
167
|
|
168
168
|
# no easy way to tell which host the client will use
|
169
169
|
host = Searchkick.client.transport.hosts.first
|
170
|
-
debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{payload[:query][:body].to_json}'"
|
170
|
+
debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{payload[:query][:body].to_json}'"
|
171
171
|
end
|
172
172
|
|
173
173
|
def request(event)
|
@@ -189,7 +189,7 @@ module Searchkick
|
|
189
189
|
|
190
190
|
# no easy way to tell which host the client will use
|
191
191
|
host = Searchkick.client.transport.hosts.first
|
192
|
-
debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -d '#{payload[:body]}'"
|
192
|
+
debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -H 'Content-Type: application/json' -d '#{payload[:body]}'"
|
193
193
|
end
|
194
194
|
end
|
195
195
|
|
@@ -232,10 +232,11 @@ module Searchkick
|
|
232
232
|
end
|
233
233
|
end
|
234
234
|
end
|
235
|
-
|
236
|
-
Searchkick::
|
237
|
-
Searchkick::
|
238
|
-
Searchkick.
|
235
|
+
|
236
|
+
Searchkick::Query.prepend(Searchkick::QueryWithInstrumentation)
|
237
|
+
Searchkick::Index.prepend(Searchkick::IndexWithInstrumentation)
|
238
|
+
Searchkick::Indexer.prepend(Searchkick::IndexerWithInstrumentation)
|
239
|
+
Searchkick.singleton_class.prepend(Searchkick::SearchkickWithInstrumentation)
|
239
240
|
Searchkick::LogSubscriber.attach_to :searchkick
|
240
241
|
ActiveSupport.on_load(:action_controller) do
|
241
242
|
include Searchkick::ControllerRuntime
|
data/lib/searchkick/model.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
module Searchkick
|
2
2
|
module Model
|
3
3
|
def searchkick(**options)
|
4
|
+
options = Searchkick.model_options.merge(options)
|
5
|
+
|
4
6
|
unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :conversions, :default_fields,
|
5
7
|
:filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
|
6
8
|
:locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,
|
@@ -12,19 +14,29 @@ module Searchkick
|
|
12
14
|
|
13
15
|
Searchkick.models << self
|
14
16
|
|
15
|
-
options[:_type] ||= -> { searchkick_index.klass_document_type(self, true) }
|
17
|
+
options[:_type] ||= -> { searchkick_index.klass_document_type(self, true) }
|
18
|
+
|
19
|
+
callbacks = options.key?(:callbacks) ? options[:callbacks] : true
|
20
|
+
unless [true, false, :async, :queue].include?(callbacks)
|
21
|
+
raise ArgumentError, "Invalid value for callbacks"
|
22
|
+
end
|
23
|
+
|
24
|
+
index_name =
|
25
|
+
if options[:index_name]
|
26
|
+
options[:index_name]
|
27
|
+
elsif options[:index_prefix].respond_to?(:call)
|
28
|
+
-> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
|
29
|
+
else
|
30
|
+
[options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
|
31
|
+
end
|
16
32
|
|
17
33
|
class_eval do
|
18
34
|
cattr_reader :searchkick_options, :searchkick_klass
|
19
35
|
|
20
|
-
callbacks = options.key?(:callbacks) ? options[:callbacks] : true
|
21
|
-
|
22
36
|
class_variable_set :@@searchkick_options, options.dup
|
23
37
|
class_variable_set :@@searchkick_klass, self
|
24
|
-
class_variable_set :@@
|
25
|
-
class_variable_set :@@
|
26
|
-
(options[:index_prefix].respond_to?(:call) && proc { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }) ||
|
27
|
-
[options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
|
38
|
+
class_variable_set :@@searchkick_index, index_name
|
39
|
+
class_variable_set :@@searchkick_index_cache, {}
|
28
40
|
|
29
41
|
class << self
|
30
42
|
def searchkick_search(term = "*", **options, &block)
|
@@ -33,44 +45,18 @@ module Searchkick
|
|
33
45
|
alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name
|
34
46
|
|
35
47
|
def searchkick_index
|
36
|
-
index = class_variable_get
|
37
|
-
index = index.call if index.respond_to?
|
38
|
-
|
48
|
+
index = class_variable_get(:@@searchkick_index)
|
49
|
+
index = index.call if index.respond_to?(:call)
|
50
|
+
index_cache = class_variable_get(:@@searchkick_index_cache)
|
51
|
+
index_cache[index] ||= Searchkick::Index.new(index, searchkick_options)
|
39
52
|
end
|
40
53
|
alias_method :search_index, :searchkick_index unless method_defined?(:search_index)
|
41
54
|
|
42
|
-
def
|
43
|
-
class_variable_set :@@searchkick_callbacks, true
|
44
|
-
end
|
45
|
-
|
46
|
-
def disable_search_callbacks
|
47
|
-
class_variable_set :@@searchkick_callbacks, false
|
48
|
-
end
|
49
|
-
|
50
|
-
def search_callbacks?
|
51
|
-
class_variable_get(:@@searchkick_callbacks) && Searchkick.callbacks?
|
52
|
-
end
|
53
|
-
|
54
|
-
def searchkick_reindex(method_name = nil, full: false, **options)
|
55
|
+
def searchkick_reindex(method_name = nil, **options)
|
55
56
|
scoped = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) ||
|
56
57
|
(respond_to?(:queryable) && queryable != unscoped.with_default_scope)
|
57
58
|
|
58
|
-
|
59
|
-
|
60
|
-
if method_name
|
61
|
-
# update
|
62
|
-
searchkick_index.import_scope(searchkick_klass, method_name: method_name)
|
63
|
-
searchkick_index.refresh if refresh
|
64
|
-
true
|
65
|
-
elsif scoped && !full
|
66
|
-
# reindex association
|
67
|
-
searchkick_index.import_scope(searchkick_klass)
|
68
|
-
searchkick_index.refresh if refresh
|
69
|
-
true
|
70
|
-
else
|
71
|
-
# full reindex
|
72
|
-
searchkick_index.reindex_scope(searchkick_klass, options)
|
73
|
-
end
|
59
|
+
searchkick_index.reindex(searchkick_klass, method_name, scoped: scoped, **options)
|
74
60
|
end
|
75
61
|
alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
|
76
62
|
|
@@ -79,68 +65,29 @@ module Searchkick
|
|
79
65
|
end
|
80
66
|
end
|
81
67
|
|
82
|
-
|
68
|
+
# always add callbacks, even when callbacks is false
|
69
|
+
# so Model.callbacks block can be used
|
83
70
|
if respond_to?(:after_commit)
|
84
|
-
after_commit
|
71
|
+
after_commit :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
|
85
72
|
elsif respond_to?(:after_save)
|
86
|
-
after_save
|
87
|
-
after_destroy
|
73
|
+
after_save :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
|
74
|
+
after_destroy :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
|
88
75
|
end
|
89
76
|
|
90
|
-
def reindex(method_name = nil,
|
91
|
-
|
92
|
-
|
93
|
-
if mode.nil?
|
94
|
-
mode =
|
95
|
-
if async
|
96
|
-
:async
|
97
|
-
elsif Searchkick.callbacks_value
|
98
|
-
Searchkick.callbacks_value
|
99
|
-
elsif klass_options.key?(:callbacks) && klass_options[:callbacks] != :async
|
100
|
-
# TODO remove 2nd condition in next major version
|
101
|
-
klass_options[:callbacks]
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
case mode
|
106
|
-
when :queue
|
107
|
-
if method_name
|
108
|
-
raise Searchkick::Error, "Partial reindex not supported with queue option"
|
109
|
-
else
|
110
|
-
self.class.searchkick_index.reindex_queue.push(id.to_s)
|
111
|
-
end
|
112
|
-
when :async
|
113
|
-
if method_name
|
114
|
-
# TODO support Mongoid and NoBrainer and non-id primary keys
|
115
|
-
Searchkick::BulkReindexJob.perform_later(
|
116
|
-
class_name: self.class.name,
|
117
|
-
record_ids: [id.to_s],
|
118
|
-
method_name: method_name ? method_name.to_s : nil
|
119
|
-
)
|
120
|
-
else
|
121
|
-
self.class.searchkick_index.reindex_record_async(self)
|
122
|
-
end
|
123
|
-
else
|
124
|
-
if method_name
|
125
|
-
self.class.searchkick_index.update_record(self, method_name)
|
126
|
-
else
|
127
|
-
self.class.searchkick_index.reindex_record(self)
|
128
|
-
end
|
129
|
-
self.class.searchkick_index.refresh if refresh
|
130
|
-
end
|
77
|
+
def reindex(method_name = nil, **options)
|
78
|
+
RecordIndexer.new(self).reindex(method_name, **options)
|
131
79
|
end unless method_defined?(:reindex)
|
132
80
|
|
133
|
-
# TODO remove this method in next major version
|
134
|
-
def reindex_async
|
135
|
-
reindex(async: true)
|
136
|
-
end unless method_defined?(:reindex_async)
|
137
|
-
|
138
81
|
def similar(options = {})
|
139
82
|
self.class.searchkick_index.similar_record(self, options)
|
140
83
|
end unless method_defined?(:similar)
|
141
84
|
|
142
85
|
def search_data
|
143
|
-
respond_to?(:to_hash) ? to_hash : serializable_hash
|
86
|
+
data = respond_to?(:to_hash) ? to_hash : serializable_hash
|
87
|
+
data.delete("id")
|
88
|
+
data.delete("_id")
|
89
|
+
data.delete("_type")
|
90
|
+
data
|
144
91
|
end unless method_defined?(:search_data)
|
145
92
|
|
146
93
|
def should_index?
|
@@ -2,25 +2,24 @@ module Searchkick
|
|
2
2
|
class MultiSearch
|
3
3
|
attr_reader :queries
|
4
4
|
|
5
|
-
def initialize(queries
|
5
|
+
def initialize(queries)
|
6
6
|
@queries = queries
|
7
|
-
@retry_misspellings = retry_misspellings
|
8
7
|
end
|
9
8
|
|
10
9
|
def perform
|
11
10
|
if queries.any?
|
12
|
-
perform_search(queries
|
11
|
+
perform_search(queries)
|
13
12
|
end
|
14
13
|
end
|
15
14
|
|
16
15
|
private
|
17
16
|
|
18
|
-
def perform_search(queries
|
17
|
+
def perform_search(queries)
|
19
18
|
responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
|
20
19
|
|
21
20
|
retry_queries = []
|
22
21
|
queries.each_with_index do |query, i|
|
23
|
-
if
|
22
|
+
if query.retry_misspellings?(responses[i])
|
24
23
|
query.send(:prepare) # okay, since we don't want to expose this method outside Searchkick
|
25
24
|
retry_queries << query
|
26
25
|
else
|
@@ -28,8 +27,8 @@ module Searchkick
|
|
28
27
|
end
|
29
28
|
end
|
30
29
|
|
31
|
-
if
|
32
|
-
perform_search(retry_queries
|
30
|
+
if retry_queries.any?
|
31
|
+
perform_search(retry_queries)
|
33
32
|
end
|
34
33
|
|
35
34
|
queries
|
data/lib/searchkick/query.rb
CHANGED
@@ -135,7 +135,7 @@ module Searchkick
|
|
135
135
|
if searchkick_index
|
136
136
|
puts "Model Search Data"
|
137
137
|
begin
|
138
|
-
pp
|
138
|
+
pp(klass.first(3).map { |r| {index: searchkick_index.record_data(r).merge(data: searchkick_index.send(:search_data, r))}})
|
139
139
|
rescue => e
|
140
140
|
puts "#{e.class.name}: #{e.message}"
|
141
141
|
end
|
@@ -179,14 +179,14 @@ module Searchkick
|
|
179
179
|
e.message.include?("No query registered for [function_score]")
|
180
180
|
)
|
181
181
|
|
182
|
-
raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch
|
182
|
+
raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
|
183
183
|
elsif status_code == 400
|
184
184
|
if (
|
185
185
|
e.message.include?("bool query does not support [filter]") ||
|
186
186
|
e.message.include?("[bool] filter does not support [filter]")
|
187
187
|
)
|
188
188
|
|
189
|
-
raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch
|
189
|
+
raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
|
190
190
|
elsif e.message.include?("[multi_match] analyzer [searchkick_search] not found")
|
191
191
|
raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
|
192
192
|
else
|
@@ -212,15 +212,13 @@ module Searchkick
|
|
212
212
|
|
213
213
|
# pagination
|
214
214
|
page = [options[:page].to_i, 1].max
|
215
|
-
per_page = (options[:limit] || options[:per_page] ||
|
215
|
+
per_page = (options[:limit] || options[:per_page] || 10_000).to_i
|
216
216
|
padding = [options[:padding].to_i, 0].max
|
217
217
|
offset = options[:offset] || (page - 1) * per_page + padding
|
218
218
|
|
219
219
|
# model and eager loading
|
220
220
|
load = options[:load].nil? ? true : options[:load]
|
221
221
|
|
222
|
-
conversions_fields = Array(options[:conversions] || searchkick_options[:conversions]).map(&:to_s)
|
223
|
-
|
224
222
|
all = term == "*"
|
225
223
|
|
226
224
|
@json = options[:body]
|
@@ -228,12 +226,15 @@ module Searchkick
|
|
228
226
|
ignored_options = options.keys & [:aggs, :boost,
|
229
227
|
:boost_by, :boost_by_distance, :boost_where, :conversions, :conversions_term, :exclude, :explain,
|
230
228
|
:fields, :highlight, :indices_boost, :limit, :match, :misspellings, :offset, :operator, :order,
|
231
|
-
:padding, :page, :per_page, :select, :smart_aggs, :suggest, :where]
|
232
|
-
|
229
|
+
:padding, :page, :per_page, :profile, :select, :smart_aggs, :suggest, :where]
|
230
|
+
raise ArgumentError, "Options incompatible with body option: #{ignored_options.join(", ")}" if ignored_options.any?
|
233
231
|
payload = @json
|
234
232
|
else
|
233
|
+
must_not = []
|
234
|
+
should = []
|
235
|
+
|
235
236
|
if options[:similar]
|
236
|
-
|
237
|
+
query = {
|
237
238
|
more_like_this: {
|
238
239
|
like: term,
|
239
240
|
min_doc_freq: 1,
|
@@ -241,11 +242,14 @@ module Searchkick
|
|
241
242
|
analyzer: "searchkick_search2"
|
242
243
|
}
|
243
244
|
}
|
245
|
+
if fields.all? { |f| f.start_with?("*.") }
|
246
|
+
raise ArgumentError, "Must specify fields to search"
|
247
|
+
end
|
244
248
|
if fields != ["_all"]
|
245
|
-
|
249
|
+
query[:more_like_this][:fields] = fields
|
246
250
|
end
|
247
251
|
elsif all
|
248
|
-
|
252
|
+
query = {
|
249
253
|
match_all: {}
|
250
254
|
}
|
251
255
|
else
|
@@ -327,10 +331,19 @@ module Searchkick
|
|
327
331
|
end
|
328
332
|
|
329
333
|
if misspellings != false && match_type == :match
|
330
|
-
qs.concat
|
334
|
+
qs.concat(qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) })
|
331
335
|
end
|
332
336
|
|
333
|
-
|
337
|
+
if field.start_with?("*.")
|
338
|
+
q2 = qs.map { |q| {multi_match: q.merge(fields: [field], type: match_type == :match_phrase ? "phrase" : "best_fields")} }
|
339
|
+
if below61?
|
340
|
+
q2.each do |q|
|
341
|
+
q[:multi_match].delete(:fuzzy_transpositions)
|
342
|
+
end
|
343
|
+
end
|
344
|
+
else
|
345
|
+
q2 = qs.map { |q| {match_type => {field => q}} }
|
346
|
+
end
|
334
347
|
|
335
348
|
# boost exact matches more
|
336
349
|
if field =~ /\.word_(start|middle|end)\z/ && searchkick_options[:word] != false
|
@@ -348,28 +361,11 @@ module Searchkick
|
|
348
361
|
queries_to_add.concat(q2)
|
349
362
|
end
|
350
363
|
|
351
|
-
|
352
|
-
must_not =
|
353
|
-
Array(options[:exclude]).map do |phrase|
|
354
|
-
{
|
355
|
-
match_phrase: {
|
356
|
-
exclude_field => {
|
357
|
-
query: phrase,
|
358
|
-
analyzer: exclude_analyzer
|
359
|
-
}
|
360
|
-
}
|
361
|
-
}
|
362
|
-
end
|
364
|
+
queries.concat(queries_to_add)
|
363
365
|
|
364
|
-
|
365
|
-
|
366
|
-
should: queries_to_add,
|
367
|
-
must_not: must_not
|
368
|
-
}
|
369
|
-
}]
|
366
|
+
if options[:exclude]
|
367
|
+
must_not.concat(set_exclude(exclude_field, exclude_analyzer))
|
370
368
|
end
|
371
|
-
|
372
|
-
queries.concat(queries_to_add)
|
373
369
|
end
|
374
370
|
|
375
371
|
payload = {
|
@@ -378,78 +374,15 @@ module Searchkick
|
|
378
374
|
}
|
379
375
|
}
|
380
376
|
|
381
|
-
|
382
|
-
shoulds = []
|
383
|
-
conversions_fields.each do |conversions_field|
|
384
|
-
# wrap payload in a bool query
|
385
|
-
script_score = {field_value_factor: {field: "#{conversions_field}.count"}}
|
386
|
-
|
387
|
-
shoulds << {
|
388
|
-
nested: {
|
389
|
-
path: conversions_field,
|
390
|
-
score_mode: "sum",
|
391
|
-
query: {
|
392
|
-
function_score: {
|
393
|
-
boost_mode: "replace",
|
394
|
-
query: {
|
395
|
-
match: {
|
396
|
-
"#{conversions_field}.query" => options[:conversions_term] || term
|
397
|
-
}
|
398
|
-
}
|
399
|
-
}.merge(script_score)
|
400
|
-
}
|
401
|
-
}
|
402
|
-
}
|
403
|
-
end
|
404
|
-
payload = {
|
405
|
-
bool: {
|
406
|
-
must: payload,
|
407
|
-
should: shoulds
|
408
|
-
}
|
409
|
-
}
|
410
|
-
end
|
411
|
-
end
|
412
|
-
|
413
|
-
custom_filters = []
|
414
|
-
multiply_filters = []
|
415
|
-
|
416
|
-
set_boost_by(multiply_filters, custom_filters)
|
417
|
-
set_boost_where(custom_filters)
|
418
|
-
set_boost_by_distance(custom_filters) if options[:boost_by_distance]
|
377
|
+
should.concat(set_conversions)
|
419
378
|
|
420
|
-
|
421
|
-
payload = {
|
422
|
-
function_score: {
|
423
|
-
functions: custom_filters,
|
424
|
-
query: payload,
|
425
|
-
score_mode: "sum"
|
426
|
-
}
|
427
|
-
}
|
428
|
-
end
|
429
|
-
|
430
|
-
if multiply_filters.any?
|
431
|
-
payload = {
|
432
|
-
function_score: {
|
433
|
-
functions: multiply_filters,
|
434
|
-
query: payload,
|
435
|
-
score_mode: "multiply"
|
436
|
-
}
|
437
|
-
}
|
379
|
+
query = payload
|
438
380
|
end
|
439
381
|
|
440
382
|
payload = {
|
441
|
-
query: payload,
|
442
383
|
size: per_page,
|
443
384
|
from: offset
|
444
385
|
}
|
445
|
-
payload[:explain] = options[:explain] if options[:explain]
|
446
|
-
payload[:profile] = options[:profile] if options[:profile]
|
447
|
-
|
448
|
-
# order
|
449
|
-
set_order(payload) if options[:order]
|
450
|
-
|
451
|
-
# indices_boost
|
452
|
-
set_boost_by_indices(payload)
|
453
386
|
|
454
387
|
# type when inheritance
|
455
388
|
where = (options[:where] || {}).dup
|
@@ -465,8 +398,26 @@ module Searchkick
|
|
465
398
|
# aggregations
|
466
399
|
set_aggregations(payload, filters, post_filters) if options[:aggs]
|
467
400
|
|
468
|
-
# filters
|
469
|
-
|
401
|
+
# post filters
|
402
|
+
set_post_filters(payload, post_filters) if post_filters.any?
|
403
|
+
|
404
|
+
custom_filters = []
|
405
|
+
multiply_filters = []
|
406
|
+
|
407
|
+
set_boost_by(multiply_filters, custom_filters)
|
408
|
+
set_boost_where(custom_filters)
|
409
|
+
set_boost_by_distance(custom_filters) if options[:boost_by_distance]
|
410
|
+
|
411
|
+
payload[:query] = build_query(query, filters, should, must_not, custom_filters, multiply_filters)
|
412
|
+
|
413
|
+
payload[:explain] = options[:explain] if options[:explain]
|
414
|
+
payload[:profile] = options[:profile] if options[:profile]
|
415
|
+
|
416
|
+
# order
|
417
|
+
set_order(payload) if options[:order]
|
418
|
+
|
419
|
+
# indices_boost
|
420
|
+
set_boost_by_indices(payload)
|
470
421
|
|
471
422
|
# suggestions
|
472
423
|
set_suggestions(payload, options[:suggest]) if options[:suggest]
|
@@ -512,7 +463,7 @@ module Searchkick
|
|
512
463
|
def set_fields
|
513
464
|
boost_fields = {}
|
514
465
|
fields = options[:fields] || searchkick_options[:default_fields] || searchkick_options[:searchable]
|
515
|
-
all = searchkick_options.key?(:_all) ? searchkick_options[:_all] :
|
466
|
+
all = searchkick_options.key?(:_all) ? searchkick_options[:_all] : false
|
516
467
|
default_match = options[:match] || searchkick_options[:match] || :word
|
517
468
|
fields =
|
518
469
|
if fields
|
@@ -529,12 +480,88 @@ module Searchkick
|
|
529
480
|
["_all.phrase"]
|
530
481
|
elsif term == "*"
|
531
482
|
[]
|
532
|
-
|
483
|
+
elsif default_match == :exact
|
533
484
|
raise ArgumentError, "Must specify fields to search"
|
485
|
+
else
|
486
|
+
[default_match == :word ? "*.analyzed" : "*.#{default_match}"]
|
534
487
|
end
|
535
488
|
[boost_fields, fields]
|
536
489
|
end
|
537
490
|
|
491
|
+
def build_query(query, filters, should, must_not, custom_filters, multiply_filters)
|
492
|
+
if filters.any? || must_not.any? || should.any?
|
493
|
+
bool = {must: query}
|
494
|
+
bool[:filter] = filters if filters.any? # where
|
495
|
+
bool[:must_not] = must_not if must_not.any? # exclude
|
496
|
+
bool[:should] = should if should.any? # conversions
|
497
|
+
query = {bool: bool}
|
498
|
+
end
|
499
|
+
|
500
|
+
if custom_filters.any?
|
501
|
+
query = {
|
502
|
+
function_score: {
|
503
|
+
functions: custom_filters,
|
504
|
+
query: query,
|
505
|
+
score_mode: "sum"
|
506
|
+
}
|
507
|
+
}
|
508
|
+
end
|
509
|
+
|
510
|
+
if multiply_filters.any?
|
511
|
+
query = {
|
512
|
+
function_score: {
|
513
|
+
functions: multiply_filters,
|
514
|
+
query: query,
|
515
|
+
score_mode: "multiply"
|
516
|
+
}
|
517
|
+
}
|
518
|
+
end
|
519
|
+
|
520
|
+
query
|
521
|
+
end
|
522
|
+
|
523
|
+
def set_conversions
|
524
|
+
conversions_fields = Array(options[:conversions] || searchkick_options[:conversions]).map(&:to_s)
|
525
|
+
if conversions_fields.present? && options[:conversions] != false
|
526
|
+
conversions_fields.map do |conversions_field|
|
527
|
+
{
|
528
|
+
nested: {
|
529
|
+
path: conversions_field,
|
530
|
+
score_mode: "sum",
|
531
|
+
query: {
|
532
|
+
function_score: {
|
533
|
+
boost_mode: "replace",
|
534
|
+
query: {
|
535
|
+
match: {
|
536
|
+
"#{conversions_field}.query" => options[:conversions_term] || term
|
537
|
+
}
|
538
|
+
},
|
539
|
+
field_value_factor: {
|
540
|
+
field: "#{conversions_field}.count"
|
541
|
+
}
|
542
|
+
}
|
543
|
+
}
|
544
|
+
}
|
545
|
+
}
|
546
|
+
end
|
547
|
+
else
|
548
|
+
[]
|
549
|
+
end
|
550
|
+
end
|
551
|
+
|
552
|
+
def set_exclude(field, analyzer)
|
553
|
+
Array(options[:exclude]).map do |phrase|
|
554
|
+
{
|
555
|
+
multi_match: {
|
556
|
+
fields: [field],
|
557
|
+
query: phrase,
|
558
|
+
analyzer: analyzer,
|
559
|
+
type: "phrase"
|
560
|
+
}
|
561
|
+
}
|
562
|
+
end
|
563
|
+
end
|
564
|
+
|
538
565
|
def set_boost_by_distance(custom_filters)
|
539
566
|
boost_by_distance = options[:boost_by_distance] || {}
|
540
567
|
|
@@ -564,11 +591,11 @@ module Searchkick
|
|
564
591
|
if boost_by.is_a?(Array)
|
565
592
|
boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
|
566
593
|
elsif boost_by.is_a?(Hash)
|
567
|
-
multiply_by, boost_by = boost_by.partition { |_, v| v
|
594
|
+
multiply_by, boost_by = boost_by.partition { |_, v| v.delete(:boost_mode) == "multiply" }.map { |i| Hash[i] }
|
568
595
|
end
|
569
596
|
boost_by[options[:boost]] = {factor: 1} if options[:boost]
|
570
597
|
|
571
|
-
custom_filters.concat boost_filters(boost_by,
|
598
|
+
custom_filters.concat boost_filters(boost_by, modifier: "ln2p")
|
572
599
|
multiply_filters.concat boost_filters(multiply_by || {})
|
573
600
|
end
|
574
601
|
|
@@ -631,7 +658,8 @@ module Searchkick
|
|
631
658
|
|
632
659
|
def set_highlights(payload, fields)
|
633
660
|
payload[:highlight] = {
|
634
|
-
fields: Hash[fields.map { |f| [f, {}] }]
|
661
|
+
fields: Hash[fields.map { |f| [f, {}] }],
|
662
|
+
fragment_size: below60? ? 30000 : 0
|
635
663
|
}
|
636
664
|
|
637
665
|
if options[:highlight].is_a?(Hash)
|
@@ -683,7 +711,7 @@ module Searchkick
|
|
683
711
|
ranges: agg_options[:date_ranges]
|
684
712
|
}.merge(shared_agg_options)
|
685
713
|
}
|
686
|
-
elsif histogram = agg_options[:date_histogram]
|
714
|
+
elsif (histogram = agg_options[:date_histogram])
|
687
715
|
interval = histogram[:interval]
|
688
716
|
payload[:aggs][field] = {
|
689
717
|
date_histogram: {
|
@@ -691,7 +719,7 @@ module Searchkick
|
|
691
719
|
interval: interval
|
692
720
|
}
|
693
721
|
}
|
694
|
-
elsif metric = @@metric_aggs.find { |k| agg_options.has_key?(k) }
|
722
|
+
elsif (metric = @@metric_aggs.find { |k| agg_options.has_key?(k) })
|
695
723
|
payload[:aggs][field] = {
|
696
724
|
metric => {
|
697
725
|
field: agg_options[metric][:field] || field
|
@@ -735,30 +763,18 @@ module Searchkick
|
|
735
763
|
end
|
736
764
|
end
|
737
765
|
|
738
|
-
def
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
filter: post_filters
|
743
|
-
}
|
766
|
+
def set_post_filters(payload, post_filters)
|
767
|
+
payload[:post_filter] = {
|
768
|
+
bool: {
|
769
|
+
filter: post_filters
|
744
770
|
}
|
745
|
-
|
746
|
-
|
747
|
-
if filters.any?
|
748
|
-
# more efficient query if no aggs
|
749
|
-
payload[:query] = {
|
750
|
-
bool: {
|
751
|
-
must: payload[:query],
|
752
|
-
filter: filters
|
753
|
-
}
|
754
|
-
}
|
755
|
-
end
|
771
|
+
}
|
756
772
|
end
|
757
773
|
|
758
774
|
# TODO id transformation for arrays
|
759
775
|
def set_order(payload)
|
760
776
|
order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
|
761
|
-
id_field =
|
777
|
+
id_field = :_uid
|
762
778
|
payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
|
763
779
|
end
|
764
780
|
|
@@ -889,50 +905,37 @@ module Searchkick
|
|
889
905
|
end
|
890
906
|
|
891
907
|
def custom_filter(field, value, factor)
|
892
|
-
|
893
|
-
|
894
|
-
|
895
|
-
|
896
|
-
|
897
|
-
|
898
|
-
|
899
|
-
|
908
|
+
{
|
909
|
+
filter: where_filters(field => value),
|
910
|
+
weight: factor
|
911
|
+
}
|
912
|
+
end
|
913
|
+
|
914
|
+
def boost_filter(field, factor: 1, modifier: nil, missing: nil)
|
915
|
+
script_score = {
|
916
|
+
field_value_factor: {
|
917
|
+
field: field,
|
918
|
+
factor: factor.to_f,
|
919
|
+
modifier: modifier
|
900
920
|
}
|
921
|
+
}
|
922
|
+
|
923
|
+
if missing
|
924
|
+
script_score[:field_value_factor][:missing] = missing.to_f
|
901
925
|
else
|
902
|
-
{
|
903
|
-
|
904
|
-
|
926
|
+
script_score[:filter] = {
|
927
|
+
exists: {
|
928
|
+
field: field
|
929
|
+
}
|
905
930
|
}
|
906
931
|
end
|
932
|
+
|
933
|
+
script_score
|
907
934
|
end
|
908
935
|
|
909
|
-
def boost_filters(boost_by,
|
936
|
+
def boost_filters(boost_by, modifier: nil)
|
910
937
|
boost_by.map do |field, value|
|
911
|
-
|
912
|
-
value[:factor] ||= 1
|
913
|
-
script_score = {
|
914
|
-
field_value_factor: {
|
915
|
-
field: field,
|
916
|
-
factor: value[:factor].to_f,
|
917
|
-
modifier: value[:modifier] || (log ? "ln2p" : nil)
|
918
|
-
}
|
919
|
-
}
|
920
|
-
|
921
|
-
if value[:missing]
|
922
|
-
if below50?
|
923
|
-
raise ArgumentError, "The missing option for boost_by is not supported in Elasticsearch < 5"
|
924
|
-
else
|
925
|
-
script_score[:field_value_factor][:missing] = value[:missing].to_f
|
926
|
-
end
|
927
|
-
else
|
928
|
-
script_score[:filter] = {
|
929
|
-
exists: {
|
930
|
-
field: field
|
931
|
-
}
|
932
|
-
}
|
933
|
-
end
|
934
|
-
|
935
|
-
script_score
|
938
|
+
boost_filter(field, modifier: modifier, **value)
|
936
939
|
end
|
937
940
|
end
|
938
941
|
|
@@ -957,12 +960,12 @@ module Searchkick
|
|
957
960
|
end
|
958
961
|
end
|
959
962
|
|
960
|
-
def below50?
|
961
|
-
Searchkick.server_below?("5.0.0-alpha1")
|
962
|
-
end
|
963
|
-
|
964
963
|
def below60?
|
965
964
|
Searchkick.server_below?("6.0.0-alpha1")
|
966
965
|
end
|
966
|
+
|
967
|
+
def below61?
|
968
|
+
Searchkick.server_below?("6.1.0-alpha1")
|
969
|
+
end
|
967
970
|
end
|
968
971
|
end
|