searchkick_bharthur 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +22 -0
- data/.travis.yml +44 -0
- data/CHANGELOG.md +360 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +1443 -0
- data/Rakefile +8 -0
- data/lib/searchkick/index.rb +662 -0
- data/lib/searchkick/logging.rb +185 -0
- data/lib/searchkick/middleware.rb +12 -0
- data/lib/searchkick/model.rb +105 -0
- data/lib/searchkick/query.rb +845 -0
- data/lib/searchkick/reindex_job.rb +26 -0
- data/lib/searchkick/reindex_v2_job.rb +23 -0
- data/lib/searchkick/results.rb +211 -0
- data/lib/searchkick/tasks.rb +33 -0
- data/lib/searchkick/version.rb +3 -0
- data/lib/searchkick.rb +159 -0
- data/searchkick.gemspec +28 -0
- data/test/aggs_test.rb +115 -0
- data/test/autocomplete_test.rb +65 -0
- data/test/boost_test.rb +144 -0
- data/test/callbacks_test.rb +27 -0
- data/test/ci/before_install.sh +21 -0
- data/test/dangerous_reindex_test.rb +27 -0
- data/test/facets_test.rb +90 -0
- data/test/gemfiles/activerecord31.gemfile +7 -0
- data/test/gemfiles/activerecord32.gemfile +7 -0
- data/test/gemfiles/activerecord40.gemfile +8 -0
- data/test/gemfiles/activerecord41.gemfile +8 -0
- data/test/gemfiles/activerecord50.gemfile +7 -0
- data/test/gemfiles/apartment.gemfile +8 -0
- data/test/gemfiles/mongoid2.gemfile +7 -0
- data/test/gemfiles/mongoid3.gemfile +6 -0
- data/test/gemfiles/mongoid4.gemfile +7 -0
- data/test/gemfiles/mongoid5.gemfile +7 -0
- data/test/gemfiles/nobrainer.gemfile +6 -0
- data/test/highlight_test.rb +63 -0
- data/test/index_test.rb +120 -0
- data/test/inheritance_test.rb +78 -0
- data/test/match_test.rb +227 -0
- data/test/misspellings_test.rb +46 -0
- data/test/model_test.rb +42 -0
- data/test/multi_search_test.rb +22 -0
- data/test/multi_tenancy_test.rb +22 -0
- data/test/order_test.rb +44 -0
- data/test/pagination_test.rb +53 -0
- data/test/query_test.rb +13 -0
- data/test/records_test.rb +8 -0
- data/test/reindex_job_test.rb +31 -0
- data/test/reindex_v2_job_test.rb +32 -0
- data/test/routing_test.rb +13 -0
- data/test/should_index_test.rb +32 -0
- data/test/similar_test.rb +28 -0
- data/test/sql_test.rb +196 -0
- data/test/suggest_test.rb +80 -0
- data/test/synonyms_test.rb +54 -0
- data/test/test_helper.rb +361 -0
- data/test/where_test.rb +171 -0
- metadata +231 -0
@@ -0,0 +1,845 @@
|
|
1
|
+
module Searchkick
|
2
|
+
class Query
|
3
|
+
extend Forwardable
|
4
|
+
|
5
|
+
attr_reader :klass, :term, :options
|
6
|
+
attr_accessor :body
|
7
|
+
|
8
|
+
def_delegators :execute, :map, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary,
|
9
|
+
:records, :results, :suggestions, :each_with_hit, :with_details, :facets, :aggregations, :aggs,
|
10
|
+
:took, :error, :model_name, :entry_name, :total_count, :total_entries,
|
11
|
+
:current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
|
12
|
+
:offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
|
13
|
+
:out_of_range?, :hits
|
14
|
+
|
15
|
+
|
16
|
+
def initialize(klass, term, options = {})
|
17
|
+
if term.is_a?(Hash)
|
18
|
+
options = term
|
19
|
+
term = "*"
|
20
|
+
else
|
21
|
+
term = term.to_s
|
22
|
+
end
|
23
|
+
|
24
|
+
if options[:emoji]
|
25
|
+
term = EmojiParser.parse_unicode(term) { |e| " #{e.name} " }.strip
|
26
|
+
end
|
27
|
+
|
28
|
+
@klass = klass
|
29
|
+
@term = term
|
30
|
+
@options = options
|
31
|
+
@match_suffix = options[:match] || searchkick_options[:match] || "analyzed"
|
32
|
+
|
33
|
+
prepare
|
34
|
+
end
|
35
|
+
|
36
|
+
def searchkick_index
|
37
|
+
klass ? klass.searchkick_index : nil
|
38
|
+
end
|
39
|
+
|
40
|
+
def searchkick_options
|
41
|
+
klass ? klass.searchkick_options : {}
|
42
|
+
end
|
43
|
+
|
44
|
+
def searchkick_klass
|
45
|
+
klass ? klass.searchkick_klass : nil
|
46
|
+
end
|
47
|
+
|
48
|
+
def params
|
49
|
+
index =
|
50
|
+
if options[:index_name]
|
51
|
+
Array(options[:index_name]).map { |v| v.respond_to?(:searchkick_index) ? v.searchkick_index.name : v }.join(",")
|
52
|
+
elsif searchkick_index
|
53
|
+
searchkick_index.name
|
54
|
+
else
|
55
|
+
"_all"
|
56
|
+
end
|
57
|
+
|
58
|
+
params = {
|
59
|
+
index: index,
|
60
|
+
body: body
|
61
|
+
}
|
62
|
+
params.merge!(type: @type) if @type
|
63
|
+
params.merge!(routing: @routing) if @routing
|
64
|
+
params
|
65
|
+
end
|
66
|
+
|
67
|
+
def execute
|
68
|
+
@execute ||= begin
|
69
|
+
begin
|
70
|
+
response = execute_search
|
71
|
+
if @misspellings_below && response["hits"]["total"] < @misspellings_below
|
72
|
+
prepare
|
73
|
+
response = execute_search
|
74
|
+
end
|
75
|
+
rescue => e # TODO rescue type
|
76
|
+
handle_error(e)
|
77
|
+
end
|
78
|
+
handle_response(response)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def to_curl
|
83
|
+
query = params
|
84
|
+
type = query[:type]
|
85
|
+
index = query[:index].is_a?(Array) ? query[:index].join(",") : query[:index]
|
86
|
+
|
87
|
+
# no easy way to tell which host the client will use
|
88
|
+
host = Searchkick.client.transport.hosts.first
|
89
|
+
credentials = (host[:user] || host[:password]) ? "#{host[:user]}:#{host[:password]}@" : nil
|
90
|
+
"curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{query[:body].to_json}'"
|
91
|
+
end
|
92
|
+
|
93
|
+
def handle_response(response)
|
94
|
+
# apply facet limit in client due to
|
95
|
+
# https://github.com/elasticsearch/elasticsearch/issues/1305
|
96
|
+
@facet_limits.each do |field, limit|
|
97
|
+
field = field.to_s
|
98
|
+
facet = response["facets"][field]
|
99
|
+
response["facets"][field]["terms"] = facet["terms"].first(limit)
|
100
|
+
response["facets"][field]["other"] = facet["total"] - facet["terms"].sum { |term| term["count"] }
|
101
|
+
end
|
102
|
+
|
103
|
+
opts = {
|
104
|
+
page: @page,
|
105
|
+
per_page: @per_page,
|
106
|
+
padding: @padding,
|
107
|
+
load: @load,
|
108
|
+
includes: options[:include] || options[:includes],
|
109
|
+
json: !options[:json].nil?,
|
110
|
+
match_suffix: @match_suffix,
|
111
|
+
highlighted_fields: @highlighted_fields || []
|
112
|
+
}
|
113
|
+
|
114
|
+
# set execute for multi search
|
115
|
+
@execute = Searchkick::Results.new(searchkick_klass, response, opts)
|
116
|
+
end
|
117
|
+
|
118
|
+
private
|
119
|
+
|
120
|
+
def handle_error(e)
|
121
|
+
status_code = e.message[1..3].to_i
|
122
|
+
if status_code == 404
|
123
|
+
raise MissingIndexError, "Index missing - run #{reindex_command}"
|
124
|
+
elsif status_code == 500 && (
|
125
|
+
e.message.include?("IllegalArgumentException[minimumSimilarity >= 1]") ||
|
126
|
+
e.message.include?("No query registered for [multi_match]") ||
|
127
|
+
e.message.include?("[match] query does not support [cutoff_frequency]]") ||
|
128
|
+
e.message.include?("No query registered for [function_score]]")
|
129
|
+
)
|
130
|
+
|
131
|
+
raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 1.0 or greater"
|
132
|
+
elsif status_code == 400
|
133
|
+
if e.message.include?("[multi_match] analyzer [searchkick_search] not found")
|
134
|
+
raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
|
135
|
+
else
|
136
|
+
raise InvalidQueryError, e.message
|
137
|
+
end
|
138
|
+
else
|
139
|
+
raise e
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def reindex_command
|
144
|
+
searchkick_klass ? "#{searchkick_klass.name}.reindex" : "reindex"
|
145
|
+
end
|
146
|
+
|
147
|
+
def execute_search
|
148
|
+
Searchkick.client.search(params)
|
149
|
+
end
|
150
|
+
|
151
|
+
def prepare
|
152
|
+
boost_fields, fields = set_fields
|
153
|
+
|
154
|
+
operator = options[:operator] || (options[:partial] ? "or" : "and")
|
155
|
+
|
156
|
+
# pagination
|
157
|
+
page = [options[:page].to_i, 1].max
|
158
|
+
per_page = (options[:limit] || options[:per_page] || 1_000).to_i
|
159
|
+
padding = [options[:padding].to_i, 0].max
|
160
|
+
offset = options[:offset] || (page - 1) * per_page + padding
|
161
|
+
|
162
|
+
# model and eagar loading
|
163
|
+
load = options[:load].nil? ? true : options[:load]
|
164
|
+
|
165
|
+
conversions_field = searchkick_options[:conversions]
|
166
|
+
personalize_field = searchkick_options[:personalize]
|
167
|
+
|
168
|
+
all = term == "*"
|
169
|
+
|
170
|
+
options[:json] ||= options[:body]
|
171
|
+
if options[:json]
|
172
|
+
payload = options[:json]
|
173
|
+
else
|
174
|
+
if options[:query]
|
175
|
+
payload = options[:query]
|
176
|
+
elsif options[:similar]
|
177
|
+
payload = {
|
178
|
+
more_like_this: {
|
179
|
+
fields: fields,
|
180
|
+
like_text: term,
|
181
|
+
min_doc_freq: 1,
|
182
|
+
min_term_freq: 1,
|
183
|
+
analyzer: "searchkick_search2"
|
184
|
+
}
|
185
|
+
}
|
186
|
+
elsif all
|
187
|
+
payload = {
|
188
|
+
match_all: {}
|
189
|
+
}
|
190
|
+
else
|
191
|
+
if options[:autocomplete]
|
192
|
+
payload = {
|
193
|
+
multi_match: {
|
194
|
+
fields: fields,
|
195
|
+
query: term,
|
196
|
+
analyzer: "searchkick_autocomplete_search"
|
197
|
+
}
|
198
|
+
}
|
199
|
+
else
|
200
|
+
queries = []
|
201
|
+
|
202
|
+
misspellings =
|
203
|
+
if options.key?(:misspellings)
|
204
|
+
options[:misspellings]
|
205
|
+
elsif options.key?(:mispellings)
|
206
|
+
options[:mispellings] # why not?
|
207
|
+
else
|
208
|
+
true
|
209
|
+
end
|
210
|
+
|
211
|
+
if misspellings.is_a?(Hash) && misspellings[:below] && !@misspellings_below
|
212
|
+
@misspellings_below = misspellings[:below].to_i
|
213
|
+
misspellings = false
|
214
|
+
end
|
215
|
+
|
216
|
+
if misspellings != false
|
217
|
+
edit_distance = (misspellings.is_a?(Hash) && (misspellings[:edit_distance] || misspellings[:distance])) || 1
|
218
|
+
transpositions =
|
219
|
+
if misspellings.is_a?(Hash) && misspellings.key?(:transpositions)
|
220
|
+
{fuzzy_transpositions: misspellings[:transpositions]}
|
221
|
+
elsif below14?
|
222
|
+
{}
|
223
|
+
else
|
224
|
+
{fuzzy_transpositions: true}
|
225
|
+
end
|
226
|
+
prefix_length = (misspellings.is_a?(Hash) && misspellings[:prefix_length]) || 0
|
227
|
+
default_max_expansions = @misspellings_below ? 20 : 3
|
228
|
+
max_expansions = (misspellings.is_a?(Hash) && misspellings[:max_expansions]) || default_max_expansions
|
229
|
+
end
|
230
|
+
|
231
|
+
fields.each do |field|
|
232
|
+
qs = []
|
233
|
+
|
234
|
+
factor = boost_fields[field] || 1
|
235
|
+
shared_options = {
|
236
|
+
query: term,
|
237
|
+
boost: 10 * factor
|
238
|
+
}
|
239
|
+
|
240
|
+
match_type =
|
241
|
+
if field.end_with?(".phrase")
|
242
|
+
field = field.sub(/\.phrase\z/, ".analyzed")
|
243
|
+
:match_phrase
|
244
|
+
else
|
245
|
+
:match
|
246
|
+
end
|
247
|
+
|
248
|
+
shared_options[:operator] = operator if match_type == :match || below50?
|
249
|
+
|
250
|
+
if field == "_all" || field.end_with?(".analyzed")
|
251
|
+
shared_options[:cutoff_frequency] = 0.001 unless operator == "and" || misspellings == false
|
252
|
+
qs.concat [
|
253
|
+
shared_options.merge(analyzer: "searchkick_search"),
|
254
|
+
shared_options.merge(analyzer: "searchkick_search2")
|
255
|
+
]
|
256
|
+
elsif field.end_with?(".exact")
|
257
|
+
f = field.split(".")[0..-2].join(".")
|
258
|
+
queries << {match: {f => shared_options.merge(analyzer: "keyword")}}
|
259
|
+
else
|
260
|
+
analyzer = field.match(/\.word_(start|middle|end)\z/) ? "searchkick_word_search" : "searchkick_autocomplete_search"
|
261
|
+
qs << shared_options.merge(analyzer: analyzer)
|
262
|
+
end
|
263
|
+
|
264
|
+
if misspellings != false && (match_type == :match || below50?)
|
265
|
+
qs.concat qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) }
|
266
|
+
end
|
267
|
+
|
268
|
+
queries.concat(qs.map { |q| {match_type => {field => q}} })
|
269
|
+
end
|
270
|
+
|
271
|
+
payload = {
|
272
|
+
dis_max: {
|
273
|
+
queries: queries
|
274
|
+
}
|
275
|
+
}
|
276
|
+
end
|
277
|
+
|
278
|
+
if conversions_field && options[:conversions] != false
|
279
|
+
# wrap payload in a bool query
|
280
|
+
script_score =
|
281
|
+
if below12?
|
282
|
+
{script_score: {script: "doc['count'].value"}}
|
283
|
+
else
|
284
|
+
{field_value_factor: {field: "#{conversions_field}.count"}}
|
285
|
+
end
|
286
|
+
|
287
|
+
payload = {
|
288
|
+
bool: {
|
289
|
+
must: payload,
|
290
|
+
should: {
|
291
|
+
nested: {
|
292
|
+
path: conversions_field,
|
293
|
+
score_mode: "sum",
|
294
|
+
query: {
|
295
|
+
function_score: {
|
296
|
+
boost_mode: "replace",
|
297
|
+
query: {
|
298
|
+
match: {
|
299
|
+
"#{conversions_field}.query" => term
|
300
|
+
}
|
301
|
+
}
|
302
|
+
}.merge(script_score)
|
303
|
+
}
|
304
|
+
}
|
305
|
+
}
|
306
|
+
}
|
307
|
+
}
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
311
|
+
custom_filters = []
|
312
|
+
multiply_filters = []
|
313
|
+
|
314
|
+
set_boost_by(multiply_filters, custom_filters)
|
315
|
+
set_boost_where(custom_filters, personalize_field)
|
316
|
+
set_boost_by_distance(custom_filters) if options[:boost_by_distance]
|
317
|
+
|
318
|
+
if custom_filters.any?
|
319
|
+
payload = {
|
320
|
+
function_score: {
|
321
|
+
functions: custom_filters,
|
322
|
+
query: payload,
|
323
|
+
score_mode: "sum"
|
324
|
+
}
|
325
|
+
}
|
326
|
+
end
|
327
|
+
|
328
|
+
if multiply_filters.any?
|
329
|
+
payload = {
|
330
|
+
function_score: {
|
331
|
+
functions: multiply_filters,
|
332
|
+
query: payload,
|
333
|
+
score_mode: "multiply"
|
334
|
+
}
|
335
|
+
}
|
336
|
+
end
|
337
|
+
|
338
|
+
payload = {
|
339
|
+
query: payload,
|
340
|
+
size: per_page,
|
341
|
+
from: offset
|
342
|
+
}
|
343
|
+
payload[:explain] = options[:explain] if options[:explain]
|
344
|
+
|
345
|
+
# order
|
346
|
+
set_order(payload) if options[:order]
|
347
|
+
|
348
|
+
# filters
|
349
|
+
filters = where_filters(options[:where])
|
350
|
+
set_filters(payload, filters) if filters.any?
|
351
|
+
|
352
|
+
# facets
|
353
|
+
set_facets(payload) if options[:facets]
|
354
|
+
|
355
|
+
# aggregations
|
356
|
+
set_aggregations(payload) if options[:aggs]
|
357
|
+
|
358
|
+
# suggestions
|
359
|
+
set_suggestions(payload) if options[:suggest]
|
360
|
+
|
361
|
+
# highlight
|
362
|
+
set_highlights(payload, fields) if options[:highlight]
|
363
|
+
|
364
|
+
# An empty array will cause only the _id and _type for each hit to be returned
|
365
|
+
# doc for :select - http://www.elasticsearch.org/guide/reference/api/search/fields/
|
366
|
+
# doc for :select_v2 - https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html
|
367
|
+
if options[:select]
|
368
|
+
payload[:fields] = options[:select] if options[:select] != true
|
369
|
+
elsif options[:select_v2]
|
370
|
+
if options[:select_v2] == []
|
371
|
+
payload[:fields] = [] # intuitively [] makes sense to return no fields, but ES by default returns all fields
|
372
|
+
else
|
373
|
+
payload[:_source] = options[:select_v2]
|
374
|
+
end
|
375
|
+
elsif load
|
376
|
+
# don't need any fields since we're going to load them from the DB anyways
|
377
|
+
payload[:fields] = []
|
378
|
+
end
|
379
|
+
|
380
|
+
if options[:type] || (klass != searchkick_klass && searchkick_index)
|
381
|
+
@type = [options[:type] || klass].flatten.map { |v| searchkick_index.klass_document_type(v) }
|
382
|
+
end
|
383
|
+
|
384
|
+
# routing
|
385
|
+
@routing = options[:routing] if options[:routing]
|
386
|
+
end
|
387
|
+
|
388
|
+
@body = payload
|
389
|
+
@facet_limits = @facet_limits || {}
|
390
|
+
@page = page
|
391
|
+
@per_page = per_page
|
392
|
+
@padding = padding
|
393
|
+
@load = load
|
394
|
+
end
|
395
|
+
|
396
|
+
def set_fields
|
397
|
+
boost_fields = {}
|
398
|
+
fields = options[:fields] || searchkick_options[:searchable]
|
399
|
+
fields =
|
400
|
+
if fields
|
401
|
+
if options[:autocomplete]
|
402
|
+
fields.map { |f| "#{f}.autocomplete" }
|
403
|
+
else
|
404
|
+
fields.map do |value|
|
405
|
+
k, v = value.is_a?(Hash) ? value.to_a.first : [value, options[:match] || searchkick_options[:match] || :word]
|
406
|
+
k2, boost = k.to_s.split("^", 2)
|
407
|
+
field = "#{k2}.#{v == :word ? 'analyzed' : v}"
|
408
|
+
boost_fields[field] = boost.to_f if boost
|
409
|
+
field
|
410
|
+
end
|
411
|
+
end
|
412
|
+
else
|
413
|
+
if options[:autocomplete]
|
414
|
+
(searchkick_options[:autocomplete] || []).map { |f| "#{f}.autocomplete" }
|
415
|
+
else
|
416
|
+
["_all"]
|
417
|
+
end
|
418
|
+
end
|
419
|
+
[boost_fields, fields]
|
420
|
+
end
|
421
|
+
|
422
|
+
def set_boost_by_distance(custom_filters)
|
423
|
+
boost_by_distance = options[:boost_by_distance] || {}
|
424
|
+
boost_by_distance = {function: :gauss, scale: "5mi"}.merge(boost_by_distance)
|
425
|
+
if !boost_by_distance[:field] || !boost_by_distance[:origin]
|
426
|
+
raise ArgumentError, "boost_by_distance requires :field and :origin"
|
427
|
+
end
|
428
|
+
function_params = boost_by_distance.select { |k, _| [:origin, :scale, :offset, :decay].include?(k) }
|
429
|
+
function_params[:origin] = location_value(function_params[:origin])
|
430
|
+
custom_filters << {
|
431
|
+
boost_by_distance[:function] => {
|
432
|
+
boost_by_distance[:field] => function_params
|
433
|
+
}
|
434
|
+
}
|
435
|
+
end
|
436
|
+
|
437
|
+
def set_boost_by(multiply_filters, custom_filters)
|
438
|
+
boost_by = options[:boost_by] || {}
|
439
|
+
if boost_by.is_a?(Array)
|
440
|
+
boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
|
441
|
+
elsif boost_by.is_a?(Hash)
|
442
|
+
multiply_by, boost_by = boost_by.partition { |_, v| v[:boost_mode] == "multiply" }.map { |i| Hash[i] }
|
443
|
+
end
|
444
|
+
boost_by[options[:boost]] = {factor: 1} if options[:boost]
|
445
|
+
|
446
|
+
custom_filters.concat boost_filters(boost_by, log: true)
|
447
|
+
multiply_filters.concat boost_filters(multiply_by || {})
|
448
|
+
end
|
449
|
+
|
450
|
+
def set_boost_where(custom_filters, personalize_field)
|
451
|
+
boost_where = options[:boost_where] || {}
|
452
|
+
if options[:user_id] && personalize_field
|
453
|
+
boost_where[personalize_field] = options[:user_id]
|
454
|
+
end
|
455
|
+
if options[:personalize]
|
456
|
+
boost_where = boost_where.merge(options[:personalize])
|
457
|
+
end
|
458
|
+
boost_where.each do |field, value|
|
459
|
+
if value.is_a?(Array) && value.first.is_a?(Hash)
|
460
|
+
value.each do |value_factor|
|
461
|
+
custom_filters << custom_filter(field, value_factor[:value], value_factor[:factor])
|
462
|
+
end
|
463
|
+
elsif value.is_a?(Hash)
|
464
|
+
custom_filters << custom_filter(field, value[:value], value[:factor])
|
465
|
+
else
|
466
|
+
factor = 1000
|
467
|
+
custom_filters << custom_filter(field, value, factor)
|
468
|
+
end
|
469
|
+
end
|
470
|
+
end
|
471
|
+
|
472
|
+
def set_suggestions(payload)
|
473
|
+
suggest_fields = (searchkick_options[:suggest] || []).map(&:to_s)
|
474
|
+
|
475
|
+
# intersection
|
476
|
+
if options[:fields]
|
477
|
+
suggest_fields &= options[:fields].map { |v| (v.is_a?(Hash) ? v.keys.first : v).to_s.split("^", 2).first }
|
478
|
+
end
|
479
|
+
|
480
|
+
if suggest_fields.any?
|
481
|
+
payload[:suggest] = {text: term}
|
482
|
+
suggest_fields.each do |field|
|
483
|
+
payload[:suggest][field] = {
|
484
|
+
phrase: {
|
485
|
+
field: "#{field}.suggest"
|
486
|
+
}
|
487
|
+
}
|
488
|
+
end
|
489
|
+
end
|
490
|
+
end
|
491
|
+
|
492
|
+
def set_highlights(payload, fields)
|
493
|
+
payload[:highlight] = {
|
494
|
+
fields: Hash[fields.map { |f| [f, {}] }]
|
495
|
+
}
|
496
|
+
|
497
|
+
if options[:highlight].is_a?(Hash)
|
498
|
+
if (tag = options[:highlight][:tag])
|
499
|
+
payload[:highlight][:pre_tags] = [tag]
|
500
|
+
payload[:highlight][:post_tags] = [tag.to_s.gsub(/\A</, "</")]
|
501
|
+
end
|
502
|
+
|
503
|
+
if (fragment_size = options[:highlight][:fragment_size])
|
504
|
+
payload[:highlight][:fragment_size] = fragment_size
|
505
|
+
end
|
506
|
+
if (encoder = options[:highlight][:encoder])
|
507
|
+
payload[:highlight][:encoder] = encoder
|
508
|
+
end
|
509
|
+
|
510
|
+
highlight_fields = options[:highlight][:fields]
|
511
|
+
if highlight_fields
|
512
|
+
payload[:highlight][:fields] = {}
|
513
|
+
|
514
|
+
highlight_fields.each do |name, opts|
|
515
|
+
payload[:highlight][:fields]["#{name}.#{@match_suffix}"] = opts || {}
|
516
|
+
end
|
517
|
+
end
|
518
|
+
end
|
519
|
+
|
520
|
+
@highlighted_fields = payload[:highlight][:fields].keys
|
521
|
+
end
|
522
|
+
|
523
|
+
def set_aggregations(payload)
|
524
|
+
aggs = options[:aggs]
|
525
|
+
payload[:aggs] = {}
|
526
|
+
|
527
|
+
if aggs.is_a?(Hash) && aggs[:body]
|
528
|
+
payload[:aggs] = aggs[:body]
|
529
|
+
else
|
530
|
+
aggs = Hash[aggs.map { |f| [f, {}] }] if aggs.is_a?(Array) # convert to more advanced syntax
|
531
|
+
|
532
|
+
aggs.each do |field, agg_options|
|
533
|
+
size = agg_options[:limit] ? agg_options[:limit] : 1_000
|
534
|
+
shared_agg_options = agg_options.slice(:order, :min_doc_count)
|
535
|
+
|
536
|
+
if agg_options[:ranges]
|
537
|
+
payload[:aggs][field] = {
|
538
|
+
range: {
|
539
|
+
field: agg_options[:field] || field,
|
540
|
+
ranges: agg_options[:ranges]
|
541
|
+
}.merge(shared_agg_options)
|
542
|
+
}
|
543
|
+
elsif agg_options[:date_ranges]
|
544
|
+
payload[:aggs][field] = {
|
545
|
+
date_range: {
|
546
|
+
field: agg_options[:field] || field,
|
547
|
+
ranges: agg_options[:date_ranges]
|
548
|
+
}.merge(shared_agg_options)
|
549
|
+
}
|
550
|
+
else
|
551
|
+
payload[:aggs][field] = {
|
552
|
+
terms: {
|
553
|
+
field: agg_options[:field] || field,
|
554
|
+
size: size
|
555
|
+
}.merge(shared_agg_options)
|
556
|
+
}
|
557
|
+
end
|
558
|
+
|
559
|
+
where = {}
|
560
|
+
where = (options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
|
561
|
+
agg_filters = where_filters(where.merge(agg_options[:where] || {}))
|
562
|
+
if agg_filters.any?
|
563
|
+
payload[:aggs][field] = {
|
564
|
+
filter: {
|
565
|
+
bool: {
|
566
|
+
must: agg_filters
|
567
|
+
}
|
568
|
+
},
|
569
|
+
aggs: {
|
570
|
+
field => payload[:aggs][field]
|
571
|
+
}
|
572
|
+
}
|
573
|
+
end
|
574
|
+
end
|
575
|
+
end
|
576
|
+
end
|
577
|
+
|
578
|
+
def set_facets(payload)
|
579
|
+
facets = options[:facets] || {}
|
580
|
+
facets = Hash[facets.map { |f| [f, {}] }] if facets.is_a?(Array) # convert to more advanced syntax
|
581
|
+
facet_limits = {}
|
582
|
+
payload[:facets] = {}
|
583
|
+
|
584
|
+
facets.each do |field, facet_options|
|
585
|
+
# ask for extra facets due to
|
586
|
+
# https://github.com/elasticsearch/elasticsearch/issues/1305
|
587
|
+
size = facet_options[:limit] ? facet_options[:limit] + 150 : 1_000
|
588
|
+
|
589
|
+
if facet_options[:ranges]
|
590
|
+
payload[:facets][field] = {
|
591
|
+
range: {
|
592
|
+
field.to_sym => facet_options[:ranges]
|
593
|
+
}
|
594
|
+
}
|
595
|
+
elsif facet_options[:stats]
|
596
|
+
payload[:facets][field] = {
|
597
|
+
terms_stats: {
|
598
|
+
key_field: field,
|
599
|
+
value_script: below14? ? "doc.score" : "_score",
|
600
|
+
size: size
|
601
|
+
}
|
602
|
+
}
|
603
|
+
else
|
604
|
+
payload[:facets][field] = {
|
605
|
+
terms: {
|
606
|
+
field: facet_options[:field] || field,
|
607
|
+
size: size
|
608
|
+
}
|
609
|
+
}
|
610
|
+
end
|
611
|
+
|
612
|
+
facet_limits[field] = facet_options[:limit] if facet_options[:limit]
|
613
|
+
|
614
|
+
# offset is not possible
|
615
|
+
# http://elasticsearch-users.115913.n3.nabble.com/Is-pagination-possible-in-termsStatsFacet-td3422943.html
|
616
|
+
|
617
|
+
facet_options.deep_merge!(where: options.fetch(:where, {}).reject { |k| k == field }) if options[:smart_facets] == true
|
618
|
+
facet_filters = where_filters(facet_options[:where])
|
619
|
+
if facet_filters.any?
|
620
|
+
payload[:facets][field][:facet_filter] = {
|
621
|
+
and: {
|
622
|
+
filters: facet_filters
|
623
|
+
}
|
624
|
+
}
|
625
|
+
end
|
626
|
+
end
|
627
|
+
|
628
|
+
@facet_limits = facet_limits
|
629
|
+
end
|
630
|
+
|
631
|
+
def set_filters(payload, filters)
|
632
|
+
if options[:facets] || options[:aggs]
|
633
|
+
if below20?
|
634
|
+
payload[:filter] = {
|
635
|
+
and: filters
|
636
|
+
}
|
637
|
+
else
|
638
|
+
payload[:post_filter] = {
|
639
|
+
bool: {
|
640
|
+
filter: filters
|
641
|
+
}
|
642
|
+
}
|
643
|
+
end
|
644
|
+
else
|
645
|
+
# more efficient query if no facets
|
646
|
+
if below20?
|
647
|
+
payload[:query] = {
|
648
|
+
filtered: {
|
649
|
+
query: payload[:query],
|
650
|
+
filter: {
|
651
|
+
and: filters
|
652
|
+
}
|
653
|
+
}
|
654
|
+
}
|
655
|
+
else
|
656
|
+
payload[:query] = {
|
657
|
+
bool: {
|
658
|
+
must: payload[:query],
|
659
|
+
filter: filters
|
660
|
+
}
|
661
|
+
}
|
662
|
+
end
|
663
|
+
end
|
664
|
+
end
|
665
|
+
|
666
|
+
# TODO id transformation for arrays
|
667
|
+
def set_order(payload)
|
668
|
+
order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
|
669
|
+
id_field = below50? ? :_id : :_uid
|
670
|
+
payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
|
671
|
+
end
|
672
|
+
|
673
|
+
def where_filters(where)
|
674
|
+
filters = []
|
675
|
+
(where || {}).each do |field, value|
|
676
|
+
field = :_id if field.to_s == "id"
|
677
|
+
|
678
|
+
if field == :or
|
679
|
+
value.each do |or_clause|
|
680
|
+
if below50?
|
681
|
+
filters << {or: or_clause.map { |or_statement| {and: where_filters(or_statement)} }}
|
682
|
+
else
|
683
|
+
filters << {bool: {should: or_clause.map { |or_statement| {bool: {filter: where_filters(or_statement)}} }}}
|
684
|
+
end
|
685
|
+
end
|
686
|
+
else
|
687
|
+
# expand ranges
|
688
|
+
if value.is_a?(Range)
|
689
|
+
value = {gte: value.first, (value.exclude_end? ? :lt : :lte) => value.last}
|
690
|
+
end
|
691
|
+
|
692
|
+
value = {in: value} if value.is_a?(Array)
|
693
|
+
|
694
|
+
if value.is_a?(Hash)
|
695
|
+
value.each do |op, op_value|
|
696
|
+
case op
|
697
|
+
when :within, :bottom_right
|
698
|
+
# do nothing
|
699
|
+
when :near
|
700
|
+
filters << {
|
701
|
+
geo_distance: {
|
702
|
+
field => location_value(op_value),
|
703
|
+
distance: value[:within] || "50mi"
|
704
|
+
}
|
705
|
+
}
|
706
|
+
when :top_left
|
707
|
+
filters << {
|
708
|
+
geo_bounding_box: {
|
709
|
+
field => {
|
710
|
+
top_left: location_value(op_value),
|
711
|
+
bottom_right: location_value(value[:bottom_right])
|
712
|
+
}
|
713
|
+
}
|
714
|
+
}
|
715
|
+
when :regexp # support for regexp queries without using a regexp ruby object
|
716
|
+
filters << {regexp: {field => {value: op_value}}}
|
717
|
+
when :not # not equal
|
718
|
+
if below50?
|
719
|
+
filters << {not: {filter: term_filters(field, op_value)}}
|
720
|
+
else
|
721
|
+
filters << {bool: {must_not: term_filters(field, op_value)}}
|
722
|
+
end
|
723
|
+
when :all
|
724
|
+
op_value.each do |value|
|
725
|
+
filters << term_filters(field, value)
|
726
|
+
end
|
727
|
+
when :in
|
728
|
+
filters << term_filters(field, op_value)
|
729
|
+
else
|
730
|
+
range_query =
|
731
|
+
case op
|
732
|
+
when :gt
|
733
|
+
{from: op_value, include_lower: false}
|
734
|
+
when :gte
|
735
|
+
{from: op_value, include_lower: true}
|
736
|
+
when :lt
|
737
|
+
{to: op_value, include_upper: false}
|
738
|
+
when :lte
|
739
|
+
{to: op_value, include_upper: true}
|
740
|
+
else
|
741
|
+
raise "Unknown where operator: #{op.inspect}"
|
742
|
+
end
|
743
|
+
# issue 132
|
744
|
+
if (existing = filters.find { |f| f[:range] && f[:range][field] })
|
745
|
+
existing[:range][field].merge!(range_query)
|
746
|
+
else
|
747
|
+
filters << {range: {field => range_query}}
|
748
|
+
end
|
749
|
+
end
|
750
|
+
end
|
751
|
+
else
|
752
|
+
filters << term_filters(field, value)
|
753
|
+
end
|
754
|
+
end
|
755
|
+
end
|
756
|
+
filters
|
757
|
+
end
|
758
|
+
|
759
|
+
def term_filters(field, value)
|
760
|
+
if value.is_a?(Array) # in query
|
761
|
+
if value.any?(&:nil?)
|
762
|
+
if below50?
|
763
|
+
{or: [term_filters(field, nil), term_filters(field, value.compact)]}
|
764
|
+
else
|
765
|
+
{bool: {should: [term_filters(field, nil), term_filters(field, value.compact)]}}
|
766
|
+
end
|
767
|
+
else
|
768
|
+
{in: {field => value}}
|
769
|
+
end
|
770
|
+
elsif value.nil?
|
771
|
+
if below50?
|
772
|
+
{missing: {field: field, existence: true, null_value: true}}
|
773
|
+
else
|
774
|
+
{bool: {must_not: {exists: {field: field}}}}
|
775
|
+
end
|
776
|
+
elsif value.is_a?(Regexp)
|
777
|
+
{regexp: {field => {value: value.source}}}
|
778
|
+
else
|
779
|
+
{term: {field => value}}
|
780
|
+
end
|
781
|
+
end
|
782
|
+
|
783
|
+
def custom_filter(field, value, factor)
|
784
|
+
if below50?
|
785
|
+
{
|
786
|
+
filter: {
|
787
|
+
and: where_filters(field => value)
|
788
|
+
},
|
789
|
+
boost_factor: factor
|
790
|
+
}
|
791
|
+
else
|
792
|
+
{
|
793
|
+
filter: where_filters(field => value),
|
794
|
+
weight: factor
|
795
|
+
}
|
796
|
+
end
|
797
|
+
end
|
798
|
+
|
799
|
+
def boost_filters(boost_by, options = {})
|
800
|
+
boost_by.map do |field, value|
|
801
|
+
log = value.key?(:log) ? value[:log] : options[:log]
|
802
|
+
value[:factor] ||= 1
|
803
|
+
script_score =
|
804
|
+
if below12?
|
805
|
+
script = log ? "log(doc['#{field}'].value + 2.718281828)" : "doc['#{field}'].value"
|
806
|
+
{script_score: {script: "#{value[:factor].to_f} * #{script}"}}
|
807
|
+
else
|
808
|
+
{field_value_factor: {field: field, factor: value[:factor].to_f, modifier: log ? "ln2p" : nil}}
|
809
|
+
end
|
810
|
+
|
811
|
+
{
|
812
|
+
filter: {
|
813
|
+
exists: {
|
814
|
+
field: field
|
815
|
+
}
|
816
|
+
}
|
817
|
+
}.merge(script_score)
|
818
|
+
end
|
819
|
+
end
|
820
|
+
|
821
|
+
def location_value(value)
|
822
|
+
if value.is_a?(Array)
|
823
|
+
value.map(&:to_f).reverse
|
824
|
+
else
|
825
|
+
value
|
826
|
+
end
|
827
|
+
end
|
828
|
+
|
829
|
+
def below12?
|
830
|
+
Searchkick.server_below?("1.2.0")
|
831
|
+
end
|
832
|
+
|
833
|
+
def below14?
|
834
|
+
Searchkick.server_below?("1.4.0")
|
835
|
+
end
|
836
|
+
|
837
|
+
def below20?
|
838
|
+
Searchkick.server_below?("2.0.0")
|
839
|
+
end
|
840
|
+
|
841
|
+
def below50?
|
842
|
+
Searchkick.server_below?("5.0.0-alpha1")
|
843
|
+
end
|
844
|
+
end
|
845
|
+
end
|