searchkick_bharthur 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +22 -0
  3. data/.travis.yml +44 -0
  4. data/CHANGELOG.md +360 -0
  5. data/Gemfile +8 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +1443 -0
  8. data/Rakefile +8 -0
  9. data/lib/searchkick/index.rb +662 -0
  10. data/lib/searchkick/logging.rb +185 -0
  11. data/lib/searchkick/middleware.rb +12 -0
  12. data/lib/searchkick/model.rb +105 -0
  13. data/lib/searchkick/query.rb +845 -0
  14. data/lib/searchkick/reindex_job.rb +26 -0
  15. data/lib/searchkick/reindex_v2_job.rb +23 -0
  16. data/lib/searchkick/results.rb +211 -0
  17. data/lib/searchkick/tasks.rb +33 -0
  18. data/lib/searchkick/version.rb +3 -0
  19. data/lib/searchkick.rb +159 -0
  20. data/searchkick.gemspec +28 -0
  21. data/test/aggs_test.rb +115 -0
  22. data/test/autocomplete_test.rb +65 -0
  23. data/test/boost_test.rb +144 -0
  24. data/test/callbacks_test.rb +27 -0
  25. data/test/ci/before_install.sh +21 -0
  26. data/test/dangerous_reindex_test.rb +27 -0
  27. data/test/facets_test.rb +90 -0
  28. data/test/gemfiles/activerecord31.gemfile +7 -0
  29. data/test/gemfiles/activerecord32.gemfile +7 -0
  30. data/test/gemfiles/activerecord40.gemfile +8 -0
  31. data/test/gemfiles/activerecord41.gemfile +8 -0
  32. data/test/gemfiles/activerecord50.gemfile +7 -0
  33. data/test/gemfiles/apartment.gemfile +8 -0
  34. data/test/gemfiles/mongoid2.gemfile +7 -0
  35. data/test/gemfiles/mongoid3.gemfile +6 -0
  36. data/test/gemfiles/mongoid4.gemfile +7 -0
  37. data/test/gemfiles/mongoid5.gemfile +7 -0
  38. data/test/gemfiles/nobrainer.gemfile +6 -0
  39. data/test/highlight_test.rb +63 -0
  40. data/test/index_test.rb +120 -0
  41. data/test/inheritance_test.rb +78 -0
  42. data/test/match_test.rb +227 -0
  43. data/test/misspellings_test.rb +46 -0
  44. data/test/model_test.rb +42 -0
  45. data/test/multi_search_test.rb +22 -0
  46. data/test/multi_tenancy_test.rb +22 -0
  47. data/test/order_test.rb +44 -0
  48. data/test/pagination_test.rb +53 -0
  49. data/test/query_test.rb +13 -0
  50. data/test/records_test.rb +8 -0
  51. data/test/reindex_job_test.rb +31 -0
  52. data/test/reindex_v2_job_test.rb +32 -0
  53. data/test/routing_test.rb +13 -0
  54. data/test/should_index_test.rb +32 -0
  55. data/test/similar_test.rb +28 -0
  56. data/test/sql_test.rb +196 -0
  57. data/test/suggest_test.rb +80 -0
  58. data/test/synonyms_test.rb +54 -0
  59. data/test/test_helper.rb +361 -0
  60. data/test/where_test.rb +171 -0
  61. metadata +231 -0
@@ -0,0 +1,845 @@
1
+ module Searchkick
2
+ class Query
3
+ extend Forwardable
4
+
5
+ attr_reader :klass, :term, :options
6
+ attr_accessor :body
7
+
8
+ def_delegators :execute, :map, :each, :any?, :empty?, :size, :length, :slice, :[], :to_ary,
9
+ :records, :results, :suggestions, :each_with_hit, :with_details, :facets, :aggregations, :aggs,
10
+ :took, :error, :model_name, :entry_name, :total_count, :total_entries,
11
+ :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
12
+ :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
13
+ :out_of_range?, :hits
14
+
15
+
16
+ def initialize(klass, term, options = {})
17
+ if term.is_a?(Hash)
18
+ options = term
19
+ term = "*"
20
+ else
21
+ term = term.to_s
22
+ end
23
+
24
+ if options[:emoji]
25
+ term = EmojiParser.parse_unicode(term) { |e| " #{e.name} " }.strip
26
+ end
27
+
28
+ @klass = klass
29
+ @term = term
30
+ @options = options
31
+ @match_suffix = options[:match] || searchkick_options[:match] || "analyzed"
32
+
33
+ prepare
34
+ end
35
+
36
+ def searchkick_index
37
+ klass ? klass.searchkick_index : nil
38
+ end
39
+
40
+ def searchkick_options
41
+ klass ? klass.searchkick_options : {}
42
+ end
43
+
44
+ def searchkick_klass
45
+ klass ? klass.searchkick_klass : nil
46
+ end
47
+
48
+ def params
49
+ index =
50
+ if options[:index_name]
51
+ Array(options[:index_name]).map { |v| v.respond_to?(:searchkick_index) ? v.searchkick_index.name : v }.join(",")
52
+ elsif searchkick_index
53
+ searchkick_index.name
54
+ else
55
+ "_all"
56
+ end
57
+
58
+ params = {
59
+ index: index,
60
+ body: body
61
+ }
62
+ params.merge!(type: @type) if @type
63
+ params.merge!(routing: @routing) if @routing
64
+ params
65
+ end
66
+
67
+ def execute
68
+ @execute ||= begin
69
+ begin
70
+ response = execute_search
71
+ if @misspellings_below && response["hits"]["total"] < @misspellings_below
72
+ prepare
73
+ response = execute_search
74
+ end
75
+ rescue => e # TODO rescue type
76
+ handle_error(e)
77
+ end
78
+ handle_response(response)
79
+ end
80
+ end
81
+
82
+ def to_curl
83
+ query = params
84
+ type = query[:type]
85
+ index = query[:index].is_a?(Array) ? query[:index].join(",") : query[:index]
86
+
87
+ # no easy way to tell which host the client will use
88
+ host = Searchkick.client.transport.hosts.first
89
+ credentials = (host[:user] || host[:password]) ? "#{host[:user]}:#{host[:password]}@" : nil
90
+ "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{query[:body].to_json}'"
91
+ end
92
+
93
+ def handle_response(response)
94
+ # apply facet limit in client due to
95
+ # https://github.com/elasticsearch/elasticsearch/issues/1305
96
+ @facet_limits.each do |field, limit|
97
+ field = field.to_s
98
+ facet = response["facets"][field]
99
+ response["facets"][field]["terms"] = facet["terms"].first(limit)
100
+ response["facets"][field]["other"] = facet["total"] - facet["terms"].sum { |term| term["count"] }
101
+ end
102
+
103
+ opts = {
104
+ page: @page,
105
+ per_page: @per_page,
106
+ padding: @padding,
107
+ load: @load,
108
+ includes: options[:include] || options[:includes],
109
+ json: !options[:json].nil?,
110
+ match_suffix: @match_suffix,
111
+ highlighted_fields: @highlighted_fields || []
112
+ }
113
+
114
+ # set execute for multi search
115
+ @execute = Searchkick::Results.new(searchkick_klass, response, opts)
116
+ end
117
+
118
+ private
119
+
120
+ def handle_error(e)
121
+ status_code = e.message[1..3].to_i
122
+ if status_code == 404
123
+ raise MissingIndexError, "Index missing - run #{reindex_command}"
124
+ elsif status_code == 500 && (
125
+ e.message.include?("IllegalArgumentException[minimumSimilarity >= 1]") ||
126
+ e.message.include?("No query registered for [multi_match]") ||
127
+ e.message.include?("[match] query does not support [cutoff_frequency]]") ||
128
+ e.message.include?("No query registered for [function_score]]")
129
+ )
130
+
131
+ raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 1.0 or greater"
132
+ elsif status_code == 400
133
+ if e.message.include?("[multi_match] analyzer [searchkick_search] not found")
134
+ raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
135
+ else
136
+ raise InvalidQueryError, e.message
137
+ end
138
+ else
139
+ raise e
140
+ end
141
+ end
142
+
143
+ def reindex_command
144
+ searchkick_klass ? "#{searchkick_klass.name}.reindex" : "reindex"
145
+ end
146
+
147
+ def execute_search
148
+ Searchkick.client.search(params)
149
+ end
150
+
151
+ def prepare
152
+ boost_fields, fields = set_fields
153
+
154
+ operator = options[:operator] || (options[:partial] ? "or" : "and")
155
+
156
+ # pagination
157
+ page = [options[:page].to_i, 1].max
158
+ per_page = (options[:limit] || options[:per_page] || 1_000).to_i
159
+ padding = [options[:padding].to_i, 0].max
160
+ offset = options[:offset] || (page - 1) * per_page + padding
161
+
162
+ # model and eagar loading
163
+ load = options[:load].nil? ? true : options[:load]
164
+
165
+ conversions_field = searchkick_options[:conversions]
166
+ personalize_field = searchkick_options[:personalize]
167
+
168
+ all = term == "*"
169
+
170
+ options[:json] ||= options[:body]
171
+ if options[:json]
172
+ payload = options[:json]
173
+ else
174
+ if options[:query]
175
+ payload = options[:query]
176
+ elsif options[:similar]
177
+ payload = {
178
+ more_like_this: {
179
+ fields: fields,
180
+ like_text: term,
181
+ min_doc_freq: 1,
182
+ min_term_freq: 1,
183
+ analyzer: "searchkick_search2"
184
+ }
185
+ }
186
+ elsif all
187
+ payload = {
188
+ match_all: {}
189
+ }
190
+ else
191
+ if options[:autocomplete]
192
+ payload = {
193
+ multi_match: {
194
+ fields: fields,
195
+ query: term,
196
+ analyzer: "searchkick_autocomplete_search"
197
+ }
198
+ }
199
+ else
200
+ queries = []
201
+
202
+ misspellings =
203
+ if options.key?(:misspellings)
204
+ options[:misspellings]
205
+ elsif options.key?(:mispellings)
206
+ options[:mispellings] # why not?
207
+ else
208
+ true
209
+ end
210
+
211
+ if misspellings.is_a?(Hash) && misspellings[:below] && !@misspellings_below
212
+ @misspellings_below = misspellings[:below].to_i
213
+ misspellings = false
214
+ end
215
+
216
+ if misspellings != false
217
+ edit_distance = (misspellings.is_a?(Hash) && (misspellings[:edit_distance] || misspellings[:distance])) || 1
218
+ transpositions =
219
+ if misspellings.is_a?(Hash) && misspellings.key?(:transpositions)
220
+ {fuzzy_transpositions: misspellings[:transpositions]}
221
+ elsif below14?
222
+ {}
223
+ else
224
+ {fuzzy_transpositions: true}
225
+ end
226
+ prefix_length = (misspellings.is_a?(Hash) && misspellings[:prefix_length]) || 0
227
+ default_max_expansions = @misspellings_below ? 20 : 3
228
+ max_expansions = (misspellings.is_a?(Hash) && misspellings[:max_expansions]) || default_max_expansions
229
+ end
230
+
231
+ fields.each do |field|
232
+ qs = []
233
+
234
+ factor = boost_fields[field] || 1
235
+ shared_options = {
236
+ query: term,
237
+ boost: 10 * factor
238
+ }
239
+
240
+ match_type =
241
+ if field.end_with?(".phrase")
242
+ field = field.sub(/\.phrase\z/, ".analyzed")
243
+ :match_phrase
244
+ else
245
+ :match
246
+ end
247
+
248
+ shared_options[:operator] = operator if match_type == :match || below50?
249
+
250
+ if field == "_all" || field.end_with?(".analyzed")
251
+ shared_options[:cutoff_frequency] = 0.001 unless operator == "and" || misspellings == false
252
+ qs.concat [
253
+ shared_options.merge(analyzer: "searchkick_search"),
254
+ shared_options.merge(analyzer: "searchkick_search2")
255
+ ]
256
+ elsif field.end_with?(".exact")
257
+ f = field.split(".")[0..-2].join(".")
258
+ queries << {match: {f => shared_options.merge(analyzer: "keyword")}}
259
+ else
260
+ analyzer = field.match(/\.word_(start|middle|end)\z/) ? "searchkick_word_search" : "searchkick_autocomplete_search"
261
+ qs << shared_options.merge(analyzer: analyzer)
262
+ end
263
+
264
+ if misspellings != false && (match_type == :match || below50?)
265
+ qs.concat qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) }
266
+ end
267
+
268
+ queries.concat(qs.map { |q| {match_type => {field => q}} })
269
+ end
270
+
271
+ payload = {
272
+ dis_max: {
273
+ queries: queries
274
+ }
275
+ }
276
+ end
277
+
278
+ if conversions_field && options[:conversions] != false
279
+ # wrap payload in a bool query
280
+ script_score =
281
+ if below12?
282
+ {script_score: {script: "doc['count'].value"}}
283
+ else
284
+ {field_value_factor: {field: "#{conversions_field}.count"}}
285
+ end
286
+
287
+ payload = {
288
+ bool: {
289
+ must: payload,
290
+ should: {
291
+ nested: {
292
+ path: conversions_field,
293
+ score_mode: "sum",
294
+ query: {
295
+ function_score: {
296
+ boost_mode: "replace",
297
+ query: {
298
+ match: {
299
+ "#{conversions_field}.query" => term
300
+ }
301
+ }
302
+ }.merge(script_score)
303
+ }
304
+ }
305
+ }
306
+ }
307
+ }
308
+ end
309
+ end
310
+
311
+ custom_filters = []
312
+ multiply_filters = []
313
+
314
+ set_boost_by(multiply_filters, custom_filters)
315
+ set_boost_where(custom_filters, personalize_field)
316
+ set_boost_by_distance(custom_filters) if options[:boost_by_distance]
317
+
318
+ if custom_filters.any?
319
+ payload = {
320
+ function_score: {
321
+ functions: custom_filters,
322
+ query: payload,
323
+ score_mode: "sum"
324
+ }
325
+ }
326
+ end
327
+
328
+ if multiply_filters.any?
329
+ payload = {
330
+ function_score: {
331
+ functions: multiply_filters,
332
+ query: payload,
333
+ score_mode: "multiply"
334
+ }
335
+ }
336
+ end
337
+
338
+ payload = {
339
+ query: payload,
340
+ size: per_page,
341
+ from: offset
342
+ }
343
+ payload[:explain] = options[:explain] if options[:explain]
344
+
345
+ # order
346
+ set_order(payload) if options[:order]
347
+
348
+ # filters
349
+ filters = where_filters(options[:where])
350
+ set_filters(payload, filters) if filters.any?
351
+
352
+ # facets
353
+ set_facets(payload) if options[:facets]
354
+
355
+ # aggregations
356
+ set_aggregations(payload) if options[:aggs]
357
+
358
+ # suggestions
359
+ set_suggestions(payload) if options[:suggest]
360
+
361
+ # highlight
362
+ set_highlights(payload, fields) if options[:highlight]
363
+
364
+ # An empty array will cause only the _id and _type for each hit to be returned
365
+ # doc for :select - http://www.elasticsearch.org/guide/reference/api/search/fields/
366
+ # doc for :select_v2 - https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html
367
+ if options[:select]
368
+ payload[:fields] = options[:select] if options[:select] != true
369
+ elsif options[:select_v2]
370
+ if options[:select_v2] == []
371
+ payload[:fields] = [] # intuitively [] makes sense to return no fields, but ES by default returns all fields
372
+ else
373
+ payload[:_source] = options[:select_v2]
374
+ end
375
+ elsif load
376
+ # don't need any fields since we're going to load them from the DB anyways
377
+ payload[:fields] = []
378
+ end
379
+
380
+ if options[:type] || (klass != searchkick_klass && searchkick_index)
381
+ @type = [options[:type] || klass].flatten.map { |v| searchkick_index.klass_document_type(v) }
382
+ end
383
+
384
+ # routing
385
+ @routing = options[:routing] if options[:routing]
386
+ end
387
+
388
+ @body = payload
389
+ @facet_limits = @facet_limits || {}
390
+ @page = page
391
+ @per_page = per_page
392
+ @padding = padding
393
+ @load = load
394
+ end
395
+
396
+ def set_fields
397
+ boost_fields = {}
398
+ fields = options[:fields] || searchkick_options[:searchable]
399
+ fields =
400
+ if fields
401
+ if options[:autocomplete]
402
+ fields.map { |f| "#{f}.autocomplete" }
403
+ else
404
+ fields.map do |value|
405
+ k, v = value.is_a?(Hash) ? value.to_a.first : [value, options[:match] || searchkick_options[:match] || :word]
406
+ k2, boost = k.to_s.split("^", 2)
407
+ field = "#{k2}.#{v == :word ? 'analyzed' : v}"
408
+ boost_fields[field] = boost.to_f if boost
409
+ field
410
+ end
411
+ end
412
+ else
413
+ if options[:autocomplete]
414
+ (searchkick_options[:autocomplete] || []).map { |f| "#{f}.autocomplete" }
415
+ else
416
+ ["_all"]
417
+ end
418
+ end
419
+ [boost_fields, fields]
420
+ end
421
+
422
+ def set_boost_by_distance(custom_filters)
423
+ boost_by_distance = options[:boost_by_distance] || {}
424
+ boost_by_distance = {function: :gauss, scale: "5mi"}.merge(boost_by_distance)
425
+ if !boost_by_distance[:field] || !boost_by_distance[:origin]
426
+ raise ArgumentError, "boost_by_distance requires :field and :origin"
427
+ end
428
+ function_params = boost_by_distance.select { |k, _| [:origin, :scale, :offset, :decay].include?(k) }
429
+ function_params[:origin] = location_value(function_params[:origin])
430
+ custom_filters << {
431
+ boost_by_distance[:function] => {
432
+ boost_by_distance[:field] => function_params
433
+ }
434
+ }
435
+ end
436
+
437
+ def set_boost_by(multiply_filters, custom_filters)
438
+ boost_by = options[:boost_by] || {}
439
+ if boost_by.is_a?(Array)
440
+ boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
441
+ elsif boost_by.is_a?(Hash)
442
+ multiply_by, boost_by = boost_by.partition { |_, v| v[:boost_mode] == "multiply" }.map { |i| Hash[i] }
443
+ end
444
+ boost_by[options[:boost]] = {factor: 1} if options[:boost]
445
+
446
+ custom_filters.concat boost_filters(boost_by, log: true)
447
+ multiply_filters.concat boost_filters(multiply_by || {})
448
+ end
449
+
450
+ def set_boost_where(custom_filters, personalize_field)
451
+ boost_where = options[:boost_where] || {}
452
+ if options[:user_id] && personalize_field
453
+ boost_where[personalize_field] = options[:user_id]
454
+ end
455
+ if options[:personalize]
456
+ boost_where = boost_where.merge(options[:personalize])
457
+ end
458
+ boost_where.each do |field, value|
459
+ if value.is_a?(Array) && value.first.is_a?(Hash)
460
+ value.each do |value_factor|
461
+ custom_filters << custom_filter(field, value_factor[:value], value_factor[:factor])
462
+ end
463
+ elsif value.is_a?(Hash)
464
+ custom_filters << custom_filter(field, value[:value], value[:factor])
465
+ else
466
+ factor = 1000
467
+ custom_filters << custom_filter(field, value, factor)
468
+ end
469
+ end
470
+ end
471
+
472
+ def set_suggestions(payload)
473
+ suggest_fields = (searchkick_options[:suggest] || []).map(&:to_s)
474
+
475
+ # intersection
476
+ if options[:fields]
477
+ suggest_fields &= options[:fields].map { |v| (v.is_a?(Hash) ? v.keys.first : v).to_s.split("^", 2).first }
478
+ end
479
+
480
+ if suggest_fields.any?
481
+ payload[:suggest] = {text: term}
482
+ suggest_fields.each do |field|
483
+ payload[:suggest][field] = {
484
+ phrase: {
485
+ field: "#{field}.suggest"
486
+ }
487
+ }
488
+ end
489
+ end
490
+ end
491
+
492
+ def set_highlights(payload, fields)
493
+ payload[:highlight] = {
494
+ fields: Hash[fields.map { |f| [f, {}] }]
495
+ }
496
+
497
+ if options[:highlight].is_a?(Hash)
498
+ if (tag = options[:highlight][:tag])
499
+ payload[:highlight][:pre_tags] = [tag]
500
+ payload[:highlight][:post_tags] = [tag.to_s.gsub(/\A</, "</")]
501
+ end
502
+
503
+ if (fragment_size = options[:highlight][:fragment_size])
504
+ payload[:highlight][:fragment_size] = fragment_size
505
+ end
506
+ if (encoder = options[:highlight][:encoder])
507
+ payload[:highlight][:encoder] = encoder
508
+ end
509
+
510
+ highlight_fields = options[:highlight][:fields]
511
+ if highlight_fields
512
+ payload[:highlight][:fields] = {}
513
+
514
+ highlight_fields.each do |name, opts|
515
+ payload[:highlight][:fields]["#{name}.#{@match_suffix}"] = opts || {}
516
+ end
517
+ end
518
+ end
519
+
520
+ @highlighted_fields = payload[:highlight][:fields].keys
521
+ end
522
+
523
+ def set_aggregations(payload)
524
+ aggs = options[:aggs]
525
+ payload[:aggs] = {}
526
+
527
+ if aggs.is_a?(Hash) && aggs[:body]
528
+ payload[:aggs] = aggs[:body]
529
+ else
530
+ aggs = Hash[aggs.map { |f| [f, {}] }] if aggs.is_a?(Array) # convert to more advanced syntax
531
+
532
+ aggs.each do |field, agg_options|
533
+ size = agg_options[:limit] ? agg_options[:limit] : 1_000
534
+ shared_agg_options = agg_options.slice(:order, :min_doc_count)
535
+
536
+ if agg_options[:ranges]
537
+ payload[:aggs][field] = {
538
+ range: {
539
+ field: agg_options[:field] || field,
540
+ ranges: agg_options[:ranges]
541
+ }.merge(shared_agg_options)
542
+ }
543
+ elsif agg_options[:date_ranges]
544
+ payload[:aggs][field] = {
545
+ date_range: {
546
+ field: agg_options[:field] || field,
547
+ ranges: agg_options[:date_ranges]
548
+ }.merge(shared_agg_options)
549
+ }
550
+ else
551
+ payload[:aggs][field] = {
552
+ terms: {
553
+ field: agg_options[:field] || field,
554
+ size: size
555
+ }.merge(shared_agg_options)
556
+ }
557
+ end
558
+
559
+ where = {}
560
+ where = (options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
561
+ agg_filters = where_filters(where.merge(agg_options[:where] || {}))
562
+ if agg_filters.any?
563
+ payload[:aggs][field] = {
564
+ filter: {
565
+ bool: {
566
+ must: agg_filters
567
+ }
568
+ },
569
+ aggs: {
570
+ field => payload[:aggs][field]
571
+ }
572
+ }
573
+ end
574
+ end
575
+ end
576
+ end
577
+
578
+ def set_facets(payload)
579
+ facets = options[:facets] || {}
580
+ facets = Hash[facets.map { |f| [f, {}] }] if facets.is_a?(Array) # convert to more advanced syntax
581
+ facet_limits = {}
582
+ payload[:facets] = {}
583
+
584
+ facets.each do |field, facet_options|
585
+ # ask for extra facets due to
586
+ # https://github.com/elasticsearch/elasticsearch/issues/1305
587
+ size = facet_options[:limit] ? facet_options[:limit] + 150 : 1_000
588
+
589
+ if facet_options[:ranges]
590
+ payload[:facets][field] = {
591
+ range: {
592
+ field.to_sym => facet_options[:ranges]
593
+ }
594
+ }
595
+ elsif facet_options[:stats]
596
+ payload[:facets][field] = {
597
+ terms_stats: {
598
+ key_field: field,
599
+ value_script: below14? ? "doc.score" : "_score",
600
+ size: size
601
+ }
602
+ }
603
+ else
604
+ payload[:facets][field] = {
605
+ terms: {
606
+ field: facet_options[:field] || field,
607
+ size: size
608
+ }
609
+ }
610
+ end
611
+
612
+ facet_limits[field] = facet_options[:limit] if facet_options[:limit]
613
+
614
+ # offset is not possible
615
+ # http://elasticsearch-users.115913.n3.nabble.com/Is-pagination-possible-in-termsStatsFacet-td3422943.html
616
+
617
+ facet_options.deep_merge!(where: options.fetch(:where, {}).reject { |k| k == field }) if options[:smart_facets] == true
618
+ facet_filters = where_filters(facet_options[:where])
619
+ if facet_filters.any?
620
+ payload[:facets][field][:facet_filter] = {
621
+ and: {
622
+ filters: facet_filters
623
+ }
624
+ }
625
+ end
626
+ end
627
+
628
+ @facet_limits = facet_limits
629
+ end
630
+
631
+ def set_filters(payload, filters)
632
+ if options[:facets] || options[:aggs]
633
+ if below20?
634
+ payload[:filter] = {
635
+ and: filters
636
+ }
637
+ else
638
+ payload[:post_filter] = {
639
+ bool: {
640
+ filter: filters
641
+ }
642
+ }
643
+ end
644
+ else
645
+ # more efficient query if no facets
646
+ if below20?
647
+ payload[:query] = {
648
+ filtered: {
649
+ query: payload[:query],
650
+ filter: {
651
+ and: filters
652
+ }
653
+ }
654
+ }
655
+ else
656
+ payload[:query] = {
657
+ bool: {
658
+ must: payload[:query],
659
+ filter: filters
660
+ }
661
+ }
662
+ end
663
+ end
664
+ end
665
+
666
+ # TODO id transformation for arrays
667
+ def set_order(payload)
668
+ order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
669
+ id_field = below50? ? :_id : :_uid
670
+ payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
671
+ end
672
+
673
+ def where_filters(where)
674
+ filters = []
675
+ (where || {}).each do |field, value|
676
+ field = :_id if field.to_s == "id"
677
+
678
+ if field == :or
679
+ value.each do |or_clause|
680
+ if below50?
681
+ filters << {or: or_clause.map { |or_statement| {and: where_filters(or_statement)} }}
682
+ else
683
+ filters << {bool: {should: or_clause.map { |or_statement| {bool: {filter: where_filters(or_statement)}} }}}
684
+ end
685
+ end
686
+ else
687
+ # expand ranges
688
+ if value.is_a?(Range)
689
+ value = {gte: value.first, (value.exclude_end? ? :lt : :lte) => value.last}
690
+ end
691
+
692
+ value = {in: value} if value.is_a?(Array)
693
+
694
+ if value.is_a?(Hash)
695
+ value.each do |op, op_value|
696
+ case op
697
+ when :within, :bottom_right
698
+ # do nothing
699
+ when :near
700
+ filters << {
701
+ geo_distance: {
702
+ field => location_value(op_value),
703
+ distance: value[:within] || "50mi"
704
+ }
705
+ }
706
+ when :top_left
707
+ filters << {
708
+ geo_bounding_box: {
709
+ field => {
710
+ top_left: location_value(op_value),
711
+ bottom_right: location_value(value[:bottom_right])
712
+ }
713
+ }
714
+ }
715
+ when :regexp # support for regexp queries without using a regexp ruby object
716
+ filters << {regexp: {field => {value: op_value}}}
717
+ when :not # not equal
718
+ if below50?
719
+ filters << {not: {filter: term_filters(field, op_value)}}
720
+ else
721
+ filters << {bool: {must_not: term_filters(field, op_value)}}
722
+ end
723
+ when :all
724
+ op_value.each do |value|
725
+ filters << term_filters(field, value)
726
+ end
727
+ when :in
728
+ filters << term_filters(field, op_value)
729
+ else
730
+ range_query =
731
+ case op
732
+ when :gt
733
+ {from: op_value, include_lower: false}
734
+ when :gte
735
+ {from: op_value, include_lower: true}
736
+ when :lt
737
+ {to: op_value, include_upper: false}
738
+ when :lte
739
+ {to: op_value, include_upper: true}
740
+ else
741
+ raise "Unknown where operator: #{op.inspect}"
742
+ end
743
+ # issue 132
744
+ if (existing = filters.find { |f| f[:range] && f[:range][field] })
745
+ existing[:range][field].merge!(range_query)
746
+ else
747
+ filters << {range: {field => range_query}}
748
+ end
749
+ end
750
+ end
751
+ else
752
+ filters << term_filters(field, value)
753
+ end
754
+ end
755
+ end
756
+ filters
757
+ end
758
+
759
+ def term_filters(field, value)
760
+ if value.is_a?(Array) # in query
761
+ if value.any?(&:nil?)
762
+ if below50?
763
+ {or: [term_filters(field, nil), term_filters(field, value.compact)]}
764
+ else
765
+ {bool: {should: [term_filters(field, nil), term_filters(field, value.compact)]}}
766
+ end
767
+ else
768
+ {in: {field => value}}
769
+ end
770
+ elsif value.nil?
771
+ if below50?
772
+ {missing: {field: field, existence: true, null_value: true}}
773
+ else
774
+ {bool: {must_not: {exists: {field: field}}}}
775
+ end
776
+ elsif value.is_a?(Regexp)
777
+ {regexp: {field => {value: value.source}}}
778
+ else
779
+ {term: {field => value}}
780
+ end
781
+ end
782
+
783
+ def custom_filter(field, value, factor)
784
+ if below50?
785
+ {
786
+ filter: {
787
+ and: where_filters(field => value)
788
+ },
789
+ boost_factor: factor
790
+ }
791
+ else
792
+ {
793
+ filter: where_filters(field => value),
794
+ weight: factor
795
+ }
796
+ end
797
+ end
798
+
799
+ def boost_filters(boost_by, options = {})
800
+ boost_by.map do |field, value|
801
+ log = value.key?(:log) ? value[:log] : options[:log]
802
+ value[:factor] ||= 1
803
+ script_score =
804
+ if below12?
805
+ script = log ? "log(doc['#{field}'].value + 2.718281828)" : "doc['#{field}'].value"
806
+ {script_score: {script: "#{value[:factor].to_f} * #{script}"}}
807
+ else
808
+ {field_value_factor: {field: field, factor: value[:factor].to_f, modifier: log ? "ln2p" : nil}}
809
+ end
810
+
811
+ {
812
+ filter: {
813
+ exists: {
814
+ field: field
815
+ }
816
+ }
817
+ }.merge(script_score)
818
+ end
819
+ end
820
+
821
+ def location_value(value)
822
+ if value.is_a?(Array)
823
+ value.map(&:to_f).reverse
824
+ else
825
+ value
826
+ end
827
+ end
828
+
829
+ def below12?
830
+ Searchkick.server_below?("1.2.0")
831
+ end
832
+
833
+ def below14?
834
+ Searchkick.server_below?("1.4.0")
835
+ end
836
+
837
+ def below20?
838
+ Searchkick.server_below?("2.0.0")
839
+ end
840
+
841
+ def below50?
842
+ Searchkick.server_below?("5.0.0-alpha1")
843
+ end
844
+ end
845
+ end