searchkick 2.5.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE.md +7 -0
  3. data/.travis.yml +2 -11
  4. data/CHANGELOG.md +22 -0
  5. data/CONTRIBUTING.md +1 -1
  6. data/Gemfile +3 -3
  7. data/LICENSE.txt +1 -1
  8. data/README.md +68 -141
  9. data/Rakefile +0 -4
  10. data/benchmark/Gemfile +3 -2
  11. data/benchmark/{benchmark.rb → index.rb} +33 -31
  12. data/benchmark/search.rb +48 -0
  13. data/docs/Searchkick-3-Upgrade.md +57 -0
  14. data/lib/searchkick.rb +50 -27
  15. data/lib/searchkick/bulk_indexer.rb +168 -0
  16. data/lib/searchkick/bulk_reindex_job.rb +1 -1
  17. data/lib/searchkick/index.rb +122 -348
  18. data/lib/searchkick/index_options.rb +29 -26
  19. data/lib/searchkick/logging.rb +8 -7
  20. data/lib/searchkick/model.rb +37 -90
  21. data/lib/searchkick/multi_search.rb +6 -7
  22. data/lib/searchkick/query.rb +169 -166
  23. data/lib/searchkick/record_data.rb +133 -0
  24. data/lib/searchkick/record_indexer.rb +55 -0
  25. data/lib/searchkick/reindex_queue.rb +1 -1
  26. data/lib/searchkick/reindex_v2_job.rb +10 -13
  27. data/lib/searchkick/results.rb +14 -25
  28. data/lib/searchkick/tasks.rb +0 -4
  29. data/lib/searchkick/version.rb +1 -1
  30. data/searchkick.gemspec +3 -3
  31. data/test/boost_test.rb +3 -9
  32. data/test/geo_shape_test.rb +0 -4
  33. data/test/highlight_test.rb +28 -12
  34. data/test/index_test.rb +9 -10
  35. data/test/language_test.rb +16 -0
  36. data/test/marshal_test.rb +6 -1
  37. data/test/match_test.rb +9 -4
  38. data/test/model_test.rb +3 -5
  39. data/test/multi_search_test.rb +0 -7
  40. data/test/order_test.rb +1 -7
  41. data/test/pagination_test.rb +1 -1
  42. data/test/reindex_v2_job_test.rb +6 -11
  43. data/test/routing_test.rb +1 -1
  44. data/test/similar_test.rb +2 -2
  45. data/test/sql_test.rb +0 -31
  46. data/test/test_helper.rb +37 -23
  47. metadata +19 -26
  48. data/test/gemfiles/activerecord31.gemfile +0 -7
  49. data/test/gemfiles/activerecord32.gemfile +0 -7
  50. data/test/gemfiles/activerecord40.gemfile +0 -8
  51. data/test/gemfiles/activerecord41.gemfile +0 -8
  52. data/test/gemfiles/mongoid2.gemfile +0 -7
  53. data/test/gemfiles/mongoid3.gemfile +0 -6
  54. data/test/gemfiles/mongoid4.gemfile +0 -7
  55. data/test/records_test.rb +0 -10
@@ -4,35 +4,23 @@ module Searchkick
4
4
  options = @options
5
5
  language = options[:language]
6
6
  language = language.call if language.respond_to?(:call)
7
- type = options[:_type] || :_default_
8
- type = type.call if type.respond_to?(:call)
7
+ index_type = options[:_type]
8
+ index_type = index_type.call if index_type.respond_to?(:call)
9
9
 
10
10
  if options[:mappings] && !options[:merge_mappings]
11
11
  settings = options[:settings] || {}
12
12
  mappings = options[:mappings]
13
13
  else
14
- below22 = Searchkick.server_below?("2.2.0")
15
- below50 = Searchkick.server_below?("5.0.0-alpha1")
16
14
  below60 = Searchkick.server_below?("6.0.0-alpha1")
17
- default_type = below50 ? "string" : "text"
15
+ default_type = "text"
18
16
  default_analyzer = :searchkick_index
19
- keyword_mapping =
20
- if below50
21
- {
22
- type: default_type,
23
- index: "not_analyzed"
24
- }
25
- else
26
- {
27
- type: "keyword"
28
- }
29
- end
17
+ keyword_mapping = {type: "keyword"}
30
18
 
31
- all = options.key?(:_all) ? options[:_all] : below60
32
- index_true_value = below50 ? "analyzed" : true
33
- index_false_value = below50 ? "no" : false
19
+ all = options.key?(:_all) ? options[:_all] : false
20
+ index_true_value = true
21
+ index_false_value = false
34
22
 
35
- keyword_mapping[:ignore_above] = (options[:ignore_above] || 30000) unless below22
23
+ keyword_mapping[:ignore_above] = options[:ignore_above] || 30000
36
24
 
37
25
  settings = {
38
26
  analysis: {
@@ -40,7 +28,7 @@ module Searchkick
40
28
  searchkick_keyword: {
41
29
  type: "custom",
42
30
  tokenizer: "keyword",
43
- filter: ["lowercase"] + (options[:stem_conversions] == false ? [] : ["searchkick_stemmer"])
31
+ filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : [])
44
32
  },
45
33
  default_analyzer => {
46
34
  type: "custom",
@@ -139,7 +127,6 @@ module Searchkick
139
127
  },
140
128
  searchkick_stemmer: {
141
129
  # use stemmer if language is lowercase, snowball otherwise
142
- # TODO deprecate language option in favor of stemmer
143
130
  type: language == language.to_s.downcase ? "stemmer" : "snowball",
144
131
  language: language || "English"
145
132
  }
@@ -155,6 +142,22 @@ module Searchkick
155
142
  }
156
143
  }
157
144
 
145
+ if language == "chinese"
146
+ settings[:analysis][:analyzer].merge!(
147
+ default_analyzer => {
148
+ type: "ik_smart"
149
+ },
150
+ searchkick_search: {
151
+ type: "ik_smart"
152
+ },
153
+ searchkick_search2: {
154
+ type: "ik_max_word"
155
+ }
156
+ )
157
+
158
+ settings[:analysis][:filter].delete(:searchkick_stemmer)
159
+ end
160
+
158
161
  if Searchkick.env == "test"
159
162
  settings[:number_of_shards] = 1
160
163
  settings[:number_of_replicas] = 0
@@ -175,7 +178,7 @@ module Searchkick
175
178
  settings[:analysis][:filter][:searchkick_synonym] = {
176
179
  type: "synonym",
177
180
  # only remove a single space from synonyms so three-word synonyms will fail noisily instead of silently
178
- synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s| s.sub(/\s+/, "") }.join(",") : s }.map(&:downcase)
181
+ synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s2| s2.sub(/\s+/, "") }.join(",") : s }.map(&:downcase)
179
182
  }
180
183
  # choosing a place for the synonym filter when stemming is not easy
181
184
  # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
@@ -210,7 +213,7 @@ module Searchkick
210
213
  end
211
214
 
212
215
  if options[:special_characters] == false
213
- settings[:analysis][:analyzer].each do |_, analyzer_settings|
216
+ settings[:analysis][:analyzer].each_value do |analyzer_settings|
214
217
  analyzer_settings[:filter].reject! { |f| f == "asciifolding" }
215
218
  end
216
219
  end
@@ -320,7 +323,7 @@ module Searchkick
320
323
  multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
321
324
 
322
325
  mappings = {
323
- type => {
326
+ index_type => {
324
327
  properties: mapping,
325
328
  _routing: routing,
326
329
  # https://gist.github.com/kimchy/2898285
@@ -338,7 +341,7 @@ module Searchkick
338
341
 
339
342
  if below60
340
343
  all_enabled = all && (!options[:searchable] || options[:searchable].to_a.map(&:to_s).include?("_all"))
341
- mappings[type][:_all] = all_enabled ? analyzed_field_options : {enabled: false}
344
+ mappings[index_type][:_all] = all_enabled ? analyzed_field_options : {enabled: false}
342
345
  end
343
346
 
344
347
  mappings = mappings.deep_merge(options[:mappings] || {})
@@ -129,7 +129,7 @@ module Searchkick
129
129
  end
130
130
 
131
131
  module SearchkickWithInstrumentation
132
- def multi_search(searches, **options)
132
+ def multi_search(searches)
133
133
  event = {
134
134
  name: "Multi Search",
135
135
  body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
@@ -167,7 +167,7 @@ module Searchkick
167
167
 
168
168
  # no easy way to tell which host the client will use
169
169
  host = Searchkick.client.transport.hosts.first
170
- debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{payload[:query][:body].to_json}'"
170
+ debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{payload[:query][:body].to_json}'"
171
171
  end
172
172
 
173
173
  def request(event)
@@ -189,7 +189,7 @@ module Searchkick
189
189
 
190
190
  # no easy way to tell which host the client will use
191
191
  host = Searchkick.client.transport.hosts.first
192
- debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -d '#{payload[:body]}'"
192
+ debug " #{color(name, YELLOW, true)} curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -H 'Content-Type: application/json' -d '#{payload[:body]}'"
193
193
  end
194
194
  end
195
195
 
@@ -232,10 +232,11 @@ module Searchkick
232
232
  end
233
233
  end
234
234
  end
235
- Searchkick::Query.send(:prepend, Searchkick::QueryWithInstrumentation)
236
- Searchkick::Index.send(:prepend, Searchkick::IndexWithInstrumentation)
237
- Searchkick::Indexer.send(:prepend, Searchkick::IndexerWithInstrumentation)
238
- Searchkick.singleton_class.send(:prepend, Searchkick::SearchkickWithInstrumentation)
235
+
236
+ Searchkick::Query.prepend(Searchkick::QueryWithInstrumentation)
237
+ Searchkick::Index.prepend(Searchkick::IndexWithInstrumentation)
238
+ Searchkick::Indexer.prepend(Searchkick::IndexerWithInstrumentation)
239
+ Searchkick.singleton_class.prepend(Searchkick::SearchkickWithInstrumentation)
239
240
  Searchkick::LogSubscriber.attach_to :searchkick
240
241
  ActiveSupport.on_load(:action_controller) do
241
242
  include Searchkick::ControllerRuntime
@@ -1,6 +1,8 @@
1
1
  module Searchkick
2
2
  module Model
3
3
  def searchkick(**options)
4
+ options = Searchkick.model_options.merge(options)
5
+
4
6
  unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :conversions, :default_fields,
5
7
  :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
6
8
  :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,
@@ -12,19 +14,29 @@ module Searchkick
12
14
 
13
15
  Searchkick.models << self
14
16
 
15
- options[:_type] ||= -> { searchkick_index.klass_document_type(self, true) } if options[:inheritance]
17
+ options[:_type] ||= -> { searchkick_index.klass_document_type(self, true) }
18
+
19
+ callbacks = options.key?(:callbacks) ? options[:callbacks] : true
20
+ unless [true, false, :async, :queue].include?(callbacks)
21
+ raise ArgumentError, "Invalid value for callbacks"
22
+ end
23
+
24
+ index_name =
25
+ if options[:index_name]
26
+ options[:index_name]
27
+ elsif options[:index_prefix].respond_to?(:call)
28
+ -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
29
+ else
30
+ [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
31
+ end
16
32
 
17
33
  class_eval do
18
34
  cattr_reader :searchkick_options, :searchkick_klass
19
35
 
20
- callbacks = options.key?(:callbacks) ? options[:callbacks] : true
21
-
22
36
  class_variable_set :@@searchkick_options, options.dup
23
37
  class_variable_set :@@searchkick_klass, self
24
- class_variable_set :@@searchkick_callbacks, callbacks
25
- class_variable_set :@@searchkick_index, options[:index_name] ||
26
- (options[:index_prefix].respond_to?(:call) && proc { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }) ||
27
- [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
38
+ class_variable_set :@@searchkick_index, index_name
39
+ class_variable_set :@@searchkick_index_cache, {}
28
40
 
29
41
  class << self
30
42
  def searchkick_search(term = "*", **options, &block)
@@ -33,44 +45,18 @@ module Searchkick
33
45
  alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name
34
46
 
35
47
  def searchkick_index
36
- index = class_variable_get :@@searchkick_index
37
- index = index.call if index.respond_to? :call
38
- Searchkick::Index.new(index, searchkick_options)
48
+ index = class_variable_get(:@@searchkick_index)
49
+ index = index.call if index.respond_to?(:call)
50
+ index_cache = class_variable_get(:@@searchkick_index_cache)
51
+ index_cache[index] ||= Searchkick::Index.new(index, searchkick_options)
39
52
  end
40
53
  alias_method :search_index, :searchkick_index unless method_defined?(:search_index)
41
54
 
42
- def enable_search_callbacks
43
- class_variable_set :@@searchkick_callbacks, true
44
- end
45
-
46
- def disable_search_callbacks
47
- class_variable_set :@@searchkick_callbacks, false
48
- end
49
-
50
- def search_callbacks?
51
- class_variable_get(:@@searchkick_callbacks) && Searchkick.callbacks?
52
- end
53
-
54
- def searchkick_reindex(method_name = nil, full: false, **options)
55
+ def searchkick_reindex(method_name = nil, **options)
55
56
  scoped = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) ||
56
57
  (respond_to?(:queryable) && queryable != unscoped.with_default_scope)
57
58
 
58
- refresh = options.fetch(:refresh, !scoped)
59
-
60
- if method_name
61
- # update
62
- searchkick_index.import_scope(searchkick_klass, method_name: method_name)
63
- searchkick_index.refresh if refresh
64
- true
65
- elsif scoped && !full
66
- # reindex association
67
- searchkick_index.import_scope(searchkick_klass)
68
- searchkick_index.refresh if refresh
69
- true
70
- else
71
- # full reindex
72
- searchkick_index.reindex_scope(searchkick_klass, options)
73
- end
59
+ searchkick_index.reindex(searchkick_klass, method_name, scoped: scoped, **options)
74
60
  end
75
61
  alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
76
62
 
@@ -79,68 +65,29 @@ module Searchkick
79
65
  end
80
66
  end
81
67
 
82
- callback_name = callbacks == :async ? :reindex_async : :reindex
68
+ # always add callbacks, even when callbacks is false
69
+ # so Model.callbacks block can be used
83
70
  if respond_to?(:after_commit)
84
- after_commit callback_name, if: proc { self.class.search_callbacks? }
71
+ after_commit :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
85
72
  elsif respond_to?(:after_save)
86
- after_save callback_name, if: proc { self.class.search_callbacks? }
87
- after_destroy callback_name, if: proc { self.class.search_callbacks? }
73
+ after_save :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
74
+ after_destroy :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
88
75
  end
89
76
 
90
- def reindex(method_name = nil, refresh: false, async: false, mode: nil)
91
- klass_options = self.class.searchkick_index.options
92
-
93
- if mode.nil?
94
- mode =
95
- if async
96
- :async
97
- elsif Searchkick.callbacks_value
98
- Searchkick.callbacks_value
99
- elsif klass_options.key?(:callbacks) && klass_options[:callbacks] != :async
100
- # TODO remove 2nd condition in next major version
101
- klass_options[:callbacks]
102
- end
103
- end
104
-
105
- case mode
106
- when :queue
107
- if method_name
108
- raise Searchkick::Error, "Partial reindex not supported with queue option"
109
- else
110
- self.class.searchkick_index.reindex_queue.push(id.to_s)
111
- end
112
- when :async
113
- if method_name
114
- # TODO support Mongoid and NoBrainer and non-id primary keys
115
- Searchkick::BulkReindexJob.perform_later(
116
- class_name: self.class.name,
117
- record_ids: [id.to_s],
118
- method_name: method_name ? method_name.to_s : nil
119
- )
120
- else
121
- self.class.searchkick_index.reindex_record_async(self)
122
- end
123
- else
124
- if method_name
125
- self.class.searchkick_index.update_record(self, method_name)
126
- else
127
- self.class.searchkick_index.reindex_record(self)
128
- end
129
- self.class.searchkick_index.refresh if refresh
130
- end
77
+ def reindex(method_name = nil, **options)
78
+ RecordIndexer.new(self).reindex(method_name, **options)
131
79
  end unless method_defined?(:reindex)
132
80
 
133
- # TODO remove this method in next major version
134
- def reindex_async
135
- reindex(async: true)
136
- end unless method_defined?(:reindex_async)
137
-
138
81
  def similar(options = {})
139
82
  self.class.searchkick_index.similar_record(self, options)
140
83
  end unless method_defined?(:similar)
141
84
 
142
85
  def search_data
143
- respond_to?(:to_hash) ? to_hash : serializable_hash
86
+ data = respond_to?(:to_hash) ? to_hash : serializable_hash
87
+ data.delete("id")
88
+ data.delete("_id")
89
+ data.delete("_type")
90
+ data
144
91
  end unless method_defined?(:search_data)
145
92
 
146
93
  def should_index?
@@ -2,25 +2,24 @@ module Searchkick
2
2
  class MultiSearch
3
3
  attr_reader :queries
4
4
 
5
- def initialize(queries, retry_misspellings: false)
5
+ def initialize(queries)
6
6
  @queries = queries
7
- @retry_misspellings = retry_misspellings
8
7
  end
9
8
 
10
9
  def perform
11
10
  if queries.any?
12
- perform_search(queries, retry_misspellings: @retry_misspellings)
11
+ perform_search(queries)
13
12
  end
14
13
  end
15
14
 
16
15
  private
17
16
 
18
- def perform_search(queries, retry_misspellings: true)
17
+ def perform_search(queries)
19
18
  responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
20
19
 
21
20
  retry_queries = []
22
21
  queries.each_with_index do |query, i|
23
- if retry_misspellings && query.retry_misspellings?(responses[i])
22
+ if query.retry_misspellings?(responses[i])
24
23
  query.send(:prepare) # okay, since we don't want to expose this method outside Searchkick
25
24
  retry_queries << query
26
25
  else
@@ -28,8 +27,8 @@ module Searchkick
28
27
  end
29
28
  end
30
29
 
31
- if retry_misspellings && retry_queries.any?
32
- perform_search(retry_queries, retry_misspellings: false)
30
+ if retry_queries.any?
31
+ perform_search(retry_queries)
33
32
  end
34
33
 
35
34
  queries
@@ -135,7 +135,7 @@ module Searchkick
135
135
  if searchkick_index
136
136
  puts "Model Search Data"
137
137
  begin
138
- pp klass.first(3).map { |r| {index: searchkick_index.record_data(r).merge(data: searchkick_index.send(:search_data, r))}}
138
+ pp(klass.first(3).map { |r| {index: searchkick_index.record_data(r).merge(data: searchkick_index.send(:search_data, r))}})
139
139
  rescue => e
140
140
  puts "#{e.class.name}: #{e.message}"
141
141
  end
@@ -179,14 +179,14 @@ module Searchkick
179
179
  e.message.include?("No query registered for [function_score]")
180
180
  )
181
181
 
182
- raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 2 or greater"
182
+ raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
183
183
  elsif status_code == 400
184
184
  if (
185
185
  e.message.include?("bool query does not support [filter]") ||
186
186
  e.message.include?("[bool] filter does not support [filter]")
187
187
  )
188
188
 
189
- raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 2 or greater"
189
+ raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
190
190
  elsif e.message.include?("[multi_match] analyzer [searchkick_search] not found")
191
191
  raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
192
192
  else
@@ -212,15 +212,13 @@ module Searchkick
212
212
 
213
213
  # pagination
214
214
  page = [options[:page].to_i, 1].max
215
- per_page = (options[:limit] || options[:per_page] || 1_000).to_i
215
+ per_page = (options[:limit] || options[:per_page] || 10_000).to_i
216
216
  padding = [options[:padding].to_i, 0].max
217
217
  offset = options[:offset] || (page - 1) * per_page + padding
218
218
 
219
219
  # model and eager loading
220
220
  load = options[:load].nil? ? true : options[:load]
221
221
 
222
- conversions_fields = Array(options[:conversions] || searchkick_options[:conversions]).map(&:to_s)
223
-
224
222
  all = term == "*"
225
223
 
226
224
  @json = options[:body]
@@ -228,12 +226,15 @@ module Searchkick
228
226
  ignored_options = options.keys & [:aggs, :boost,
229
227
  :boost_by, :boost_by_distance, :boost_where, :conversions, :conversions_term, :exclude, :explain,
230
228
  :fields, :highlight, :indices_boost, :limit, :match, :misspellings, :offset, :operator, :order,
231
- :padding, :page, :per_page, :select, :smart_aggs, :suggest, :where]
232
- warn "The body option replaces the entire body, so the following options are ignored: #{ignored_options.join(", ")}" if ignored_options.any?
229
+ :padding, :page, :per_page, :profile, :select, :smart_aggs, :suggest, :where]
230
+ raise ArgumentError, "Options incompatible with body option: #{ignored_options.join(", ")}" if ignored_options.any?
233
231
  payload = @json
234
232
  else
233
+ must_not = []
234
+ should = []
235
+
235
236
  if options[:similar]
236
- payload = {
237
+ query = {
237
238
  more_like_this: {
238
239
  like: term,
239
240
  min_doc_freq: 1,
@@ -241,11 +242,14 @@ module Searchkick
241
242
  analyzer: "searchkick_search2"
242
243
  }
243
244
  }
245
+ if fields.all? { |f| f.start_with?("*.") }
246
+ raise ArgumentError, "Must specify fields to search"
247
+ end
244
248
  if fields != ["_all"]
245
- payload[:more_like_this][:fields] = fields
249
+ query[:more_like_this][:fields] = fields
246
250
  end
247
251
  elsif all
248
- payload = {
252
+ query = {
249
253
  match_all: {}
250
254
  }
251
255
  else
@@ -327,10 +331,19 @@ module Searchkick
327
331
  end
328
332
 
329
333
  if misspellings != false && match_type == :match
330
- qs.concat qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) }
334
+ qs.concat(qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) })
331
335
  end
332
336
 
333
- q2 = qs.map { |q| {match_type => {field => q}} }
337
+ if field.start_with?("*.")
338
+ q2 = qs.map { |q| {multi_match: q.merge(fields: [field], type: match_type == :match_phrase ? "phrase" : "best_fields")} }
339
+ if below61?
340
+ q2.each do |q|
341
+ q[:multi_match].delete(:fuzzy_transpositions)
342
+ end
343
+ end
344
+ else
345
+ q2 = qs.map { |q| {match_type => {field => q}} }
346
+ end
334
347
 
335
348
  # boost exact matches more
336
349
  if field =~ /\.word_(start|middle|end)\z/ && searchkick_options[:word] != false
@@ -348,28 +361,11 @@ module Searchkick
348
361
  queries_to_add.concat(q2)
349
362
  end
350
363
 
351
- if options[:exclude]
352
- must_not =
353
- Array(options[:exclude]).map do |phrase|
354
- {
355
- match_phrase: {
356
- exclude_field => {
357
- query: phrase,
358
- analyzer: exclude_analyzer
359
- }
360
- }
361
- }
362
- end
364
+ queries.concat(queries_to_add)
363
365
 
364
- queries_to_add = [{
365
- bool: {
366
- should: queries_to_add,
367
- must_not: must_not
368
- }
369
- }]
366
+ if options[:exclude]
367
+ must_not.concat(set_exclude(exclude_field, exclude_analyzer))
370
368
  end
371
-
372
- queries.concat(queries_to_add)
373
369
  end
374
370
 
375
371
  payload = {
@@ -378,78 +374,15 @@ module Searchkick
378
374
  }
379
375
  }
380
376
 
381
- if conversions_fields.present? && options[:conversions] != false
382
- shoulds = []
383
- conversions_fields.each do |conversions_field|
384
- # wrap payload in a bool query
385
- script_score = {field_value_factor: {field: "#{conversions_field}.count"}}
386
-
387
- shoulds << {
388
- nested: {
389
- path: conversions_field,
390
- score_mode: "sum",
391
- query: {
392
- function_score: {
393
- boost_mode: "replace",
394
- query: {
395
- match: {
396
- "#{conversions_field}.query" => options[:conversions_term] || term
397
- }
398
- }
399
- }.merge(script_score)
400
- }
401
- }
402
- }
403
- end
404
- payload = {
405
- bool: {
406
- must: payload,
407
- should: shoulds
408
- }
409
- }
410
- end
411
- end
412
-
413
- custom_filters = []
414
- multiply_filters = []
415
-
416
- set_boost_by(multiply_filters, custom_filters)
417
- set_boost_where(custom_filters)
418
- set_boost_by_distance(custom_filters) if options[:boost_by_distance]
377
+ should.concat(set_conversions)
419
378
 
420
- if custom_filters.any?
421
- payload = {
422
- function_score: {
423
- functions: custom_filters,
424
- query: payload,
425
- score_mode: "sum"
426
- }
427
- }
428
- end
429
-
430
- if multiply_filters.any?
431
- payload = {
432
- function_score: {
433
- functions: multiply_filters,
434
- query: payload,
435
- score_mode: "multiply"
436
- }
437
- }
379
+ query = payload
438
380
  end
439
381
 
440
382
  payload = {
441
- query: payload,
442
383
  size: per_page,
443
384
  from: offset
444
385
  }
445
- payload[:explain] = options[:explain] if options[:explain]
446
- payload[:profile] = options[:profile] if options[:profile]
447
-
448
- # order
449
- set_order(payload) if options[:order]
450
-
451
- # indices_boost
452
- set_boost_by_indices(payload)
453
386
 
454
387
  # type when inheritance
455
388
  where = (options[:where] || {}).dup
@@ -465,8 +398,26 @@ module Searchkick
465
398
  # aggregations
466
399
  set_aggregations(payload, filters, post_filters) if options[:aggs]
467
400
 
468
- # filters
469
- set_filters(payload, filters, post_filters)
401
+ # post filters
402
+ set_post_filters(payload, post_filters) if post_filters.any?
403
+
404
+ custom_filters = []
405
+ multiply_filters = []
406
+
407
+ set_boost_by(multiply_filters, custom_filters)
408
+ set_boost_where(custom_filters)
409
+ set_boost_by_distance(custom_filters) if options[:boost_by_distance]
410
+
411
+ payload[:query] = build_query(query, filters, should, must_not, custom_filters, multiply_filters)
412
+
413
+ payload[:explain] = options[:explain] if options[:explain]
414
+ payload[:profile] = options[:profile] if options[:profile]
415
+
416
+ # order
417
+ set_order(payload) if options[:order]
418
+
419
+ # indices_boost
420
+ set_boost_by_indices(payload)
470
421
 
471
422
  # suggestions
472
423
  set_suggestions(payload, options[:suggest]) if options[:suggest]
@@ -512,7 +463,7 @@ module Searchkick
512
463
  def set_fields
513
464
  boost_fields = {}
514
465
  fields = options[:fields] || searchkick_options[:default_fields] || searchkick_options[:searchable]
515
- all = searchkick_options.key?(:_all) ? searchkick_options[:_all] : below60?
466
+ all = searchkick_options.key?(:_all) ? searchkick_options[:_all] : false
516
467
  default_match = options[:match] || searchkick_options[:match] || :word
517
468
  fields =
518
469
  if fields
@@ -529,12 +480,88 @@ module Searchkick
529
480
  ["_all.phrase"]
530
481
  elsif term == "*"
531
482
  []
532
- else
483
+ elsif default_match == :exact
533
484
  raise ArgumentError, "Must specify fields to search"
485
+ else
486
+ [default_match == :word ? "*.analyzed" : "*.#{default_match}"]
534
487
  end
535
488
  [boost_fields, fields]
536
489
  end
537
490
 
491
+ def build_query(query, filters, should, must_not, custom_filters, multiply_filters)
492
+ if filters.any? || must_not.any? || should.any?
493
+ bool = {must: query}
494
+ bool[:filter] = filters if filters.any? # where
495
+ bool[:must_not] = must_not if must_not.any? # exclude
496
+ bool[:should] = should if should.any? # conversions
497
+ query = {bool: bool}
498
+ end
499
+
500
+ if custom_filters.any?
501
+ query = {
502
+ function_score: {
503
+ functions: custom_filters,
504
+ query: query,
505
+ score_mode: "sum"
506
+ }
507
+ }
508
+ end
509
+
510
+ if multiply_filters.any?
511
+ query = {
512
+ function_score: {
513
+ functions: multiply_filters,
514
+ query: query,
515
+ score_mode: "multiply"
516
+ }
517
+ }
518
+ end
519
+
520
+ query
521
+ end
522
+
523
+ def set_conversions
524
+ conversions_fields = Array(options[:conversions] || searchkick_options[:conversions]).map(&:to_s)
525
+ if conversions_fields.present? && options[:conversions] != false
526
+ conversions_fields.map do |conversions_field|
527
+ {
528
+ nested: {
529
+ path: conversions_field,
530
+ score_mode: "sum",
531
+ query: {
532
+ function_score: {
533
+ boost_mode: "replace",
534
+ query: {
535
+ match: {
536
+ "#{conversions_field}.query" => options[:conversions_term] || term
537
+ }
538
+ },
539
+ field_value_factor: {
540
+ field: "#{conversions_field}.count"
541
+ }
542
+ }
543
+ }
544
+ }
545
+ }
546
+ end
547
+ else
548
+ []
549
+ end
550
+ end
551
+
552
+ def set_exclude(field, analyzer)
553
+ Array(options[:exclude]).map do |phrase|
554
+ {
555
+ multi_match: {
556
+ fields: [field],
557
+ query: phrase,
558
+ analyzer: analyzer,
559
+ type: "phrase"
560
+ }
561
+ }
562
+ end
563
+ end
564
+
538
565
  def set_boost_by_distance(custom_filters)
539
566
  boost_by_distance = options[:boost_by_distance] || {}
540
567
 
@@ -564,11 +591,11 @@ module Searchkick
564
591
  if boost_by.is_a?(Array)
565
592
  boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
566
593
  elsif boost_by.is_a?(Hash)
567
- multiply_by, boost_by = boost_by.partition { |_, v| v[:boost_mode] == "multiply" }.map { |i| Hash[i] }
594
+ multiply_by, boost_by = boost_by.partition { |_, v| v.delete(:boost_mode) == "multiply" }.map { |i| Hash[i] }
568
595
  end
569
596
  boost_by[options[:boost]] = {factor: 1} if options[:boost]
570
597
 
571
- custom_filters.concat boost_filters(boost_by, log: true)
598
+ custom_filters.concat boost_filters(boost_by, modifier: "ln2p")
572
599
  multiply_filters.concat boost_filters(multiply_by || {})
573
600
  end
574
601
 
@@ -631,7 +658,8 @@ module Searchkick
631
658
 
632
659
  def set_highlights(payload, fields)
633
660
  payload[:highlight] = {
634
- fields: Hash[fields.map { |f| [f, {}] }]
661
+ fields: Hash[fields.map { |f| [f, {}] }],
662
+ fragment_size: below60? ? 30000 : 0
635
663
  }
636
664
 
637
665
  if options[:highlight].is_a?(Hash)
@@ -683,7 +711,7 @@ module Searchkick
683
711
  ranges: agg_options[:date_ranges]
684
712
  }.merge(shared_agg_options)
685
713
  }
686
- elsif histogram = agg_options[:date_histogram]
714
+ elsif (histogram = agg_options[:date_histogram])
687
715
  interval = histogram[:interval]
688
716
  payload[:aggs][field] = {
689
717
  date_histogram: {
@@ -691,7 +719,7 @@ module Searchkick
691
719
  interval: interval
692
720
  }
693
721
  }
694
- elsif metric = @@metric_aggs.find { |k| agg_options.has_key?(k) }
722
+ elsif (metric = @@metric_aggs.find { |k| agg_options.has_key?(k) })
695
723
  payload[:aggs][field] = {
696
724
  metric => {
697
725
  field: agg_options[metric][:field] || field
@@ -735,30 +763,18 @@ module Searchkick
735
763
  end
736
764
  end
737
765
 
738
- def set_filters(payload, filters, post_filters)
739
- if post_filters.any?
740
- payload[:post_filter] = {
741
- bool: {
742
- filter: post_filters
743
- }
766
+ def set_post_filters(payload, post_filters)
767
+ payload[:post_filter] = {
768
+ bool: {
769
+ filter: post_filters
744
770
  }
745
- end
746
-
747
- if filters.any?
748
- # more efficient query if no aggs
749
- payload[:query] = {
750
- bool: {
751
- must: payload[:query],
752
- filter: filters
753
- }
754
- }
755
- end
771
+ }
756
772
  end
757
773
 
758
774
  # TODO id transformation for arrays
759
775
  def set_order(payload)
760
776
  order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
761
- id_field = below50? ? :_id : :_uid
777
+ id_field = :_uid
762
778
  payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
763
779
  end
764
780
 
@@ -889,50 +905,37 @@ module Searchkick
889
905
  end
890
906
 
891
907
  def custom_filter(field, value, factor)
892
- if below50?
893
- {
894
- filter: {
895
- bool: {
896
- must: where_filters(field => value)
897
- }
898
- },
899
- boost_factor: factor
908
+ {
909
+ filter: where_filters(field => value),
910
+ weight: factor
911
+ }
912
+ end
913
+
914
+ def boost_filter(field, factor: 1, modifier: nil, missing: nil)
915
+ script_score = {
916
+ field_value_factor: {
917
+ field: field,
918
+ factor: factor.to_f,
919
+ modifier: modifier
900
920
  }
921
+ }
922
+
923
+ if missing
924
+ script_score[:field_value_factor][:missing] = missing.to_f
901
925
  else
902
- {
903
- filter: where_filters(field => value),
904
- weight: factor
926
+ script_score[:filter] = {
927
+ exists: {
928
+ field: field
929
+ }
905
930
  }
906
931
  end
932
+
933
+ script_score
907
934
  end
908
935
 
909
- def boost_filters(boost_by, options = {})
936
+ def boost_filters(boost_by, modifier: nil)
910
937
  boost_by.map do |field, value|
911
- log = value.key?(:log) ? value[:log] : options[:log]
912
- value[:factor] ||= 1
913
- script_score = {
914
- field_value_factor: {
915
- field: field,
916
- factor: value[:factor].to_f,
917
- modifier: value[:modifier] || (log ? "ln2p" : nil)
918
- }
919
- }
920
-
921
- if value[:missing]
922
- if below50?
923
- raise ArgumentError, "The missing option for boost_by is not supported in Elasticsearch < 5"
924
- else
925
- script_score[:field_value_factor][:missing] = value[:missing].to_f
926
- end
927
- else
928
- script_score[:filter] = {
929
- exists: {
930
- field: field
931
- }
932
- }
933
- end
934
-
935
- script_score
938
+ boost_filter(field, modifier: modifier, **value)
936
939
  end
937
940
  end
938
941
 
@@ -957,12 +960,12 @@ module Searchkick
957
960
  end
958
961
  end
959
962
 
960
- def below50?
961
- Searchkick.server_below?("5.0.0-alpha1")
962
- end
963
-
964
963
  def below60?
965
964
  Searchkick.server_below?("6.0.0-alpha1")
966
965
  end
966
+
967
+ def below61?
968
+ Searchkick.server_below?("6.1.0-alpha1")
969
+ end
967
970
  end
968
971
  end