searchkick 2.3.2 → 5.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +377 -84
  3. data/LICENSE.txt +1 -1
  4. data/README.md +859 -602
  5. data/lib/searchkick/bulk_reindex_job.rb +13 -9
  6. data/lib/searchkick/controller_runtime.rb +40 -0
  7. data/lib/searchkick/hash_wrapper.rb +12 -0
  8. data/lib/searchkick/index.rb +281 -356
  9. data/lib/searchkick/index_cache.rb +30 -0
  10. data/lib/searchkick/index_options.rb +487 -281
  11. data/lib/searchkick/indexer.rb +15 -8
  12. data/lib/searchkick/log_subscriber.rb +57 -0
  13. data/lib/searchkick/middleware.rb +9 -2
  14. data/lib/searchkick/model.rb +72 -118
  15. data/lib/searchkick/multi_search.rb +9 -10
  16. data/lib/searchkick/process_batch_job.rb +12 -15
  17. data/lib/searchkick/process_queue_job.rb +22 -13
  18. data/lib/searchkick/query.rb +458 -217
  19. data/lib/searchkick/railtie.rb +7 -0
  20. data/lib/searchkick/record_data.rb +128 -0
  21. data/lib/searchkick/record_indexer.rb +164 -0
  22. data/lib/searchkick/reindex_queue.rb +51 -9
  23. data/lib/searchkick/reindex_v2_job.rb +10 -32
  24. data/lib/searchkick/relation.rb +247 -0
  25. data/lib/searchkick/relation_indexer.rb +155 -0
  26. data/lib/searchkick/results.rb +201 -82
  27. data/lib/searchkick/version.rb +1 -1
  28. data/lib/searchkick/where.rb +11 -0
  29. data/lib/searchkick.rb +269 -97
  30. data/lib/tasks/searchkick.rake +37 -0
  31. metadata +24 -178
  32. data/.gitignore +0 -22
  33. data/.travis.yml +0 -39
  34. data/Gemfile +0 -16
  35. data/Rakefile +0 -20
  36. data/benchmark/Gemfile +0 -23
  37. data/benchmark/benchmark.rb +0 -97
  38. data/lib/searchkick/logging.rb +0 -242
  39. data/lib/searchkick/tasks.rb +0 -33
  40. data/searchkick.gemspec +0 -28
  41. data/test/aggs_test.rb +0 -197
  42. data/test/autocomplete_test.rb +0 -75
  43. data/test/boost_test.rb +0 -202
  44. data/test/callbacks_test.rb +0 -59
  45. data/test/ci/before_install.sh +0 -17
  46. data/test/errors_test.rb +0 -19
  47. data/test/gemfiles/activerecord31.gemfile +0 -7
  48. data/test/gemfiles/activerecord32.gemfile +0 -7
  49. data/test/gemfiles/activerecord40.gemfile +0 -8
  50. data/test/gemfiles/activerecord41.gemfile +0 -8
  51. data/test/gemfiles/activerecord42.gemfile +0 -7
  52. data/test/gemfiles/activerecord50.gemfile +0 -7
  53. data/test/gemfiles/apartment.gemfile +0 -8
  54. data/test/gemfiles/cequel.gemfile +0 -8
  55. data/test/gemfiles/mongoid2.gemfile +0 -7
  56. data/test/gemfiles/mongoid3.gemfile +0 -6
  57. data/test/gemfiles/mongoid4.gemfile +0 -7
  58. data/test/gemfiles/mongoid5.gemfile +0 -7
  59. data/test/gemfiles/mongoid6.gemfile +0 -12
  60. data/test/gemfiles/nobrainer.gemfile +0 -8
  61. data/test/gemfiles/parallel_tests.gemfile +0 -8
  62. data/test/geo_shape_test.rb +0 -175
  63. data/test/highlight_test.rb +0 -78
  64. data/test/index_test.rb +0 -166
  65. data/test/inheritance_test.rb +0 -83
  66. data/test/marshal_test.rb +0 -8
  67. data/test/match_test.rb +0 -276
  68. data/test/misspellings_test.rb +0 -56
  69. data/test/model_test.rb +0 -42
  70. data/test/multi_search_test.rb +0 -36
  71. data/test/multi_tenancy_test.rb +0 -22
  72. data/test/order_test.rb +0 -46
  73. data/test/pagination_test.rb +0 -70
  74. data/test/partial_reindex_test.rb +0 -58
  75. data/test/query_test.rb +0 -35
  76. data/test/records_test.rb +0 -10
  77. data/test/reindex_test.rb +0 -64
  78. data/test/reindex_v2_job_test.rb +0 -32
  79. data/test/routing_test.rb +0 -23
  80. data/test/should_index_test.rb +0 -32
  81. data/test/similar_test.rb +0 -28
  82. data/test/sql_test.rb +0 -214
  83. data/test/suggest_test.rb +0 -95
  84. data/test/support/kaminari.yml +0 -21
  85. data/test/synonyms_test.rb +0 -67
  86. data/test/test_helper.rb +0 -567
  87. data/test/where_test.rb +0 -223
@@ -1,7 +1,5 @@
1
1
  module Searchkick
2
2
  class Index
3
- include IndexOptions
4
-
5
3
  attr_reader :name, :options
6
4
 
7
5
  def initialize(name, options = {})
@@ -10,12 +8,16 @@ module Searchkick
10
8
  @klass_document_type = {} # cache
11
9
  end
12
10
 
11
+ def index_options
12
+ IndexOptions.new(self).index_options
13
+ end
14
+
13
15
  def create(body = {})
14
16
  client.indices.create index: name, body: body
15
17
  end
16
18
 
17
19
  def delete
18
- if !Searchkick.server_below?("6.0.0-alpha1") && alias_exists?
20
+ if alias_exists?
19
21
  # can't call delete directly on aliases in ES 6
20
22
  indices = client.indices.get_alias(name: name).keys
21
23
  client.indices.delete index: indices
@@ -36,25 +38,45 @@ module Searchkick
36
38
  client.indices.exists_alias name: name
37
39
  end
38
40
 
41
+ # call to_h for consistent results between elasticsearch gem 7 and 8
42
+ # could do for all API calls, but just do for ones where return value is focus for now
39
43
  def mapping
40
- client.indices.get_mapping index: name
44
+ client.indices.get_mapping(index: name).to_h
41
45
  end
42
46
 
47
+ # call to_h for consistent results between elasticsearch gem 7 and 8
43
48
  def settings
44
- client.indices.get_settings index: name
49
+ client.indices.get_settings(index: name).to_h
45
50
  end
46
51
 
47
52
  def refresh_interval
48
- settings.values.first["settings"]["index"]["refresh_interval"]
53
+ index_settings["refresh_interval"]
49
54
  end
50
55
 
51
56
  def update_settings(settings)
52
57
  client.indices.put_settings index: name, body: settings
53
58
  end
54
59
 
60
+ def tokens(text, options = {})
61
+ client.indices.analyze(body: {text: text}.merge(options), index: name)["tokens"].map { |t| t["token"] }
62
+ end
63
+
64
+ def total_docs
65
+ response =
66
+ client.search(
67
+ index: name,
68
+ body: {
69
+ query: {match_all: {}},
70
+ size: 0
71
+ }
72
+ )
73
+
74
+ Results.new(nil, response).total_count
75
+ end
76
+
55
77
  def promote(new_name, update_refresh_interval: false)
56
78
  if update_refresh_interval
57
- new_index = Searchkick::Index.new(new_name)
79
+ new_index = Index.new(new_name, @options)
58
80
  settings = options[:settings] || {}
59
81
  refresh_interval = (settings[:index] && settings[:index][:refresh_interval]) || "1s"
60
82
  new_index.update_settings(index: {refresh_interval: refresh_interval})
@@ -63,7 +85,8 @@ module Searchkick
63
85
  old_indices =
64
86
  begin
65
87
  client.indices.get_alias(name: name).keys
66
- rescue Elasticsearch::Transport::Transport::Errors::NotFound
88
+ rescue => e
89
+ raise e unless Searchkick.not_found_error?(e)
67
90
  {}
68
91
  end
69
92
  actions = old_indices.map { |old_name| {remove: {index: old_name, alias: name}} } + [{add: {index: new_name, alias: name}}]
@@ -71,176 +94,293 @@ module Searchkick
71
94
  end
72
95
  alias_method :swap, :promote
73
96
 
74
- # record based
75
- # use helpers for notifications
97
+ def retrieve(record)
98
+ record_data = RecordData.new(self, record).record_data
99
+
100
+ # remove underscore
101
+ get_options = record_data.to_h { |k, v| [k.to_s.sub(/\A_/, "").to_sym, v] }
102
+
103
+ client.get(get_options)["_source"]
104
+ end
105
+
106
+ def all_indices(unaliased: false)
107
+ indices =
108
+ begin
109
+ if client.indices.respond_to?(:get_alias)
110
+ client.indices.get_alias(index: "#{name}*")
111
+ else
112
+ client.indices.get_aliases
113
+ end
114
+ rescue => e
115
+ raise e unless Searchkick.not_found_error?(e)
116
+ {}
117
+ end
118
+ indices = indices.select { |_k, v| v.empty? || v["aliases"].empty? } if unaliased
119
+ indices.select { |k, _v| k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
120
+ end
121
+
122
+ # remove old indices that start w/ index_name
123
+ def clean_indices
124
+ indices = all_indices(unaliased: true)
125
+ indices.each do |index|
126
+ Index.new(index).delete
127
+ end
128
+ indices
129
+ end
76
130
 
77
131
  def store(record)
78
- bulk_index_helper([record])
132
+ notify(record, "Store") do
133
+ queue_index([record])
134
+ end
79
135
  end
80
136
 
81
137
  def remove(record)
82
- bulk_delete_helper([record])
138
+ notify(record, "Remove") do
139
+ queue_delete([record])
140
+ end
83
141
  end
84
142
 
85
143
  def update_record(record, method_name)
86
- bulk_update_helper([record], method_name)
144
+ notify(record, "Update") do
145
+ queue_update([record], method_name)
146
+ end
87
147
  end
88
148
 
89
149
  def bulk_delete(records)
90
- bulk_delete_helper(records)
150
+ return if records.empty?
151
+
152
+ notify_bulk(records, "Delete") do
153
+ queue_delete(records)
154
+ end
91
155
  end
92
156
 
93
157
  def bulk_index(records)
94
- bulk_index_helper(records)
158
+ return if records.empty?
159
+
160
+ notify_bulk(records, "Import") do
161
+ queue_index(records)
162
+ end
95
163
  end
96
164
  alias_method :import, :bulk_index
97
165
 
98
166
  def bulk_update(records, method_name)
99
- bulk_update_helper(records, method_name)
100
- end
167
+ return if records.empty?
101
168
 
102
- def record_data(r)
103
- data = {
104
- _index: name,
105
- _id: search_id(r),
106
- _type: document_type(r)
107
- }
108
- data[:_routing] = r.search_routing if r.respond_to?(:search_routing)
109
- data
110
- end
111
-
112
- def retrieve(record)
113
- client.get(
114
- index: name,
115
- type: document_type(record),
116
- id: search_id(record)
117
- )["_source"]
169
+ notify_bulk(records, "Update") do
170
+ queue_update(records, method_name)
171
+ end
118
172
  end
119
173
 
120
- def reindex_record(record)
121
- if record.destroyed? || !record.should_index?
122
- begin
123
- remove(record)
124
- rescue Elasticsearch::Transport::Transport::Errors::NotFound
125
- # do nothing
126
- end
127
- else
128
- store(record)
129
- end
174
+ def search_id(record)
175
+ RecordData.new(self, record).search_id
130
176
  end
131
177
 
132
- def reindex_record_async(record)
133
- if Searchkick.callbacks_value.nil?
134
- if defined?(Searchkick::ReindexV2Job)
135
- Searchkick::ReindexV2Job.perform_later(record.class.name, record.id.to_s)
136
- else
137
- raise Searchkick::Error, "Active Job not found"
138
- end
139
- else
140
- reindex_record(record)
141
- end
178
+ def document_type(record)
179
+ RecordData.new(self, record).document_type
142
180
  end
143
181
 
144
182
  def similar_record(record, **options)
145
- like_text = retrieve(record).to_hash
146
- .keep_if { |k, _| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
147
- .values.compact.join(" ")
148
-
149
- # TODO deep merge method
150
- options[:where] ||= {}
151
- options[:where][:_id] ||= {}
152
- options[:where][:_id][:not] = record.id.to_s
153
183
  options[:per_page] ||= 10
154
- options[:similar] = true
184
+ options[:similar] = [RecordData.new(self, record).record_data]
185
+ options[:models] ||= [record.class] unless options.key?(:model)
155
186
 
156
- # TODO use index class instead of record class
157
- search_model(record.class, like_text, options)
187
+ Searchkick.search("*", **options)
188
+ end
189
+
190
+ def reload_synonyms
191
+ if Searchkick.opensearch?
192
+ client.transport.perform_request "POST", "_plugins/_refresh_search_analyzers/#{CGI.escape(name)}"
193
+ else
194
+ raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
195
+ begin
196
+ client.transport.perform_request("GET", "#{CGI.escape(name)}/_reload_search_analyzers")
197
+ rescue => e
198
+ raise Error, "Requires non-OSS version of Elasticsearch" if Searchkick.not_allowed_error?(e)
199
+ raise e
200
+ end
201
+ end
158
202
  end
159
203
 
160
204
  # queue
161
205
 
162
206
  def reindex_queue
163
- Searchkick::ReindexQueue.new(name)
207
+ ReindexQueue.new(name)
164
208
  end
165
209
 
166
- # search
210
+ # reindex
211
+
212
+ # note: this is designed to be used internally
213
+ # so it does not check object matches index class
214
+ def reindex(object, method_name: nil, full: false, **options)
215
+ if object.is_a?(Array)
216
+ # note: purposefully skip full
217
+ return reindex_records(object, method_name: method_name, **options)
218
+ end
219
+
220
+ if !object.respond_to?(:searchkick_klass)
221
+ raise Error, "Cannot reindex object"
222
+ end
223
+
224
+ scoped = Searchkick.relation?(object)
225
+ # call searchkick_klass for inheritance
226
+ relation = scoped ? object.all : Searchkick.scope(object.searchkick_klass).all
227
+
228
+ refresh = options.fetch(:refresh, !scoped)
229
+ options.delete(:refresh)
167
230
 
168
- # TODO remove in next major version
169
- def search_model(searchkick_klass, term = "*", **options, &block)
170
- query = Searchkick::Query.new(searchkick_klass, term, options)
171
- yield(query.body) if block
172
- if options[:execute] == false
173
- query
231
+ if method_name || (scoped && !full)
232
+ mode = options.delete(:mode) || :inline
233
+ raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
234
+
235
+ # import only
236
+ import_scope(relation, method_name: method_name, mode: mode)
237
+ self.refresh if refresh
238
+ true
174
239
  else
175
- query.execute
240
+ async = options.delete(:async)
241
+ if async
242
+ if async.is_a?(Hash) && async[:wait]
243
+ # TODO warn in 5.1
244
+ # Searchkick.warn "async option is deprecated - use mode: :async, wait: true instead"
245
+ options[:wait] = true unless options.key?(:wait)
246
+ else
247
+ # TODO warn in 5.1
248
+ # Searchkick.warn "async option is deprecated - use mode: :async instead"
249
+ end
250
+ options[:mode] ||= :async
251
+ end
252
+
253
+ full_reindex(relation, **options)
176
254
  end
177
255
  end
178
256
 
179
- # reindex
180
-
181
257
  def create_index(index_options: nil)
182
258
  index_options ||= self.index_options
183
- index = Searchkick::Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
259
+ index = Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
184
260
  index.create(index_options)
185
261
  index
186
262
  end
187
263
 
188
- def all_indices(unaliased: false)
189
- indices =
190
- begin
191
- client.indices.get_aliases
192
- rescue Elasticsearch::Transport::Transport::Errors::NotFound
193
- {}
264
+ def import_scope(relation, **options)
265
+ relation_indexer.reindex(relation, **options)
266
+ end
267
+
268
+ def batches_left
269
+ relation_indexer.batches_left
270
+ end
271
+
272
+ # private
273
+ def klass_document_type(klass, ignore_type = false)
274
+ @klass_document_type[[klass, ignore_type]] ||= begin
275
+ if !ignore_type && klass.searchkick_klass.searchkick_options[:_type]
276
+ type = klass.searchkick_klass.searchkick_options[:_type]
277
+ type = type.call if type.respond_to?(:call)
278
+ type
279
+ else
280
+ klass.model_name.to_s.underscore
194
281
  end
195
- indices = indices.select { |_k, v| v.empty? || v["aliases"].empty? } if unaliased
196
- indices.select { |k, _v| k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
282
+ end
197
283
  end
198
284
 
199
- # remove old indices that start w/ index_name
200
- def clean_indices
201
- indices = all_indices(unaliased: true)
202
- indices.each do |index|
203
- Searchkick::Index.new(index).delete
285
+ # private
286
+ def conversions_fields
287
+ @conversions_fields ||= begin
288
+ conversions = Array(options[:conversions])
289
+ conversions.map(&:to_s) + conversions.map(&:to_sym)
204
290
  end
205
- indices
206
291
  end
207
292
 
208
- def total_docs
209
- response =
210
- client.search(
211
- index: name,
212
- body: {
213
- query: {match_all: {}},
214
- size: 0
215
- }
216
- )
293
+ # private
294
+ def suggest_fields
295
+ @suggest_fields ||= Array(options[:suggest]).map(&:to_s)
296
+ end
297
+
298
+ # private
299
+ def locations_fields
300
+ @locations_fields ||= begin
301
+ locations = Array(options[:locations])
302
+ locations.map(&:to_s) + locations.map(&:to_sym)
303
+ end
304
+ end
305
+
306
+ # private
307
+ def uuid
308
+ index_settings["uuid"]
309
+ end
217
310
 
218
- response["hits"]["total"]
311
+ protected
312
+
313
+ def client
314
+ Searchkick.client
315
+ end
316
+
317
+ def queue_index(records)
318
+ Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).index_data })
319
+ end
320
+
321
+ def queue_delete(records)
322
+ Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data })
323
+ end
324
+
325
+ def queue_update(records, method_name)
326
+ Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) })
327
+ end
328
+
329
+ def relation_indexer
330
+ @relation_indexer ||= RelationIndexer.new(self)
331
+ end
332
+
333
+ def index_settings
334
+ settings.values.first["settings"]["index"]
335
+ end
336
+
337
+ def import_before_promotion(index, relation, **import_options)
338
+ index.import_scope(relation, **import_options)
339
+ end
340
+
341
+ def reindex_records(object, mode: nil, refresh: false, **options)
342
+ mode ||= Searchkick.callbacks_value || @options[:callbacks] || :inline
343
+ mode = :inline if mode == :bulk
344
+
345
+ result = RecordIndexer.new(self).reindex(object, mode: mode, full: false, **options)
346
+ self.refresh if refresh
347
+ result
219
348
  end
220
349
 
221
350
  # https://gist.github.com/jarosan/3124884
222
351
  # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
223
- def reindex_scope(scope, import: true, resume: false, retain: false, async: false, refresh_interval: nil)
352
+ def full_reindex(relation, import: true, resume: false, retain: false, mode: nil, refresh_interval: nil, scope: nil, wait: nil)
353
+ raise ArgumentError, "wait only available in :async mode" if !wait.nil? && mode != :async
354
+
224
355
  if resume
225
356
  index_name = all_indices.sort.last
226
- raise Searchkick::Error, "No index to resume" unless index_name
227
- index = Searchkick::Index.new(index_name)
357
+ raise Error, "No index to resume" unless index_name
358
+ index = Index.new(index_name, @options)
228
359
  else
229
360
  clean_indices unless retain
230
361
 
231
- index_options = scope.searchkick_index_options
362
+ index_options = relation.searchkick_index_options
232
363
  index_options.deep_merge!(settings: {index: {refresh_interval: refresh_interval}}) if refresh_interval
233
364
  index = create_index(index_options: index_options)
234
365
  end
235
366
 
367
+ import_options = {
368
+ mode: (mode || :inline),
369
+ full: true,
370
+ resume: resume,
371
+ scope: scope
372
+ }
373
+
374
+ uuid = index.uuid
375
+
236
376
  # check if alias exists
237
377
  alias_exists = alias_exists?
238
378
  if alias_exists
239
- # import before promotion
240
- index.import_scope(scope, resume: resume, async: async, full: true) if import
379
+ import_before_promotion(index, relation, **import_options) if import
241
380
 
242
381
  # get existing indices to remove
243
- unless async
382
+ unless mode == :async
383
+ check_uuid(uuid, index.uuid)
244
384
  promote(index.name, update_refresh_interval: !refresh_interval.nil?)
245
385
  clean_indices unless retain
246
386
  end
@@ -249,11 +389,11 @@ module Searchkick
249
389
  promote(index.name, update_refresh_interval: !refresh_interval.nil?)
250
390
 
251
391
  # import after promotion
252
- index.import_scope(scope, resume: resume, async: async, full: true) if import
392
+ index.import_scope(relation, **import_options) if import
253
393
  end
254
394
 
255
- if async
256
- if async.is_a?(Hash) && async[:wait]
395
+ if mode == :async
396
+ if wait
257
397
  puts "Created index: #{index.name}"
258
398
  puts "Jobs queued. Waiting..."
259
399
  loop do
@@ -265,6 +405,7 @@ module Searchkick
265
405
  # already promoted if alias didn't exist
266
406
  if alias_exists
267
407
  puts "Jobs complete. Promoting..."
408
+ check_uuid(uuid, index.uuid)
268
409
  promote(index.name, update_refresh_interval: !refresh_interval.nil?)
269
410
  end
270
411
  clean_indices unless retain
@@ -276,267 +417,51 @@ module Searchkick
276
417
  index.refresh
277
418
  true
278
419
  end
279
- end
280
-
281
- def import_scope(scope, resume: false, method_name: nil, async: false, batch: false, batch_id: nil, full: false)
282
- # use scope for import
283
- scope = scope.search_import if scope.respond_to?(:search_import)
284
-
285
- if batch
286
- import_or_update scope.to_a, method_name, async
287
- Searchkick.with_redis { |r| r.srem(batches_key, batch_id) } if batch_id
288
- elsif full && async
289
- full_reindex_async(scope)
290
- elsif scope.respond_to?(:find_in_batches)
291
- if resume
292
- # use total docs instead of max id since there's not a great way
293
- # to get the max _id without scripting since it's a string
294
-
295
- # TODO use primary key and prefix with table name
296
- scope = scope.where("id > ?", total_docs)
297
- end
298
-
299
- scope = scope.select("id").except(:includes, :preload) if async
300
-
301
- scope.find_in_batches batch_size: batch_size do |items|
302
- import_or_update items, method_name, async
303
- end
304
- else
305
- each_batch(scope) do |items|
306
- import_or_update items, method_name, async
307
- end
308
- end
309
- end
310
-
311
- def batches_left
312
- Searchkick.with_redis { |r| r.scard(batches_key) }
313
- end
314
-
315
- # other
316
-
317
- def tokens(text, options = {})
318
- client.indices.analyze(body: {text: text}.merge(options), index: name)["tokens"].map { |t| t["token"] }
319
- end
320
-
321
- def klass_document_type(klass)
322
- @klass_document_type[klass] ||= begin
323
- if klass.respond_to?(:document_type)
324
- klass.document_type
325
- else
326
- klass.model_name.to_s.underscore
327
- end
420
+ rescue => e
421
+ if Searchkick.transport_error?(e) && (e.message.include?("No handler for type [text]") || e.message.include?("class java.util.ArrayList cannot be cast to class java.util.Map"))
422
+ raise UnsupportedVersionError
328
423
  end
329
- end
330
-
331
- protected
332
424
 
333
- def client
334
- Searchkick.client
425
+ raise e
335
426
  end
336
427
 
337
- def document_type(record)
338
- if record.respond_to?(:search_document_type)
339
- record.search_document_type
340
- else
341
- klass_document_type(record.class)
428
+ # safety check
429
+ # still a chance for race condition since its called before promotion
430
+ # ideal is for user to disable automatic index creation
431
+ # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation
432
+ def check_uuid(old_uuid, new_uuid)
433
+ if old_uuid != new_uuid
434
+ raise Error, "Safety check failed - only run one Model.reindex per model at a time"
342
435
  end
343
436
  end
344
437
 
345
- def search_id(record)
346
- id = record.respond_to?(:search_document_id) ? record.search_document_id : record.id
347
- id.is_a?(Numeric) ? id : id.to_s
348
- end
349
-
350
- EXCLUDED_ATTRIBUTES = ["_id", "_type"]
351
-
352
- def search_data(record, method_name = nil)
353
- partial_reindex = !method_name.nil?
354
- options = record.class.searchkick_options
355
-
356
- # remove _id since search_id is used instead
357
- source = record.send(method_name || :search_data).each_with_object({}) { |(k, v), memo| memo[k.to_s] = v; memo }.except(*EXCLUDED_ATTRIBUTES)
358
-
359
- # conversions
360
- if options[:conversions]
361
- Array(options[:conversions]).map(&:to_s).each do |conversions_field|
362
- if source[conversions_field]
363
- source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
364
- end
365
- end
366
- end
367
-
368
- # hack to prevent generator field doesn't exist error
369
- if options[:suggest]
370
- options[:suggest].map(&:to_s).each do |field|
371
- source[field] = nil if !source[field] && !partial_reindex
372
- end
373
- end
374
-
375
- # locations
376
- if options[:locations]
377
- options[:locations].map(&:to_s).each do |field|
378
- if source[field]
379
- if !source[field].is_a?(Hash) && (source[field].first.is_a?(Array) || source[field].first.is_a?(Hash))
380
- # multiple locations
381
- source[field] = source[field].map { |a| location_value(a) }
382
- else
383
- source[field] = location_value(source[field])
384
- end
385
- end
386
- end
387
- end
388
-
389
- cast_big_decimal(source)
390
-
391
- source
392
- end
393
-
394
- def location_value(value)
395
- if value.is_a?(Array)
396
- value.map(&:to_f).reverse
397
- elsif value.is_a?(Hash)
398
- {lat: value[:lat].to_f, lon: value[:lon].to_f}
399
- else
400
- value
401
- end
402
- end
403
-
404
- # change all BigDecimal values to floats due to
405
- # https://github.com/rails/rails/issues/6033
406
- # possible loss of precision :/
407
- def cast_big_decimal(obj)
408
- case obj
409
- when BigDecimal
410
- obj.to_f
411
- when Hash
412
- obj.each do |k, v|
413
- obj[k] = cast_big_decimal(v)
414
- end
415
- when Enumerable
416
- obj.map do |v|
417
- cast_big_decimal(v)
418
- end
419
- else
420
- obj
421
- end
422
- end
423
-
424
- def import_or_update(records, method_name, async)
425
- if records.any?
426
- if async
427
- Searchkick::BulkReindexJob.perform_later(
428
- class_name: records.first.class.name,
429
- record_ids: records.map(&:id),
430
- index_name: name,
431
- method_name: method_name ? method_name.to_s : nil
432
- )
433
- else
434
- records = records.select(&:should_index?)
435
- if records.any?
436
- with_retries do
437
- method_name ? bulk_update(records, method_name) : import(records)
438
- end
439
- end
440
- end
441
- end
442
- end
443
-
444
- def full_reindex_async(scope)
445
- if scope.respond_to?(:primary_key)
446
- # TODO expire Redis key
447
- primary_key = scope.primary_key
448
-
449
- starting_id =
450
- begin
451
- scope.minimum(primary_key)
452
- rescue ActiveRecord::StatementInvalid
453
- false
454
- end
455
-
456
- if starting_id.nil?
457
- # no records, do nothing
458
- elsif starting_id.is_a?(Numeric)
459
- max_id = scope.maximum(primary_key)
460
- batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil
461
-
462
- batches_count.times do |i|
463
- batch_id = i + 1
464
- min_id = starting_id + (i * batch_size)
465
- bulk_reindex_job scope, batch_id, min_id: min_id, max_id: min_id + batch_size - 1
466
- end
467
- else
468
- scope.find_in_batches(batch_size: batch_size).each_with_index do |batch, i|
469
- batch_id = i + 1
470
-
471
- bulk_reindex_job scope, batch_id, record_ids: batch.map { |record| record.id.to_s }
472
- end
473
- end
438
+ def notify(record, name)
439
+ if Searchkick.callbacks_value == :bulk
440
+ yield
474
441
  else
475
- batch_id = 1
476
- # TODO remove any eager loading
477
- scope = scope.only(:_id) if scope.respond_to?(:only)
478
- each_batch(scope) do |items|
479
- bulk_reindex_job scope, batch_id, record_ids: items.map { |i| i.id.to_s }
480
- batch_id += 1
481
- end
482
- end
483
- end
484
-
485
- def each_batch(scope)
486
- # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
487
- # use cursor for Mongoid
488
- items = []
489
- scope.all.each do |item|
490
- items << item
491
- if items.length == batch_size
492
- yield items
493
- items = []
442
+ name = "#{record.class.searchkick_klass.name} #{name}" if record && record.class.searchkick_klass
443
+ event = {
444
+ name: name,
445
+ id: search_id(record)
446
+ }
447
+ ActiveSupport::Notifications.instrument("request.searchkick", event) do
448
+ yield
494
449
  end
495
450
  end
496
- yield items if items.any?
497
451
  end
498
452
 
499
- def bulk_reindex_job(scope, batch_id, options)
500
- Searchkick::BulkReindexJob.perform_later({
501
- class_name: scope.model_name.name,
502
- index_name: name,
503
- batch_id: batch_id
504
- }.merge(options))
505
- Searchkick.with_redis { |r| r.sadd(batches_key, batch_id) }
506
- end
507
-
508
- def batch_size
509
- @batch_size ||= @options[:batch_size] || 1000
510
- end
511
-
512
- def with_retries
513
- retries = 0
514
-
515
- begin
453
+ def notify_bulk(records, name)
454
+ if Searchkick.callbacks_value == :bulk
516
455
  yield
517
- rescue Faraday::ClientError => e
518
- if retries < 1
519
- retries += 1
520
- retry
456
+ else
457
+ event = {
458
+ name: "#{records.first.class.searchkick_klass.name} #{name}",
459
+ count: records.size
460
+ }
461
+ ActiveSupport::Notifications.instrument("request.searchkick", event) do
462
+ yield
521
463
  end
522
- raise e
523
464
  end
524
465
  end
525
-
526
- def bulk_index_helper(records)
527
- Searchkick.indexer.queue(records.map { |r| {index: record_data(r).merge(data: search_data(r))} })
528
- end
529
-
530
- def bulk_delete_helper(records)
531
- Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| {delete: record_data(r)} })
532
- end
533
-
534
- def bulk_update_helper(records, method_name)
535
- Searchkick.indexer.queue(records.map { |r| {update: record_data(r).merge(data: {doc: search_data(r, method_name)})} })
536
- end
537
-
538
- def batches_key
539
- "searchkick:reindex:#{name}:batches"
540
- end
541
466
  end
542
467
  end