searchkick 4.6.3 → 5.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,16 +2,20 @@ module Searchkick
2
2
  class BulkReindexJob < ActiveJob::Base
3
3
  queue_as { Searchkick.queue_name }
4
4
 
5
+ # TODO remove min_id and max_id in Searchkick 6
5
6
  def perform(class_name:, record_ids: nil, index_name: nil, method_name: nil, batch_id: nil, min_id: nil, max_id: nil)
6
- klass = class_name.constantize
7
- index = index_name ? Searchkick::Index.new(index_name, **klass.searchkick_options) : klass.searchkick_index
7
+ model = Searchkick.load_model(class_name)
8
+ index = model.searchkick_index(name: index_name)
9
+
10
+ # legacy
8
11
  record_ids ||= min_id..max_id
9
- index.import_scope(
10
- Searchkick.load_records(klass, record_ids),
11
- method_name: method_name,
12
- batch: true,
13
- batch_id: batch_id
14
- )
12
+
13
+ relation = Searchkick.scope(model)
14
+ relation = Searchkick.load_records(relation, record_ids)
15
+ relation = relation.search_import if relation.respond_to?(:search_import)
16
+
17
+ RecordIndexer.new(index).reindex(relation, mode: :inline, method_name: method_name, full: false)
18
+ RelationIndexer.new(index).batch_completed(batch_id) if batch_id
15
19
  end
16
20
  end
17
21
  end
@@ -0,0 +1,40 @@
1
+ # based on https://gist.github.com/mnutt/566725
2
+ module Searchkick
3
+ module ControllerRuntime
4
+ extend ActiveSupport::Concern
5
+
6
+ protected
7
+
8
+ attr_internal :searchkick_runtime
9
+
10
+ def process_action(action, *args)
11
+ # We also need to reset the runtime before each action
12
+ # because of queries in middleware or in cases we are streaming
13
+ # and it won't be cleaned up by the method below.
14
+ Searchkick::LogSubscriber.reset_runtime
15
+ super
16
+ end
17
+
18
+ def cleanup_view_runtime
19
+ searchkick_rt_before_render = Searchkick::LogSubscriber.reset_runtime
20
+ runtime = super
21
+ searchkick_rt_after_render = Searchkick::LogSubscriber.reset_runtime
22
+ self.searchkick_runtime = searchkick_rt_before_render + searchkick_rt_after_render
23
+ runtime - searchkick_rt_after_render
24
+ end
25
+
26
+ def append_info_to_payload(payload)
27
+ super
28
+ payload[:searchkick_runtime] = (searchkick_runtime || 0) + Searchkick::LogSubscriber.reset_runtime
29
+ end
30
+
31
+ module ClassMethods
32
+ def log_process_action(payload)
33
+ messages = super
34
+ runtime = payload[:searchkick_runtime]
35
+ messages << ("Searchkick: %.1fms" % runtime.to_f) if runtime.to_f > 0
36
+ messages
37
+ end
38
+ end
39
+ end
40
+ end
@@ -1,5 +1,3 @@
1
- require "searchkick/index_options"
2
-
3
1
  module Searchkick
4
2
  class Index
5
3
  attr_reader :name, :options
@@ -40,12 +38,15 @@ module Searchkick
40
38
  client.indices.exists_alias name: name
41
39
  end
42
40
 
41
+ # call to_h for consistent results between elasticsearch gem 7 and 8
42
+ # could do for all API calls, but just do for ones where return value is focus for now
43
43
  def mapping
44
- client.indices.get_mapping index: name
44
+ client.indices.get_mapping(index: name).to_h
45
45
  end
46
46
 
47
+ # call to_h for consistent results between elasticsearch gem 7 and 8
47
48
  def settings
48
- client.indices.get_settings index: name
49
+ client.indices.get_settings(index: name).to_h
49
50
  end
50
51
 
51
52
  def refresh_interval
@@ -66,16 +67,17 @@ module Searchkick
66
67
  index: name,
67
68
  body: {
68
69
  query: {match_all: {}},
69
- size: 0
70
+ size: 0,
71
+ track_total_hits: true
70
72
  }
71
73
  )
72
74
 
73
- Searchkick::Results.new(nil, response).total_count
75
+ Results.new(nil, response).total_count
74
76
  end
75
77
 
76
78
  def promote(new_name, update_refresh_interval: false)
77
79
  if update_refresh_interval
78
- new_index = Searchkick::Index.new(new_name, @options)
80
+ new_index = Index.new(new_name, @options)
79
81
  settings = options[:settings] || {}
80
82
  refresh_interval = (settings[:index] && settings[:index][:refresh_interval]) || "1s"
81
83
  new_index.update_settings(index: {refresh_interval: refresh_interval})
@@ -97,7 +99,7 @@ module Searchkick
97
99
  record_data = RecordData.new(self, record).record_data
98
100
 
99
101
  # remove underscore
100
- get_options = Hash[record_data.map { |k, v| [k.to_s.sub(/\A_/, "").to_sym, v] }]
102
+ get_options = record_data.to_h { |k, v| [k.to_s.delete_prefix("_").to_sym, v] }
101
103
 
102
104
  client.get(get_options)["_source"]
103
105
  end
@@ -122,37 +124,52 @@ module Searchkick
122
124
  def clean_indices
123
125
  indices = all_indices(unaliased: true)
124
126
  indices.each do |index|
125
- Searchkick::Index.new(index).delete
127
+ Index.new(index).delete
126
128
  end
127
129
  indices
128
130
  end
129
131
 
130
- # record based
131
- # use helpers for notifications
132
-
133
132
  def store(record)
134
- bulk_indexer.bulk_index([record])
133
+ notify(record, "Store") do
134
+ queue_index([record])
135
+ end
135
136
  end
136
137
 
137
138
  def remove(record)
138
- bulk_indexer.bulk_delete([record])
139
+ notify(record, "Remove") do
140
+ queue_delete([record])
141
+ end
139
142
  end
140
143
 
141
144
  def update_record(record, method_name)
142
- bulk_indexer.bulk_update([record], method_name)
145
+ notify(record, "Update") do
146
+ queue_update([record], method_name)
147
+ end
143
148
  end
144
149
 
145
150
  def bulk_delete(records)
146
- bulk_indexer.bulk_delete(records)
151
+ return if records.empty?
152
+
153
+ notify_bulk(records, "Delete") do
154
+ queue_delete(records)
155
+ end
147
156
  end
148
157
 
149
158
  def bulk_index(records)
150
- bulk_indexer.bulk_index(records)
159
+ return if records.empty?
160
+
161
+ notify_bulk(records, "Import") do
162
+ queue_index(records)
163
+ end
151
164
  end
152
165
  alias_method :import, :bulk_index
153
166
 
154
167
  def bulk_update(records, method_name)
155
- bulk_indexer.bulk_update(records, method_name)
168
+ return if records.empty?
169
+
170
+ notify_bulk(records, "Update") do
171
+ queue_update(records, method_name)
172
+ end
156
173
  end
157
174
 
158
175
  def search_id(record)
@@ -163,20 +180,12 @@ module Searchkick
163
180
  RecordData.new(self, record).document_type
164
181
  end
165
182
 
166
- # TODO use like: [{_index: ..., _id: ...}] in Searchkick 5
167
183
  def similar_record(record, **options)
168
- like_text = retrieve(record).to_hash
169
- .keep_if { |k, _| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
170
- .values.compact.join(" ")
171
-
172
- options[:where] ||= {}
173
- options[:where][:_id] ||= {}
174
- options[:where][:_id][:not] = Array(options[:where][:_id][:not]) + [record.id.to_s]
175
184
  options[:per_page] ||= 10
176
- options[:similar] = true
185
+ options[:similar] = [RecordData.new(self, record).record_data]
186
+ options[:models] ||= [record.class] unless options.key?(:model)
177
187
 
178
- # TODO use index class instead of record class
179
- Searchkick.search(like_text, model: record.class, **options)
188
+ Searchkick.search("*", **options)
180
189
  end
181
190
 
182
191
  def reload_synonyms
@@ -186,8 +195,9 @@ module Searchkick
186
195
  raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
187
196
  begin
188
197
  client.transport.perform_request("GET", "#{CGI.escape(name)}/_reload_search_analyzers")
189
- rescue Elasticsearch::Transport::Transport::Errors::MethodNotAllowed
190
- raise Error, "Requires non-OSS version of Elasticsearch"
198
+ rescue => e
199
+ raise Error, "Requires non-OSS version of Elasticsearch" if Searchkick.not_allowed_error?(e)
200
+ raise e
191
201
  end
192
202
  end
193
203
  end
@@ -195,54 +205,73 @@ module Searchkick
195
205
  # queue
196
206
 
197
207
  def reindex_queue
198
- Searchkick::ReindexQueue.new(name)
208
+ ReindexQueue.new(name)
199
209
  end
200
210
 
201
211
  # reindex
202
212
 
203
- def reindex(relation, method_name, scoped:, full: false, scope: nil, **options)
213
+ # note: this is designed to be used internally
214
+ # so it does not check object matches index class
215
+ def reindex(object, method_name: nil, full: false, **options)
216
+ if object.is_a?(Array)
217
+ # note: purposefully skip full
218
+ return reindex_records(object, method_name: method_name, **options)
219
+ end
220
+
221
+ if !object.respond_to?(:searchkick_klass)
222
+ raise Error, "Cannot reindex object"
223
+ end
224
+
225
+ scoped = Searchkick.relation?(object)
226
+ # call searchkick_klass for inheritance
227
+ relation = scoped ? object.all : Searchkick.scope(object.searchkick_klass).all
228
+
204
229
  refresh = options.fetch(:refresh, !scoped)
205
230
  options.delete(:refresh)
206
231
 
207
- if method_name
208
- # TODO throw ArgumentError
209
- Searchkick.warn("unsupported keywords: #{options.keys.map(&:inspect).join(", ")}") if options.any?
232
+ if method_name || (scoped && !full)
233
+ mode = options.delete(:mode) || :inline
234
+ scope = options.delete(:scope)
235
+ raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
210
236
 
211
- # update
212
- import_scope(relation, method_name: method_name, scope: scope)
213
- self.refresh if refresh
214
- true
215
- elsif scoped && !full
216
- # TODO throw ArgumentError
217
- Searchkick.warn("unsupported keywords: #{options.keys.map(&:inspect).join(", ")}") if options.any?
218
-
219
- # reindex association
220
- import_scope(relation, scope: scope)
237
+ # import only
238
+ import_scope(relation, method_name: method_name, mode: mode, scope: scope)
221
239
  self.refresh if refresh
222
240
  true
223
241
  else
224
- # full reindex
225
- reindex_scope(relation, scope: scope, **options)
242
+ async = options.delete(:async)
243
+ if async
244
+ if async.is_a?(Hash) && async[:wait]
245
+ # TODO warn in 5.1
246
+ # Searchkick.warn "async option is deprecated - use mode: :async, wait: true instead"
247
+ options[:wait] = true unless options.key?(:wait)
248
+ else
249
+ # TODO warn in 5.1
250
+ # Searchkick.warn "async option is deprecated - use mode: :async instead"
251
+ end
252
+ options[:mode] ||= :async
253
+ end
254
+
255
+ full_reindex(relation, **options)
226
256
  end
227
257
  end
228
258
 
229
259
  def create_index(index_options: nil)
230
260
  index_options ||= self.index_options
231
- index = Searchkick::Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
261
+ index = Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
232
262
  index.create(index_options)
233
263
  index
234
264
  end
235
265
 
236
266
  def import_scope(relation, **options)
237
- bulk_indexer.import_scope(relation, **options)
267
+ relation_indexer.reindex(relation, **options)
238
268
  end
239
269
 
240
270
  def batches_left
241
- bulk_indexer.batches_left
271
+ relation_indexer.batches_left
242
272
  end
243
273
 
244
- # other
245
-
274
+ # private
246
275
  def klass_document_type(klass, ignore_type = false)
247
276
  @klass_document_type[[klass, ignore_type]] ||= begin
248
277
  if !ignore_type && klass.searchkick_klass.searchkick_options[:_type]
@@ -255,7 +284,7 @@ module Searchkick
255
284
  end
256
285
  end
257
286
 
258
- # should not be public
287
+ # private
259
288
  def conversions_fields
260
289
  @conversions_fields ||= begin
261
290
  conversions = Array(options[:conversions])
@@ -263,10 +292,12 @@ module Searchkick
263
292
  end
264
293
  end
265
294
 
295
+ # private
266
296
  def suggest_fields
267
297
  @suggest_fields ||= Array(options[:suggest]).map(&:to_s)
268
298
  end
269
299
 
300
+ # private
270
301
  def locations_fields
271
302
  @locations_fields ||= begin
272
303
  locations = Array(options[:locations])
@@ -285,8 +316,20 @@ module Searchkick
285
316
  Searchkick.client
286
317
  end
287
318
 
288
- def bulk_indexer
289
- @bulk_indexer ||= BulkIndexer.new(self)
319
+ def queue_index(records)
320
+ Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).index_data })
321
+ end
322
+
323
+ def queue_delete(records)
324
+ Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data })
325
+ end
326
+
327
+ def queue_update(records, method_name)
328
+ Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) })
329
+ end
330
+
331
+ def relation_indexer
332
+ @relation_indexer ||= RelationIndexer.new(self)
290
333
  end
291
334
 
292
335
  def index_settings
@@ -297,13 +340,26 @@ module Searchkick
297
340
  index.import_scope(relation, **import_options)
298
341
  end
299
342
 
343
+ def reindex_records(object, mode: nil, refresh: false, **options)
344
+ mode ||= Searchkick.callbacks_value || @options[:callbacks] || :inline
345
+ mode = :inline if mode == :bulk
346
+
347
+ result = RecordIndexer.new(self).reindex(object, mode: mode, full: false, **options)
348
+ self.refresh if refresh
349
+ result
350
+ end
351
+
300
352
  # https://gist.github.com/jarosan/3124884
301
353
  # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
302
- def reindex_scope(relation, import: true, resume: false, retain: false, async: false, refresh_interval: nil, scope: nil)
354
+ def full_reindex(relation, import: true, resume: false, retain: false, mode: nil, refresh_interval: nil, scope: nil, wait: nil)
355
+ raise ArgumentError, "wait only available in :async mode" if !wait.nil? && mode != :async
356
+ # TODO raise ArgumentError in Searchkick 6
357
+ Searchkick.warn("Full reindex does not support :queue mode - use :async mode instead") if mode == :queue
358
+
303
359
  if resume
304
360
  index_name = all_indices.sort.last
305
- raise Searchkick::Error, "No index to resume" unless index_name
306
- index = Searchkick::Index.new(index_name, @options)
361
+ raise Error, "No index to resume" unless index_name
362
+ index = Index.new(index_name, @options)
307
363
  else
308
364
  clean_indices unless retain
309
365
 
@@ -313,9 +369,9 @@ module Searchkick
313
369
  end
314
370
 
315
371
  import_options = {
316
- resume: resume,
317
- async: async,
372
+ mode: (mode || :inline),
318
373
  full: true,
374
+ resume: resume,
319
375
  scope: scope
320
376
  }
321
377
 
@@ -327,7 +383,7 @@ module Searchkick
327
383
  import_before_promotion(index, relation, **import_options) if import
328
384
 
329
385
  # get existing indices to remove
330
- unless async
386
+ unless mode == :async
331
387
  check_uuid(uuid, index.uuid)
332
388
  promote(index.name, update_refresh_interval: !refresh_interval.nil?)
333
389
  clean_indices unless retain
@@ -340,8 +396,8 @@ module Searchkick
340
396
  index.import_scope(relation, **import_options) if import
341
397
  end
342
398
 
343
- if async
344
- if async.is_a?(Hash) && async[:wait]
399
+ if mode == :async
400
+ if wait
345
401
  puts "Created index: #{index.name}"
346
402
  puts "Jobs queued. Waiting..."
347
403
  loop do
@@ -366,8 +422,8 @@ module Searchkick
366
422
  true
367
423
  end
368
424
  rescue => e
369
- if Searchkick.transport_error?(e) && e.message.include?("No handler for type [text]")
370
- raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 6 or greater"
425
+ if Searchkick.transport_error?(e) && (e.message.include?("No handler for type [text]") || e.message.include?("class java.util.ArrayList cannot be cast to class java.util.Map"))
426
+ raise UnsupportedVersionError
371
427
  end
372
428
 
373
429
  raise e
@@ -379,7 +435,36 @@ module Searchkick
379
435
  # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation
380
436
  def check_uuid(old_uuid, new_uuid)
381
437
  if old_uuid != new_uuid
382
- raise Searchkick::Error, "Safety check failed - only run one Model.reindex per model at a time"
438
+ raise Error, "Safety check failed - only run one Model.reindex per model at a time"
439
+ end
440
+ end
441
+
442
+ def notify(record, name)
443
+ if Searchkick.callbacks_value == :bulk
444
+ yield
445
+ else
446
+ name = "#{record.class.searchkick_klass.name} #{name}" if record && record.class.searchkick_klass
447
+ event = {
448
+ name: name,
449
+ id: search_id(record)
450
+ }
451
+ ActiveSupport::Notifications.instrument("request.searchkick", event) do
452
+ yield
453
+ end
454
+ end
455
+ end
456
+
457
+ def notify_bulk(records, name)
458
+ if Searchkick.callbacks_value == :bulk
459
+ yield
460
+ else
461
+ event = {
462
+ name: "#{records.first.class.searchkick_klass.name} #{name}",
463
+ count: records.size
464
+ }
465
+ ActiveSupport::Notifications.instrument("request.searchkick", event) do
466
+ yield
467
+ end
383
468
  end
384
469
  end
385
470
  end
@@ -0,0 +1,30 @@
1
+ module Searchkick
2
+ class IndexCache
3
+ def initialize(max_size: 20)
4
+ @data = {}
5
+ @mutex = Mutex.new
6
+ @max_size = max_size
7
+ end
8
+
9
+ # probably a better pattern for this
10
+ # but keep it simple
11
+ def fetch(name)
12
+ # thread-safe in MRI without mutex
13
+ # due to how context switching works
14
+ @mutex.synchronize do
15
+ if @data.key?(name)
16
+ @data[name]
17
+ else
18
+ @data.clear if @data.size >= @max_size
19
+ @data[name] = yield
20
+ end
21
+ end
22
+ end
23
+
24
+ def clear
25
+ @mutex.synchronize do
26
+ @data.clear
27
+ end
28
+ end
29
+ end
30
+ end