searchkick 4.6.3 → 5.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +142 -2
- data/LICENSE.txt +1 -1
- data/README.md +424 -228
- data/lib/searchkick/bulk_reindex_job.rb +12 -8
- data/lib/searchkick/controller_runtime.rb +40 -0
- data/lib/searchkick/index.rb +152 -67
- data/lib/searchkick/index_cache.rb +30 -0
- data/lib/searchkick/index_options.rb +102 -68
- data/lib/searchkick/indexer.rb +15 -8
- data/lib/searchkick/log_subscriber.rb +57 -0
- data/lib/searchkick/middleware.rb +9 -2
- data/lib/searchkick/model.rb +50 -51
- data/lib/searchkick/process_batch_job.rb +9 -25
- data/lib/searchkick/process_queue_job.rb +3 -2
- data/lib/searchkick/query.rb +212 -69
- data/lib/searchkick/railtie.rb +1 -1
- data/lib/searchkick/record_data.rb +1 -1
- data/lib/searchkick/record_indexer.rb +136 -52
- data/lib/searchkick/reindex_queue.rb +36 -8
- data/lib/searchkick/reindex_v2_job.rb +10 -34
- data/lib/searchkick/relation.rb +247 -0
- data/lib/searchkick/relation_indexer.rb +155 -0
- data/lib/searchkick/reranking.rb +28 -0
- data/lib/searchkick/results.rb +29 -28
- data/lib/searchkick/script.rb +11 -0
- data/lib/searchkick/version.rb +1 -1
- data/lib/searchkick/where.rb +11 -0
- data/lib/searchkick.rb +197 -95
- data/lib/tasks/searchkick.rake +6 -3
- metadata +14 -32
- data/lib/searchkick/bulk_indexer.rb +0 -173
- data/lib/searchkick/logging.rb +0 -246
@@ -2,16 +2,20 @@ module Searchkick
|
|
2
2
|
class BulkReindexJob < ActiveJob::Base
|
3
3
|
queue_as { Searchkick.queue_name }
|
4
4
|
|
5
|
+
# TODO remove min_id and max_id in Searchkick 6
|
5
6
|
def perform(class_name:, record_ids: nil, index_name: nil, method_name: nil, batch_id: nil, min_id: nil, max_id: nil)
|
6
|
-
|
7
|
-
index =
|
7
|
+
model = Searchkick.load_model(class_name)
|
8
|
+
index = model.searchkick_index(name: index_name)
|
9
|
+
|
10
|
+
# legacy
|
8
11
|
record_ids ||= min_id..max_id
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
)
|
12
|
+
|
13
|
+
relation = Searchkick.scope(model)
|
14
|
+
relation = Searchkick.load_records(relation, record_ids)
|
15
|
+
relation = relation.search_import if relation.respond_to?(:search_import)
|
16
|
+
|
17
|
+
RecordIndexer.new(index).reindex(relation, mode: :inline, method_name: method_name, full: false)
|
18
|
+
RelationIndexer.new(index).batch_completed(batch_id) if batch_id
|
15
19
|
end
|
16
20
|
end
|
17
21
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# based on https://gist.github.com/mnutt/566725
|
2
|
+
module Searchkick
|
3
|
+
module ControllerRuntime
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
protected
|
7
|
+
|
8
|
+
attr_internal :searchkick_runtime
|
9
|
+
|
10
|
+
def process_action(action, *args)
|
11
|
+
# We also need to reset the runtime before each action
|
12
|
+
# because of queries in middleware or in cases we are streaming
|
13
|
+
# and it won't be cleaned up by the method below.
|
14
|
+
Searchkick::LogSubscriber.reset_runtime
|
15
|
+
super
|
16
|
+
end
|
17
|
+
|
18
|
+
def cleanup_view_runtime
|
19
|
+
searchkick_rt_before_render = Searchkick::LogSubscriber.reset_runtime
|
20
|
+
runtime = super
|
21
|
+
searchkick_rt_after_render = Searchkick::LogSubscriber.reset_runtime
|
22
|
+
self.searchkick_runtime = searchkick_rt_before_render + searchkick_rt_after_render
|
23
|
+
runtime - searchkick_rt_after_render
|
24
|
+
end
|
25
|
+
|
26
|
+
def append_info_to_payload(payload)
|
27
|
+
super
|
28
|
+
payload[:searchkick_runtime] = (searchkick_runtime || 0) + Searchkick::LogSubscriber.reset_runtime
|
29
|
+
end
|
30
|
+
|
31
|
+
module ClassMethods
|
32
|
+
def log_process_action(payload)
|
33
|
+
messages = super
|
34
|
+
runtime = payload[:searchkick_runtime]
|
35
|
+
messages << ("Searchkick: %.1fms" % runtime.to_f) if runtime.to_f > 0
|
36
|
+
messages
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/lib/searchkick/index.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
require "searchkick/index_options"
|
2
|
-
|
3
1
|
module Searchkick
|
4
2
|
class Index
|
5
3
|
attr_reader :name, :options
|
@@ -40,12 +38,15 @@ module Searchkick
|
|
40
38
|
client.indices.exists_alias name: name
|
41
39
|
end
|
42
40
|
|
41
|
+
# call to_h for consistent results between elasticsearch gem 7 and 8
|
42
|
+
# could do for all API calls, but just do for ones where return value is focus for now
|
43
43
|
def mapping
|
44
|
-
client.indices.get_mapping
|
44
|
+
client.indices.get_mapping(index: name).to_h
|
45
45
|
end
|
46
46
|
|
47
|
+
# call to_h for consistent results between elasticsearch gem 7 and 8
|
47
48
|
def settings
|
48
|
-
client.indices.get_settings
|
49
|
+
client.indices.get_settings(index: name).to_h
|
49
50
|
end
|
50
51
|
|
51
52
|
def refresh_interval
|
@@ -66,16 +67,17 @@ module Searchkick
|
|
66
67
|
index: name,
|
67
68
|
body: {
|
68
69
|
query: {match_all: {}},
|
69
|
-
size: 0
|
70
|
+
size: 0,
|
71
|
+
track_total_hits: true
|
70
72
|
}
|
71
73
|
)
|
72
74
|
|
73
|
-
|
75
|
+
Results.new(nil, response).total_count
|
74
76
|
end
|
75
77
|
|
76
78
|
def promote(new_name, update_refresh_interval: false)
|
77
79
|
if update_refresh_interval
|
78
|
-
new_index =
|
80
|
+
new_index = Index.new(new_name, @options)
|
79
81
|
settings = options[:settings] || {}
|
80
82
|
refresh_interval = (settings[:index] && settings[:index][:refresh_interval]) || "1s"
|
81
83
|
new_index.update_settings(index: {refresh_interval: refresh_interval})
|
@@ -97,7 +99,7 @@ module Searchkick
|
|
97
99
|
record_data = RecordData.new(self, record).record_data
|
98
100
|
|
99
101
|
# remove underscore
|
100
|
-
get_options =
|
102
|
+
get_options = record_data.to_h { |k, v| [k.to_s.delete_prefix("_").to_sym, v] }
|
101
103
|
|
102
104
|
client.get(get_options)["_source"]
|
103
105
|
end
|
@@ -122,37 +124,52 @@ module Searchkick
|
|
122
124
|
def clean_indices
|
123
125
|
indices = all_indices(unaliased: true)
|
124
126
|
indices.each do |index|
|
125
|
-
|
127
|
+
Index.new(index).delete
|
126
128
|
end
|
127
129
|
indices
|
128
130
|
end
|
129
131
|
|
130
|
-
# record based
|
131
|
-
# use helpers for notifications
|
132
|
-
|
133
132
|
def store(record)
|
134
|
-
|
133
|
+
notify(record, "Store") do
|
134
|
+
queue_index([record])
|
135
|
+
end
|
135
136
|
end
|
136
137
|
|
137
138
|
def remove(record)
|
138
|
-
|
139
|
+
notify(record, "Remove") do
|
140
|
+
queue_delete([record])
|
141
|
+
end
|
139
142
|
end
|
140
143
|
|
141
144
|
def update_record(record, method_name)
|
142
|
-
|
145
|
+
notify(record, "Update") do
|
146
|
+
queue_update([record], method_name)
|
147
|
+
end
|
143
148
|
end
|
144
149
|
|
145
150
|
def bulk_delete(records)
|
146
|
-
|
151
|
+
return if records.empty?
|
152
|
+
|
153
|
+
notify_bulk(records, "Delete") do
|
154
|
+
queue_delete(records)
|
155
|
+
end
|
147
156
|
end
|
148
157
|
|
149
158
|
def bulk_index(records)
|
150
|
-
|
159
|
+
return if records.empty?
|
160
|
+
|
161
|
+
notify_bulk(records, "Import") do
|
162
|
+
queue_index(records)
|
163
|
+
end
|
151
164
|
end
|
152
165
|
alias_method :import, :bulk_index
|
153
166
|
|
154
167
|
def bulk_update(records, method_name)
|
155
|
-
|
168
|
+
return if records.empty?
|
169
|
+
|
170
|
+
notify_bulk(records, "Update") do
|
171
|
+
queue_update(records, method_name)
|
172
|
+
end
|
156
173
|
end
|
157
174
|
|
158
175
|
def search_id(record)
|
@@ -163,20 +180,12 @@ module Searchkick
|
|
163
180
|
RecordData.new(self, record).document_type
|
164
181
|
end
|
165
182
|
|
166
|
-
# TODO use like: [{_index: ..., _id: ...}] in Searchkick 5
|
167
183
|
def similar_record(record, **options)
|
168
|
-
like_text = retrieve(record).to_hash
|
169
|
-
.keep_if { |k, _| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
|
170
|
-
.values.compact.join(" ")
|
171
|
-
|
172
|
-
options[:where] ||= {}
|
173
|
-
options[:where][:_id] ||= {}
|
174
|
-
options[:where][:_id][:not] = Array(options[:where][:_id][:not]) + [record.id.to_s]
|
175
184
|
options[:per_page] ||= 10
|
176
|
-
options[:similar] =
|
185
|
+
options[:similar] = [RecordData.new(self, record).record_data]
|
186
|
+
options[:models] ||= [record.class] unless options.key?(:model)
|
177
187
|
|
178
|
-
|
179
|
-
Searchkick.search(like_text, model: record.class, **options)
|
188
|
+
Searchkick.search("*", **options)
|
180
189
|
end
|
181
190
|
|
182
191
|
def reload_synonyms
|
@@ -186,8 +195,9 @@ module Searchkick
|
|
186
195
|
raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
|
187
196
|
begin
|
188
197
|
client.transport.perform_request("GET", "#{CGI.escape(name)}/_reload_search_analyzers")
|
189
|
-
rescue
|
190
|
-
raise Error, "Requires non-OSS version of Elasticsearch"
|
198
|
+
rescue => e
|
199
|
+
raise Error, "Requires non-OSS version of Elasticsearch" if Searchkick.not_allowed_error?(e)
|
200
|
+
raise e
|
191
201
|
end
|
192
202
|
end
|
193
203
|
end
|
@@ -195,54 +205,73 @@ module Searchkick
|
|
195
205
|
# queue
|
196
206
|
|
197
207
|
def reindex_queue
|
198
|
-
|
208
|
+
ReindexQueue.new(name)
|
199
209
|
end
|
200
210
|
|
201
211
|
# reindex
|
202
212
|
|
203
|
-
|
213
|
+
# note: this is designed to be used internally
|
214
|
+
# so it does not check object matches index class
|
215
|
+
def reindex(object, method_name: nil, full: false, **options)
|
216
|
+
if object.is_a?(Array)
|
217
|
+
# note: purposefully skip full
|
218
|
+
return reindex_records(object, method_name: method_name, **options)
|
219
|
+
end
|
220
|
+
|
221
|
+
if !object.respond_to?(:searchkick_klass)
|
222
|
+
raise Error, "Cannot reindex object"
|
223
|
+
end
|
224
|
+
|
225
|
+
scoped = Searchkick.relation?(object)
|
226
|
+
# call searchkick_klass for inheritance
|
227
|
+
relation = scoped ? object.all : Searchkick.scope(object.searchkick_klass).all
|
228
|
+
|
204
229
|
refresh = options.fetch(:refresh, !scoped)
|
205
230
|
options.delete(:refresh)
|
206
231
|
|
207
|
-
if method_name
|
208
|
-
|
209
|
-
|
232
|
+
if method_name || (scoped && !full)
|
233
|
+
mode = options.delete(:mode) || :inline
|
234
|
+
scope = options.delete(:scope)
|
235
|
+
raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
|
210
236
|
|
211
|
-
#
|
212
|
-
import_scope(relation, method_name: method_name, scope: scope)
|
213
|
-
self.refresh if refresh
|
214
|
-
true
|
215
|
-
elsif scoped && !full
|
216
|
-
# TODO throw ArgumentError
|
217
|
-
Searchkick.warn("unsupported keywords: #{options.keys.map(&:inspect).join(", ")}") if options.any?
|
218
|
-
|
219
|
-
# reindex association
|
220
|
-
import_scope(relation, scope: scope)
|
237
|
+
# import only
|
238
|
+
import_scope(relation, method_name: method_name, mode: mode, scope: scope)
|
221
239
|
self.refresh if refresh
|
222
240
|
true
|
223
241
|
else
|
224
|
-
|
225
|
-
|
242
|
+
async = options.delete(:async)
|
243
|
+
if async
|
244
|
+
if async.is_a?(Hash) && async[:wait]
|
245
|
+
# TODO warn in 5.1
|
246
|
+
# Searchkick.warn "async option is deprecated - use mode: :async, wait: true instead"
|
247
|
+
options[:wait] = true unless options.key?(:wait)
|
248
|
+
else
|
249
|
+
# TODO warn in 5.1
|
250
|
+
# Searchkick.warn "async option is deprecated - use mode: :async instead"
|
251
|
+
end
|
252
|
+
options[:mode] ||= :async
|
253
|
+
end
|
254
|
+
|
255
|
+
full_reindex(relation, **options)
|
226
256
|
end
|
227
257
|
end
|
228
258
|
|
229
259
|
def create_index(index_options: nil)
|
230
260
|
index_options ||= self.index_options
|
231
|
-
index =
|
261
|
+
index = Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
|
232
262
|
index.create(index_options)
|
233
263
|
index
|
234
264
|
end
|
235
265
|
|
236
266
|
def import_scope(relation, **options)
|
237
|
-
|
267
|
+
relation_indexer.reindex(relation, **options)
|
238
268
|
end
|
239
269
|
|
240
270
|
def batches_left
|
241
|
-
|
271
|
+
relation_indexer.batches_left
|
242
272
|
end
|
243
273
|
|
244
|
-
#
|
245
|
-
|
274
|
+
# private
|
246
275
|
def klass_document_type(klass, ignore_type = false)
|
247
276
|
@klass_document_type[[klass, ignore_type]] ||= begin
|
248
277
|
if !ignore_type && klass.searchkick_klass.searchkick_options[:_type]
|
@@ -255,7 +284,7 @@ module Searchkick
|
|
255
284
|
end
|
256
285
|
end
|
257
286
|
|
258
|
-
#
|
287
|
+
# private
|
259
288
|
def conversions_fields
|
260
289
|
@conversions_fields ||= begin
|
261
290
|
conversions = Array(options[:conversions])
|
@@ -263,10 +292,12 @@ module Searchkick
|
|
263
292
|
end
|
264
293
|
end
|
265
294
|
|
295
|
+
# private
|
266
296
|
def suggest_fields
|
267
297
|
@suggest_fields ||= Array(options[:suggest]).map(&:to_s)
|
268
298
|
end
|
269
299
|
|
300
|
+
# private
|
270
301
|
def locations_fields
|
271
302
|
@locations_fields ||= begin
|
272
303
|
locations = Array(options[:locations])
|
@@ -285,8 +316,20 @@ module Searchkick
|
|
285
316
|
Searchkick.client
|
286
317
|
end
|
287
318
|
|
288
|
-
def
|
289
|
-
|
319
|
+
def queue_index(records)
|
320
|
+
Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).index_data })
|
321
|
+
end
|
322
|
+
|
323
|
+
def queue_delete(records)
|
324
|
+
Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data })
|
325
|
+
end
|
326
|
+
|
327
|
+
def queue_update(records, method_name)
|
328
|
+
Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) })
|
329
|
+
end
|
330
|
+
|
331
|
+
def relation_indexer
|
332
|
+
@relation_indexer ||= RelationIndexer.new(self)
|
290
333
|
end
|
291
334
|
|
292
335
|
def index_settings
|
@@ -297,13 +340,26 @@ module Searchkick
|
|
297
340
|
index.import_scope(relation, **import_options)
|
298
341
|
end
|
299
342
|
|
343
|
+
def reindex_records(object, mode: nil, refresh: false, **options)
|
344
|
+
mode ||= Searchkick.callbacks_value || @options[:callbacks] || :inline
|
345
|
+
mode = :inline if mode == :bulk
|
346
|
+
|
347
|
+
result = RecordIndexer.new(self).reindex(object, mode: mode, full: false, **options)
|
348
|
+
self.refresh if refresh
|
349
|
+
result
|
350
|
+
end
|
351
|
+
|
300
352
|
# https://gist.github.com/jarosan/3124884
|
301
353
|
# http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
|
302
|
-
def
|
354
|
+
def full_reindex(relation, import: true, resume: false, retain: false, mode: nil, refresh_interval: nil, scope: nil, wait: nil)
|
355
|
+
raise ArgumentError, "wait only available in :async mode" if !wait.nil? && mode != :async
|
356
|
+
# TODO raise ArgumentError in Searchkick 6
|
357
|
+
Searchkick.warn("Full reindex does not support :queue mode - use :async mode instead") if mode == :queue
|
358
|
+
|
303
359
|
if resume
|
304
360
|
index_name = all_indices.sort.last
|
305
|
-
raise
|
306
|
-
index =
|
361
|
+
raise Error, "No index to resume" unless index_name
|
362
|
+
index = Index.new(index_name, @options)
|
307
363
|
else
|
308
364
|
clean_indices unless retain
|
309
365
|
|
@@ -313,9 +369,9 @@ module Searchkick
|
|
313
369
|
end
|
314
370
|
|
315
371
|
import_options = {
|
316
|
-
|
317
|
-
async: async,
|
372
|
+
mode: (mode || :inline),
|
318
373
|
full: true,
|
374
|
+
resume: resume,
|
319
375
|
scope: scope
|
320
376
|
}
|
321
377
|
|
@@ -327,7 +383,7 @@ module Searchkick
|
|
327
383
|
import_before_promotion(index, relation, **import_options) if import
|
328
384
|
|
329
385
|
# get existing indices to remove
|
330
|
-
unless async
|
386
|
+
unless mode == :async
|
331
387
|
check_uuid(uuid, index.uuid)
|
332
388
|
promote(index.name, update_refresh_interval: !refresh_interval.nil?)
|
333
389
|
clean_indices unless retain
|
@@ -340,8 +396,8 @@ module Searchkick
|
|
340
396
|
index.import_scope(relation, **import_options) if import
|
341
397
|
end
|
342
398
|
|
343
|
-
if async
|
344
|
-
if
|
399
|
+
if mode == :async
|
400
|
+
if wait
|
345
401
|
puts "Created index: #{index.name}"
|
346
402
|
puts "Jobs queued. Waiting..."
|
347
403
|
loop do
|
@@ -366,8 +422,8 @@ module Searchkick
|
|
366
422
|
true
|
367
423
|
end
|
368
424
|
rescue => e
|
369
|
-
if Searchkick.transport_error?(e) && e.message.include?("No handler for type [text]")
|
370
|
-
raise UnsupportedVersionError
|
425
|
+
if Searchkick.transport_error?(e) && (e.message.include?("No handler for type [text]") || e.message.include?("class java.util.ArrayList cannot be cast to class java.util.Map"))
|
426
|
+
raise UnsupportedVersionError
|
371
427
|
end
|
372
428
|
|
373
429
|
raise e
|
@@ -379,7 +435,36 @@ module Searchkick
|
|
379
435
|
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation
|
380
436
|
def check_uuid(old_uuid, new_uuid)
|
381
437
|
if old_uuid != new_uuid
|
382
|
-
raise
|
438
|
+
raise Error, "Safety check failed - only run one Model.reindex per model at a time"
|
439
|
+
end
|
440
|
+
end
|
441
|
+
|
442
|
+
def notify(record, name)
|
443
|
+
if Searchkick.callbacks_value == :bulk
|
444
|
+
yield
|
445
|
+
else
|
446
|
+
name = "#{record.class.searchkick_klass.name} #{name}" if record && record.class.searchkick_klass
|
447
|
+
event = {
|
448
|
+
name: name,
|
449
|
+
id: search_id(record)
|
450
|
+
}
|
451
|
+
ActiveSupport::Notifications.instrument("request.searchkick", event) do
|
452
|
+
yield
|
453
|
+
end
|
454
|
+
end
|
455
|
+
end
|
456
|
+
|
457
|
+
def notify_bulk(records, name)
|
458
|
+
if Searchkick.callbacks_value == :bulk
|
459
|
+
yield
|
460
|
+
else
|
461
|
+
event = {
|
462
|
+
name: "#{records.first.class.searchkick_klass.name} #{name}",
|
463
|
+
count: records.size
|
464
|
+
}
|
465
|
+
ActiveSupport::Notifications.instrument("request.searchkick", event) do
|
466
|
+
yield
|
467
|
+
end
|
383
468
|
end
|
384
469
|
end
|
385
470
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Searchkick
|
2
|
+
class IndexCache
|
3
|
+
def initialize(max_size: 20)
|
4
|
+
@data = {}
|
5
|
+
@mutex = Mutex.new
|
6
|
+
@max_size = max_size
|
7
|
+
end
|
8
|
+
|
9
|
+
# probably a better pattern for this
|
10
|
+
# but keep it simple
|
11
|
+
def fetch(name)
|
12
|
+
# thread-safe in MRI without mutex
|
13
|
+
# due to how context switching works
|
14
|
+
@mutex.synchronize do
|
15
|
+
if @data.key?(name)
|
16
|
+
@data[name]
|
17
|
+
else
|
18
|
+
@data.clear if @data.size >= @max_size
|
19
|
+
@data[name] = yield
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def clear
|
25
|
+
@mutex.synchronize do
|
26
|
+
@data.clear
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|