searchkick 4.6.3 → 5.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -1
- data/README.md +187 -124
- data/lib/searchkick/bulk_reindex_job.rb +12 -8
- data/lib/searchkick/controller_runtime.rb +40 -0
- data/lib/searchkick/index.rb +146 -65
- data/lib/searchkick/index_cache.rb +30 -0
- data/lib/searchkick/index_options.rb +17 -67
- data/lib/searchkick/indexer.rb +15 -8
- data/lib/searchkick/log_subscriber.rb +57 -0
- data/lib/searchkick/middleware.rb +1 -1
- data/lib/searchkick/model.rb +48 -49
- data/lib/searchkick/process_batch_job.rb +9 -25
- data/lib/searchkick/process_queue_job.rb +3 -2
- data/lib/searchkick/query.rb +38 -54
- data/lib/searchkick/record_data.rb +1 -1
- data/lib/searchkick/record_indexer.rb +136 -52
- data/lib/searchkick/reindex_queue.rb +26 -3
- data/lib/searchkick/reindex_v2_job.rb +10 -34
- data/lib/searchkick/relation.rb +112 -0
- data/lib/searchkick/relation_indexer.rb +150 -0
- data/lib/searchkick/results.rb +27 -28
- data/lib/searchkick/version.rb +1 -1
- data/lib/searchkick.rb +159 -84
- data/lib/tasks/searchkick.rake +6 -3
- metadata +11 -28
- data/lib/searchkick/bulk_indexer.rb +0 -173
- data/lib/searchkick/logging.rb +0 -246
@@ -2,16 +2,20 @@ module Searchkick
|
|
2
2
|
class BulkReindexJob < ActiveJob::Base
|
3
3
|
queue_as { Searchkick.queue_name }
|
4
4
|
|
5
|
+
# TODO remove min_id and max_id in Searchkick 6
|
5
6
|
def perform(class_name:, record_ids: nil, index_name: nil, method_name: nil, batch_id: nil, min_id: nil, max_id: nil)
|
6
|
-
|
7
|
-
index =
|
7
|
+
model = Searchkick.load_model(class_name)
|
8
|
+
index = model.searchkick_index(name: index_name)
|
9
|
+
|
10
|
+
# legacy
|
8
11
|
record_ids ||= min_id..max_id
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
)
|
12
|
+
|
13
|
+
relation = Searchkick.scope(model)
|
14
|
+
relation = Searchkick.load_records(relation, record_ids)
|
15
|
+
relation = relation.search_import if relation.respond_to?(:search_import)
|
16
|
+
|
17
|
+
RecordIndexer.new(index).reindex(relation, mode: :inline, method_name: method_name, full: false)
|
18
|
+
RelationIndexer.new(index).batch_completed(batch_id) if batch_id
|
15
19
|
end
|
16
20
|
end
|
17
21
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# based on https://gist.github.com/mnutt/566725
|
2
|
+
module Searchkick
|
3
|
+
module ControllerRuntime
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
protected
|
7
|
+
|
8
|
+
attr_internal :searchkick_runtime
|
9
|
+
|
10
|
+
def process_action(action, *args)
|
11
|
+
# We also need to reset the runtime before each action
|
12
|
+
# because of queries in middleware or in cases we are streaming
|
13
|
+
# and it won't be cleaned up by the method below.
|
14
|
+
Searchkick::LogSubscriber.reset_runtime
|
15
|
+
super
|
16
|
+
end
|
17
|
+
|
18
|
+
def cleanup_view_runtime
|
19
|
+
searchkick_rt_before_render = Searchkick::LogSubscriber.reset_runtime
|
20
|
+
runtime = super
|
21
|
+
searchkick_rt_after_render = Searchkick::LogSubscriber.reset_runtime
|
22
|
+
self.searchkick_runtime = searchkick_rt_before_render + searchkick_rt_after_render
|
23
|
+
runtime - searchkick_rt_after_render
|
24
|
+
end
|
25
|
+
|
26
|
+
def append_info_to_payload(payload)
|
27
|
+
super
|
28
|
+
payload[:searchkick_runtime] = (searchkick_runtime || 0) + Searchkick::LogSubscriber.reset_runtime
|
29
|
+
end
|
30
|
+
|
31
|
+
module ClassMethods
|
32
|
+
def log_process_action(payload)
|
33
|
+
messages = super
|
34
|
+
runtime = payload[:searchkick_runtime]
|
35
|
+
messages << ("Searchkick: %.1fms" % runtime.to_f) if runtime.to_f > 0
|
36
|
+
messages
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/lib/searchkick/index.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
require "searchkick/index_options"
|
2
|
-
|
3
1
|
module Searchkick
|
4
2
|
class Index
|
5
3
|
attr_reader :name, :options
|
@@ -40,12 +38,15 @@ module Searchkick
|
|
40
38
|
client.indices.exists_alias name: name
|
41
39
|
end
|
42
40
|
|
41
|
+
# call to_h for consistent results between elasticsearch gem 7 and 8
|
42
|
+
# could do for all API calls, but just do for ones where return value is focus for now
|
43
43
|
def mapping
|
44
|
-
client.indices.get_mapping
|
44
|
+
client.indices.get_mapping(index: name).to_h
|
45
45
|
end
|
46
46
|
|
47
|
+
# call to_h for consistent results between elasticsearch gem 7 and 8
|
47
48
|
def settings
|
48
|
-
client.indices.get_settings
|
49
|
+
client.indices.get_settings(index: name).to_h
|
49
50
|
end
|
50
51
|
|
51
52
|
def refresh_interval
|
@@ -70,12 +71,12 @@ module Searchkick
|
|
70
71
|
}
|
71
72
|
)
|
72
73
|
|
73
|
-
|
74
|
+
Results.new(nil, response).total_count
|
74
75
|
end
|
75
76
|
|
76
77
|
def promote(new_name, update_refresh_interval: false)
|
77
78
|
if update_refresh_interval
|
78
|
-
new_index =
|
79
|
+
new_index = Index.new(new_name, @options)
|
79
80
|
settings = options[:settings] || {}
|
80
81
|
refresh_interval = (settings[:index] && settings[:index][:refresh_interval]) || "1s"
|
81
82
|
new_index.update_settings(index: {refresh_interval: refresh_interval})
|
@@ -97,7 +98,7 @@ module Searchkick
|
|
97
98
|
record_data = RecordData.new(self, record).record_data
|
98
99
|
|
99
100
|
# remove underscore
|
100
|
-
get_options =
|
101
|
+
get_options = record_data.to_h { |k, v| [k.to_s.sub(/\A_/, "").to_sym, v] }
|
101
102
|
|
102
103
|
client.get(get_options)["_source"]
|
103
104
|
end
|
@@ -122,37 +123,52 @@ module Searchkick
|
|
122
123
|
def clean_indices
|
123
124
|
indices = all_indices(unaliased: true)
|
124
125
|
indices.each do |index|
|
125
|
-
|
126
|
+
Index.new(index).delete
|
126
127
|
end
|
127
128
|
indices
|
128
129
|
end
|
129
130
|
|
130
|
-
# record based
|
131
|
-
# use helpers for notifications
|
132
|
-
|
133
131
|
def store(record)
|
134
|
-
|
132
|
+
notify(record, "Store") do
|
133
|
+
queue_index([record])
|
134
|
+
end
|
135
135
|
end
|
136
136
|
|
137
137
|
def remove(record)
|
138
|
-
|
138
|
+
notify(record, "Remove") do
|
139
|
+
queue_delete([record])
|
140
|
+
end
|
139
141
|
end
|
140
142
|
|
141
143
|
def update_record(record, method_name)
|
142
|
-
|
144
|
+
notify(record, "Update") do
|
145
|
+
queue_update([record], method_name)
|
146
|
+
end
|
143
147
|
end
|
144
148
|
|
145
149
|
def bulk_delete(records)
|
146
|
-
|
150
|
+
return if records.empty?
|
151
|
+
|
152
|
+
notify_bulk(records, "Delete") do
|
153
|
+
queue_delete(records)
|
154
|
+
end
|
147
155
|
end
|
148
156
|
|
149
157
|
def bulk_index(records)
|
150
|
-
|
158
|
+
return if records.empty?
|
159
|
+
|
160
|
+
notify_bulk(records, "Import") do
|
161
|
+
queue_index(records)
|
162
|
+
end
|
151
163
|
end
|
152
164
|
alias_method :import, :bulk_index
|
153
165
|
|
154
166
|
def bulk_update(records, method_name)
|
155
|
-
|
167
|
+
return if records.empty?
|
168
|
+
|
169
|
+
notify_bulk(records, "Update") do
|
170
|
+
queue_update(records, method_name)
|
171
|
+
end
|
156
172
|
end
|
157
173
|
|
158
174
|
def search_id(record)
|
@@ -163,20 +179,12 @@ module Searchkick
|
|
163
179
|
RecordData.new(self, record).document_type
|
164
180
|
end
|
165
181
|
|
166
|
-
# TODO use like: [{_index: ..., _id: ...}] in Searchkick 5
|
167
182
|
def similar_record(record, **options)
|
168
|
-
like_text = retrieve(record).to_hash
|
169
|
-
.keep_if { |k, _| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
|
170
|
-
.values.compact.join(" ")
|
171
|
-
|
172
|
-
options[:where] ||= {}
|
173
|
-
options[:where][:_id] ||= {}
|
174
|
-
options[:where][:_id][:not] = Array(options[:where][:_id][:not]) + [record.id.to_s]
|
175
183
|
options[:per_page] ||= 10
|
176
|
-
options[:similar] =
|
184
|
+
options[:similar] = [RecordData.new(self, record).record_data]
|
185
|
+
options[:models] ||= [record.class] unless options.key?(:model)
|
177
186
|
|
178
|
-
|
179
|
-
Searchkick.search(like_text, model: record.class, **options)
|
187
|
+
Searchkick.search("*", **options)
|
180
188
|
end
|
181
189
|
|
182
190
|
def reload_synonyms
|
@@ -186,8 +194,9 @@ module Searchkick
|
|
186
194
|
raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
|
187
195
|
begin
|
188
196
|
client.transport.perform_request("GET", "#{CGI.escape(name)}/_reload_search_analyzers")
|
189
|
-
rescue
|
190
|
-
raise Error, "Requires non-OSS version of Elasticsearch"
|
197
|
+
rescue => e
|
198
|
+
raise Error, "Requires non-OSS version of Elasticsearch" if Searchkick.not_allowed_error?(e)
|
199
|
+
raise e
|
191
200
|
end
|
192
201
|
end
|
193
202
|
end
|
@@ -195,54 +204,72 @@ module Searchkick
|
|
195
204
|
# queue
|
196
205
|
|
197
206
|
def reindex_queue
|
198
|
-
|
207
|
+
ReindexQueue.new(name)
|
199
208
|
end
|
200
209
|
|
201
210
|
# reindex
|
202
211
|
|
203
|
-
|
212
|
+
# note: this is designed to be used internally
|
213
|
+
# so it does not check object matches index class
|
214
|
+
def reindex(object, method_name: nil, full: false, **options)
|
215
|
+
if object.is_a?(Array)
|
216
|
+
# note: purposefully skip full
|
217
|
+
return reindex_records(object, method_name: method_name, **options)
|
218
|
+
end
|
219
|
+
|
220
|
+
if !object.respond_to?(:searchkick_klass)
|
221
|
+
raise Error, "Cannot reindex object"
|
222
|
+
end
|
223
|
+
|
224
|
+
scoped = Searchkick.relation?(object)
|
225
|
+
# call searchkick_klass for inheritance
|
226
|
+
relation = scoped ? object.all : Searchkick.scope(object.searchkick_klass).all
|
227
|
+
|
204
228
|
refresh = options.fetch(:refresh, !scoped)
|
205
229
|
options.delete(:refresh)
|
206
230
|
|
207
|
-
if method_name
|
208
|
-
|
209
|
-
|
231
|
+
if method_name || (scoped && !full)
|
232
|
+
mode = options.delete(:mode) || :inline
|
233
|
+
raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
|
210
234
|
|
211
|
-
#
|
212
|
-
import_scope(relation, method_name: method_name,
|
213
|
-
self.refresh if refresh
|
214
|
-
true
|
215
|
-
elsif scoped && !full
|
216
|
-
# TODO throw ArgumentError
|
217
|
-
Searchkick.warn("unsupported keywords: #{options.keys.map(&:inspect).join(", ")}") if options.any?
|
218
|
-
|
219
|
-
# reindex association
|
220
|
-
import_scope(relation, scope: scope)
|
235
|
+
# import only
|
236
|
+
import_scope(relation, method_name: method_name, mode: mode)
|
221
237
|
self.refresh if refresh
|
222
238
|
true
|
223
239
|
else
|
224
|
-
|
225
|
-
|
240
|
+
async = options.delete(:async)
|
241
|
+
if async
|
242
|
+
if async.is_a?(Hash) && async[:wait]
|
243
|
+
# TODO warn in 5.1
|
244
|
+
# Searchkick.warn "async option is deprecated - use mode: :async, wait: true instead"
|
245
|
+
options[:wait] = true unless options.key?(:wait)
|
246
|
+
else
|
247
|
+
# TODO warn in 5.1
|
248
|
+
# Searchkick.warn "async option is deprecated - use mode: :async instead"
|
249
|
+
end
|
250
|
+
options[:mode] ||= :async
|
251
|
+
end
|
252
|
+
|
253
|
+
full_reindex(relation, **options)
|
226
254
|
end
|
227
255
|
end
|
228
256
|
|
229
257
|
def create_index(index_options: nil)
|
230
258
|
index_options ||= self.index_options
|
231
|
-
index =
|
259
|
+
index = Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
|
232
260
|
index.create(index_options)
|
233
261
|
index
|
234
262
|
end
|
235
263
|
|
236
264
|
def import_scope(relation, **options)
|
237
|
-
|
265
|
+
relation_indexer.reindex(relation, **options)
|
238
266
|
end
|
239
267
|
|
240
268
|
def batches_left
|
241
|
-
|
269
|
+
relation_indexer.batches_left
|
242
270
|
end
|
243
271
|
|
244
|
-
#
|
245
|
-
|
272
|
+
# private
|
246
273
|
def klass_document_type(klass, ignore_type = false)
|
247
274
|
@klass_document_type[[klass, ignore_type]] ||= begin
|
248
275
|
if !ignore_type && klass.searchkick_klass.searchkick_options[:_type]
|
@@ -255,7 +282,7 @@ module Searchkick
|
|
255
282
|
end
|
256
283
|
end
|
257
284
|
|
258
|
-
#
|
285
|
+
# private
|
259
286
|
def conversions_fields
|
260
287
|
@conversions_fields ||= begin
|
261
288
|
conversions = Array(options[:conversions])
|
@@ -263,10 +290,12 @@ module Searchkick
|
|
263
290
|
end
|
264
291
|
end
|
265
292
|
|
293
|
+
# private
|
266
294
|
def suggest_fields
|
267
295
|
@suggest_fields ||= Array(options[:suggest]).map(&:to_s)
|
268
296
|
end
|
269
297
|
|
298
|
+
# private
|
270
299
|
def locations_fields
|
271
300
|
@locations_fields ||= begin
|
272
301
|
locations = Array(options[:locations])
|
@@ -285,8 +314,20 @@ module Searchkick
|
|
285
314
|
Searchkick.client
|
286
315
|
end
|
287
316
|
|
288
|
-
def
|
289
|
-
|
317
|
+
def queue_index(records)
|
318
|
+
Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).index_data })
|
319
|
+
end
|
320
|
+
|
321
|
+
def queue_delete(records)
|
322
|
+
Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data })
|
323
|
+
end
|
324
|
+
|
325
|
+
def queue_update(records, method_name)
|
326
|
+
Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) })
|
327
|
+
end
|
328
|
+
|
329
|
+
def relation_indexer
|
330
|
+
@relation_indexer ||= RelationIndexer.new(self)
|
290
331
|
end
|
291
332
|
|
292
333
|
def index_settings
|
@@ -297,13 +338,24 @@ module Searchkick
|
|
297
338
|
index.import_scope(relation, **import_options)
|
298
339
|
end
|
299
340
|
|
341
|
+
def reindex_records(object, mode: nil, refresh: false, **options)
|
342
|
+
mode ||= Searchkick.callbacks_value || @options[:callbacks] || :inline
|
343
|
+
mode = :inline if mode == :bulk
|
344
|
+
|
345
|
+
result = RecordIndexer.new(self).reindex(object, mode: mode, full: false, **options)
|
346
|
+
self.refresh if refresh
|
347
|
+
result
|
348
|
+
end
|
349
|
+
|
300
350
|
# https://gist.github.com/jarosan/3124884
|
301
351
|
# http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
|
302
|
-
def
|
352
|
+
def full_reindex(relation, import: true, resume: false, retain: false, mode: nil, refresh_interval: nil, scope: nil, wait: nil)
|
353
|
+
raise ArgumentError, "wait only available in :async mode" if !wait.nil? && mode != :async
|
354
|
+
|
303
355
|
if resume
|
304
356
|
index_name = all_indices.sort.last
|
305
|
-
raise
|
306
|
-
index =
|
357
|
+
raise Error, "No index to resume" unless index_name
|
358
|
+
index = Index.new(index_name, @options)
|
307
359
|
else
|
308
360
|
clean_indices unless retain
|
309
361
|
|
@@ -313,9 +365,9 @@ module Searchkick
|
|
313
365
|
end
|
314
366
|
|
315
367
|
import_options = {
|
316
|
-
|
317
|
-
async: async,
|
368
|
+
mode: (mode || :inline),
|
318
369
|
full: true,
|
370
|
+
resume: resume,
|
319
371
|
scope: scope
|
320
372
|
}
|
321
373
|
|
@@ -327,7 +379,7 @@ module Searchkick
|
|
327
379
|
import_before_promotion(index, relation, **import_options) if import
|
328
380
|
|
329
381
|
# get existing indices to remove
|
330
|
-
unless async
|
382
|
+
unless mode == :async
|
331
383
|
check_uuid(uuid, index.uuid)
|
332
384
|
promote(index.name, update_refresh_interval: !refresh_interval.nil?)
|
333
385
|
clean_indices unless retain
|
@@ -340,8 +392,8 @@ module Searchkick
|
|
340
392
|
index.import_scope(relation, **import_options) if import
|
341
393
|
end
|
342
394
|
|
343
|
-
if async
|
344
|
-
if
|
395
|
+
if mode == :async
|
396
|
+
if wait
|
345
397
|
puts "Created index: #{index.name}"
|
346
398
|
puts "Jobs queued. Waiting..."
|
347
399
|
loop do
|
@@ -367,7 +419,7 @@ module Searchkick
|
|
367
419
|
end
|
368
420
|
rescue => e
|
369
421
|
if Searchkick.transport_error?(e) && e.message.include?("No handler for type [text]")
|
370
|
-
raise UnsupportedVersionError
|
422
|
+
raise UnsupportedVersionError
|
371
423
|
end
|
372
424
|
|
373
425
|
raise e
|
@@ -379,7 +431,36 @@ module Searchkick
|
|
379
431
|
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation
|
380
432
|
def check_uuid(old_uuid, new_uuid)
|
381
433
|
if old_uuid != new_uuid
|
382
|
-
raise
|
434
|
+
raise Error, "Safety check failed - only run one Model.reindex per model at a time"
|
435
|
+
end
|
436
|
+
end
|
437
|
+
|
438
|
+
def notify(record, name)
|
439
|
+
if Searchkick.callbacks_value == :bulk
|
440
|
+
yield
|
441
|
+
else
|
442
|
+
name = "#{record.class.searchkick_klass.name} #{name}" if record && record.class.searchkick_klass
|
443
|
+
event = {
|
444
|
+
name: name,
|
445
|
+
id: search_id(record)
|
446
|
+
}
|
447
|
+
ActiveSupport::Notifications.instrument("request.searchkick", event) do
|
448
|
+
yield
|
449
|
+
end
|
450
|
+
end
|
451
|
+
end
|
452
|
+
|
453
|
+
def notify_bulk(records, name)
|
454
|
+
if Searchkick.callbacks_value == :bulk
|
455
|
+
yield
|
456
|
+
else
|
457
|
+
event = {
|
458
|
+
name: "#{records.first.class.searchkick_klass.name} #{name}",
|
459
|
+
count: records.size
|
460
|
+
}
|
461
|
+
ActiveSupport::Notifications.instrument("request.searchkick", event) do
|
462
|
+
yield
|
463
|
+
end
|
383
464
|
end
|
384
465
|
end
|
385
466
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Searchkick
|
2
|
+
class IndexCache
|
3
|
+
def initialize(max_size: 20)
|
4
|
+
@data = {}
|
5
|
+
@mutex = Mutex.new
|
6
|
+
@max_size = max_size
|
7
|
+
end
|
8
|
+
|
9
|
+
# probably a better pattern for this
|
10
|
+
# but keep it simple
|
11
|
+
def fetch(name)
|
12
|
+
# thread-safe in MRI without mutex
|
13
|
+
# due to how context switching works
|
14
|
+
@mutex.synchronize do
|
15
|
+
if @data.key?(name)
|
16
|
+
@data[name]
|
17
|
+
else
|
18
|
+
@data.clear if @data.size >= @max_size
|
19
|
+
@data[name] = yield
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def clear
|
25
|
+
@mutex.synchronize do
|
26
|
+
@data.clear
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -7,18 +7,16 @@ module Searchkick
|
|
7
7
|
end
|
8
8
|
|
9
9
|
def index_options
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
custom_mapping = {index_type => custom_mapping}
|
14
|
-
end
|
10
|
+
# mortal symbols are garbage collected in Ruby 2.2+
|
11
|
+
custom_settings = (options[:settings] || {}).deep_symbolize_keys
|
12
|
+
custom_mappings = (options[:mappings] || {}).deep_symbolize_keys
|
15
13
|
|
16
14
|
if options[:mappings] && !options[:merge_mappings]
|
17
|
-
settings =
|
18
|
-
mappings =
|
15
|
+
settings = custom_settings
|
16
|
+
mappings = custom_mappings
|
19
17
|
else
|
20
|
-
settings = generate_settings
|
21
|
-
mappings = generate_mappings.
|
18
|
+
settings = generate_settings.deep_symbolize_keys.deep_merge(custom_settings)
|
19
|
+
mappings = generate_mappings.deep_symbolize_keys.deep_merge(custom_mappings)
|
22
20
|
end
|
23
21
|
|
24
22
|
set_deep_paging(settings) if options[:deep_paging]
|
@@ -162,17 +160,14 @@ module Searchkick
|
|
162
160
|
settings[:number_of_replicas] = 0
|
163
161
|
end
|
164
162
|
|
165
|
-
# TODO remove in Searchkick 5 (classic no longer supported)
|
166
163
|
if options[:similarity]
|
167
164
|
settings[:similarity] = {default: {type: options[:similarity]}}
|
168
165
|
end
|
169
166
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
}
|
175
|
-
end
|
167
|
+
settings[:index] = {
|
168
|
+
max_ngram_diff: 49,
|
169
|
+
max_shingle_diff: 4
|
170
|
+
}
|
176
171
|
|
177
172
|
if options[:case_sensitive]
|
178
173
|
settings[:analysis][:analyzer].each do |_, analyzer|
|
@@ -180,13 +175,8 @@ module Searchkick
|
|
180
175
|
end
|
181
176
|
end
|
182
177
|
|
183
|
-
# TODO do this last in Searchkick 5
|
184
|
-
settings = settings.symbolize_keys.deep_merge((options[:settings] || {}).symbolize_keys)
|
185
|
-
|
186
178
|
add_synonyms(settings)
|
187
179
|
add_search_synonyms(settings)
|
188
|
-
# TODO remove in Searchkick 5
|
189
|
-
add_wordnet(settings) if options[:wordnet]
|
190
180
|
|
191
181
|
if options[:special_characters] == false
|
192
182
|
settings[:analysis][:analyzer].each_value do |analyzer_settings|
|
@@ -223,19 +213,7 @@ module Searchkick
|
|
223
213
|
type: "smartcn"
|
224
214
|
}
|
225
215
|
)
|
226
|
-
when "japanese"
|
227
|
-
settings[:analysis][:analyzer].merge!(
|
228
|
-
default_analyzer => {
|
229
|
-
type: "kuromoji"
|
230
|
-
},
|
231
|
-
searchkick_search: {
|
232
|
-
type: "kuromoji"
|
233
|
-
},
|
234
|
-
searchkick_search2: {
|
235
|
-
type: "kuromoji"
|
236
|
-
}
|
237
|
-
)
|
238
|
-
when "japanese2"
|
216
|
+
when "japanese", "japanese2"
|
239
217
|
analyzer = {
|
240
218
|
type: "custom",
|
241
219
|
tokenizer: "kuromoji_tokenizer",
|
@@ -379,16 +357,15 @@ module Searchkick
|
|
379
357
|
}
|
380
358
|
end
|
381
359
|
|
382
|
-
mapping_options =
|
360
|
+
mapping_options =
|
383
361
|
[:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable]
|
384
|
-
.
|
385
|
-
]
|
362
|
+
.to_h { |type| [type, (options[type] || []).map(&:to_s)] }
|
386
363
|
|
387
364
|
word = options[:word] != false && (!options[:match] || options[:match] == :word)
|
388
365
|
|
389
366
|
mapping_options[:searchable].delete("_all")
|
390
367
|
|
391
|
-
analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer}
|
368
|
+
analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer.to_s}
|
392
369
|
|
393
370
|
mapping_options.values.flatten.uniq.each do |field|
|
394
371
|
fields = {}
|
@@ -481,10 +458,6 @@ module Searchkick
|
|
481
458
|
]
|
482
459
|
}
|
483
460
|
|
484
|
-
if below70?
|
485
|
-
mappings = {index_type => mappings}
|
486
|
-
end
|
487
|
-
|
488
461
|
mappings
|
489
462
|
end
|
490
463
|
|
@@ -533,14 +506,14 @@ module Searchkick
|
|
533
506
|
end
|
534
507
|
settings[:analysis][:filter][:searchkick_synonym_graph] = synonym_graph
|
535
508
|
|
536
|
-
if options[:language]
|
509
|
+
if ["japanese", "japanese2"].include?(options[:language])
|
537
510
|
[:searchkick_search, :searchkick_search2].each do |analyzer|
|
538
511
|
settings[:analysis][:analyzer][analyzer][:filter].insert(4, "searchkick_synonym_graph")
|
539
512
|
end
|
540
513
|
else
|
541
514
|
[:searchkick_search2, :searchkick_word_search].each do |analyzer|
|
542
515
|
unless settings[:analysis][:analyzer][analyzer].key?(:filter)
|
543
|
-
raise
|
516
|
+
raise Error, "Search synonyms are not supported yet for language"
|
544
517
|
end
|
545
518
|
|
546
519
|
settings[:analysis][:analyzer][analyzer][:filter].insert(2, "searchkick_synonym_graph")
|
@@ -549,21 +522,6 @@ module Searchkick
|
|
549
522
|
end
|
550
523
|
end
|
551
524
|
|
552
|
-
def add_wordnet(settings)
|
553
|
-
settings[:analysis][:filter][:searchkick_wordnet] = {
|
554
|
-
type: "synonym",
|
555
|
-
format: "wordnet",
|
556
|
-
synonyms_path: Searchkick.wordnet_path
|
557
|
-
}
|
558
|
-
|
559
|
-
settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_wordnet")
|
560
|
-
settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_wordnet"
|
561
|
-
|
562
|
-
%w(word_start word_middle word_end).each do |type|
|
563
|
-
settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_wordnet")
|
564
|
-
end
|
565
|
-
end
|
566
|
-
|
567
525
|
def set_deep_paging(settings)
|
568
526
|
if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
|
569
527
|
settings[:index] ||= {}
|
@@ -587,14 +545,6 @@ module Searchkick
|
|
587
545
|
:searchkick_index
|
588
546
|
end
|
589
547
|
|
590
|
-
def below62?
|
591
|
-
Searchkick.server_below?("6.2.0")
|
592
|
-
end
|
593
|
-
|
594
|
-
def below70?
|
595
|
-
Searchkick.server_below?("7.0.0")
|
596
|
-
end
|
597
|
-
|
598
548
|
def below73?
|
599
549
|
Searchkick.server_below?("7.3.0")
|
600
550
|
end
|
data/lib/searchkick/indexer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# thread-local (technically fiber-local) indexer
|
2
|
+
# used to aggregate bulk callbacks across models
|
1
3
|
module Searchkick
|
2
4
|
class Indexer
|
3
5
|
attr_reader :queued_items
|
@@ -14,15 +16,20 @@ module Searchkick
|
|
14
16
|
def perform
|
15
17
|
items = @queued_items
|
16
18
|
@queued_items = []
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
19
|
+
|
20
|
+
return if items.empty?
|
21
|
+
|
22
|
+
response = Searchkick.client.bulk(body: items)
|
23
|
+
if response["errors"]
|
24
|
+
# note: delete does not set error when item not found
|
25
|
+
first_with_error = response["items"].map do |item|
|
26
|
+
(item["index"] || item["delete"] || item["update"])
|
27
|
+
end.find { |item| item["error"] }
|
28
|
+
raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
|
25
29
|
end
|
30
|
+
|
31
|
+
# maybe return response in future
|
32
|
+
nil
|
26
33
|
end
|
27
34
|
end
|
28
35
|
end
|