searchkick 4.6.3 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +26 -1
- data/README.md +108 -55
- data/lib/searchkick/bulk_reindex_job.rb +12 -8
- data/lib/searchkick/controller_runtime.rb +40 -0
- data/lib/searchkick/index.rb +121 -54
- data/lib/searchkick/index_cache.rb +30 -0
- data/lib/searchkick/index_options.rb +16 -66
- data/lib/searchkick/indexer.rb +15 -8
- data/lib/searchkick/log_subscriber.rb +57 -0
- data/lib/searchkick/middleware.rb +1 -1
- data/lib/searchkick/model.rb +44 -47
- data/lib/searchkick/process_batch_job.rb +9 -25
- data/lib/searchkick/process_queue_job.rb +3 -2
- data/lib/searchkick/query.rb +36 -52
- data/lib/searchkick/record_data.rb +0 -1
- data/lib/searchkick/record_indexer.rb +135 -51
- data/lib/searchkick/reindex_queue.rb +25 -2
- data/lib/searchkick/reindex_v2_job.rb +10 -34
- data/lib/searchkick/relation.rb +36 -0
- data/lib/searchkick/relation_indexer.rb +150 -0
- data/lib/searchkick/results.rb +22 -23
- data/lib/searchkick/version.rb +1 -1
- data/lib/searchkick.rb +157 -82
- data/lib/tasks/searchkick.rake +6 -3
- metadata +11 -28
- data/lib/searchkick/bulk_indexer.rb +0 -173
- data/lib/searchkick/logging.rb +0 -246
data/lib/searchkick.rb
CHANGED
@@ -1,29 +1,40 @@
|
|
1
1
|
# dependencies
|
2
2
|
require "active_support"
|
3
3
|
require "active_support/core_ext/hash/deep_merge"
|
4
|
-
require "
|
4
|
+
require "active_support/core_ext/module/attr_internal"
|
5
|
+
require "active_support/core_ext/module/delegation"
|
6
|
+
require "active_support/notifications"
|
5
7
|
require "hashie"
|
6
8
|
|
9
|
+
# stdlib
|
10
|
+
require "forwardable"
|
11
|
+
|
7
12
|
# modules
|
8
|
-
require "searchkick/
|
13
|
+
require "searchkick/controller_runtime"
|
9
14
|
require "searchkick/index"
|
15
|
+
require "searchkick/index_cache"
|
16
|
+
require "searchkick/index_options"
|
10
17
|
require "searchkick/indexer"
|
11
18
|
require "searchkick/hash_wrapper"
|
12
|
-
require "searchkick/
|
19
|
+
require "searchkick/log_subscriber"
|
13
20
|
require "searchkick/model"
|
14
21
|
require "searchkick/multi_search"
|
15
22
|
require "searchkick/query"
|
16
23
|
require "searchkick/reindex_queue"
|
17
24
|
require "searchkick/record_data"
|
18
25
|
require "searchkick/record_indexer"
|
26
|
+
require "searchkick/relation"
|
27
|
+
require "searchkick/relation_indexer"
|
19
28
|
require "searchkick/results"
|
20
29
|
require "searchkick/version"
|
21
30
|
|
22
31
|
# integrations
|
23
32
|
require "searchkick/railtie" if defined?(Rails)
|
24
|
-
require "searchkick/logging" if defined?(ActiveSupport::Notifications)
|
25
33
|
|
26
34
|
module Searchkick
|
35
|
+
# requires faraday
|
36
|
+
autoload :Middleware, "searchkick/middleware"
|
37
|
+
|
27
38
|
# background jobs
|
28
39
|
autoload :BulkReindexJob, "searchkick/bulk_reindex_job"
|
29
40
|
autoload :ProcessBatchJob, "searchkick/process_batch_job"
|
@@ -33,19 +44,21 @@ module Searchkick
|
|
33
44
|
# errors
|
34
45
|
class Error < StandardError; end
|
35
46
|
class MissingIndexError < Error; end
|
36
|
-
class UnsupportedVersionError < Error
|
37
|
-
|
38
|
-
|
47
|
+
class UnsupportedVersionError < Error
|
48
|
+
def message
|
49
|
+
"This version of Searchkick requires Elasticsearch 7+ or OpenSearch 1+"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
class InvalidQueryError < Error; end
|
39
53
|
class DangerousOperation < Error; end
|
40
54
|
class ImportError < Error; end
|
41
55
|
|
42
56
|
class << self
|
43
|
-
attr_accessor :search_method_name, :
|
57
|
+
attr_accessor :search_method_name, :timeout, :models, :client_options, :redis, :index_prefix, :index_suffix, :queue_name, :model_options, :client_type
|
44
58
|
attr_writer :client, :env, :search_timeout
|
45
59
|
attr_reader :aws_credentials
|
46
60
|
end
|
47
61
|
self.search_method_name = :search
|
48
|
-
self.wordnet_path = "/var/lib/wn_s.pl"
|
49
62
|
self.timeout = 10
|
50
63
|
self.models = []
|
51
64
|
self.client_options = {}
|
@@ -54,15 +67,45 @@ module Searchkick
|
|
54
67
|
|
55
68
|
def self.client
|
56
69
|
@client ||= begin
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
70
|
+
client_type =
|
71
|
+
if self.client_type
|
72
|
+
self.client_type
|
73
|
+
elsif defined?(OpenSearch::Client) && defined?(Elasticsearch::Client)
|
74
|
+
raise Error, "Multiple clients found - set Searchkick.client_type = :elasticsearch or :opensearch"
|
75
|
+
elsif defined?(OpenSearch::Client)
|
76
|
+
:opensearch
|
77
|
+
elsif defined?(Elasticsearch::Client)
|
78
|
+
:elasticsearch
|
79
|
+
else
|
80
|
+
raise Error, "No client found - install the `elasticsearch` or `opensearch-ruby` gem"
|
81
|
+
end
|
82
|
+
|
83
|
+
# check after client to ensure faraday is installed
|
84
|
+
# TODO remove in Searchkick 6
|
85
|
+
if defined?(Typhoeus) && Gem::Version.new(Faraday::VERSION) < Gem::Version.new("0.14.0")
|
86
|
+
require "typhoeus/adapters/faraday"
|
87
|
+
end
|
88
|
+
|
89
|
+
if client_type == :opensearch
|
90
|
+
OpenSearch::Client.new({
|
91
|
+
url: ENV["OPENSEARCH_URL"],
|
92
|
+
transport_options: {request: {timeout: timeout}, headers: {content_type: "application/json"}},
|
93
|
+
retry_on_failure: 2
|
94
|
+
}.deep_merge(client_options)) do |f|
|
95
|
+
f.use Searchkick::Middleware
|
96
|
+
f.request :aws_sigv4, signer_middleware_aws_params if aws_credentials
|
97
|
+
end
|
98
|
+
else
|
99
|
+
raise Error, "The `elasticsearch` gem must be 7+" if Elasticsearch::VERSION.to_i < 7
|
100
|
+
|
101
|
+
Elasticsearch::Client.new({
|
102
|
+
url: ENV["ELASTICSEARCH_URL"],
|
103
|
+
transport_options: {request: {timeout: timeout}, headers: {content_type: "application/json"}},
|
104
|
+
retry_on_failure: 2
|
105
|
+
}.deep_merge(client_options)) do |f|
|
106
|
+
f.use Searchkick::Middleware
|
107
|
+
f.request :aws_sigv4, signer_middleware_aws_params if aws_credentials
|
108
|
+
end
|
66
109
|
end
|
67
110
|
end
|
68
111
|
end
|
@@ -96,14 +139,6 @@ module Searchkick
|
|
96
139
|
Gem::Version.new(server_version.split("-")[0]) < Gem::Version.new(version.split("-")[0])
|
97
140
|
end
|
98
141
|
|
99
|
-
# memoize for performance
|
100
|
-
def self.server_below7?
|
101
|
-
unless defined?(@server_below7)
|
102
|
-
@server_below7 = server_below?("7.0.0")
|
103
|
-
end
|
104
|
-
@server_below7
|
105
|
-
end
|
106
|
-
|
107
142
|
def self.search(term = "*", model: nil, **options, &block)
|
108
143
|
options = options.dup
|
109
144
|
klass = model
|
@@ -129,17 +164,27 @@ module Searchkick
|
|
129
164
|
end
|
130
165
|
end
|
131
166
|
|
132
|
-
|
133
|
-
query = Searchkick::Query.new(klass, term, **options)
|
167
|
+
# TODO remove in Searchkick 6
|
134
168
|
if options[:execute] == false
|
135
|
-
|
136
|
-
|
137
|
-
query.execute
|
169
|
+
Searchkick.warn("The execute option is no longer needed")
|
170
|
+
options.delete(:execute)
|
138
171
|
end
|
172
|
+
|
173
|
+
options = options.merge(block: block) if block
|
174
|
+
Searchkick::Relation.new(klass, term, **options)
|
139
175
|
end
|
140
176
|
|
141
177
|
def self.multi_search(queries)
|
142
|
-
|
178
|
+
return if queries.empty?
|
179
|
+
|
180
|
+
queries = queries.map { |q| q.send(:query) }
|
181
|
+
event = {
|
182
|
+
name: "Multi Search",
|
183
|
+
body: queries.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join,
|
184
|
+
}
|
185
|
+
ActiveSupport::Notifications.instrument("multi_search.searchkick", event) do
|
186
|
+
Searchkick::MultiSearch.new(queries).perform
|
187
|
+
end
|
143
188
|
end
|
144
189
|
|
145
190
|
# callbacks
|
@@ -160,13 +205,25 @@ module Searchkick
|
|
160
205
|
end
|
161
206
|
end
|
162
207
|
|
163
|
-
|
208
|
+
# message is private
|
209
|
+
def self.callbacks(value = nil, message: nil)
|
164
210
|
if block_given?
|
165
211
|
previous_value = callbacks_value
|
166
212
|
begin
|
167
213
|
self.callbacks_value = value
|
168
214
|
result = yield
|
169
|
-
|
215
|
+
if callbacks_value == :bulk && indexer.queued_items.any?
|
216
|
+
event = {}
|
217
|
+
if message
|
218
|
+
message.call(event)
|
219
|
+
else
|
220
|
+
event[:name] = "Bulk"
|
221
|
+
event[:count] = indexer.queued_items.size
|
222
|
+
end
|
223
|
+
ActiveSupport::Notifications.instrument("request.searchkick", event) do
|
224
|
+
indexer.perform
|
225
|
+
end
|
226
|
+
end
|
170
227
|
result
|
171
228
|
ensure
|
172
229
|
self.callbacks_value = previous_value
|
@@ -177,12 +234,8 @@ module Searchkick
|
|
177
234
|
end
|
178
235
|
|
179
236
|
def self.aws_credentials=(creds)
|
180
|
-
|
181
|
-
|
182
|
-
require "faraday_middleware/aws_signers_v4"
|
183
|
-
rescue LoadError
|
184
|
-
require "faraday_middleware/aws_sigv4"
|
185
|
-
end
|
237
|
+
require "faraday_middleware/aws_sigv4"
|
238
|
+
|
186
239
|
@aws_credentials = creds
|
187
240
|
@client = nil # reset client
|
188
241
|
end
|
@@ -197,7 +250,6 @@ module Searchkick
|
|
197
250
|
}
|
198
251
|
end
|
199
252
|
|
200
|
-
# TODO use ConnectionPool::Wrapper when redis is set so this is no longer needed
|
201
253
|
def self.with_redis
|
202
254
|
if redis
|
203
255
|
if redis.respond_to?(:with)
|
@@ -215,29 +267,41 @@ module Searchkick
|
|
215
267
|
end
|
216
268
|
|
217
269
|
# private
|
218
|
-
def self.load_records(
|
219
|
-
|
220
|
-
if
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
# Nobrainer
|
228
|
-
records.unscoped.where(:id.in => ids)
|
229
|
-
elsif records.respond_to?(:key_column_names)
|
230
|
-
records.where(records.key_column_names.first => ids)
|
270
|
+
def self.load_records(relation, ids)
|
271
|
+
relation =
|
272
|
+
if relation.respond_to?(:primary_key)
|
273
|
+
primary_key = relation.primary_key
|
274
|
+
raise Error, "Need primary key to load records" if !primary_key
|
275
|
+
|
276
|
+
relation.where(primary_key => ids)
|
277
|
+
elsif relation.respond_to?(:queryable)
|
278
|
+
relation.queryable.for_ids(ids)
|
231
279
|
end
|
232
280
|
|
233
|
-
raise
|
281
|
+
raise Error, "Not sure how to load records" if !relation
|
282
|
+
|
283
|
+
relation
|
284
|
+
end
|
234
285
|
|
235
|
-
|
286
|
+
# private
|
287
|
+
def self.load_model(class_name, allow_child: false)
|
288
|
+
model = class_name.safe_constantize
|
289
|
+
raise Error, "Could not find class: #{class_name}" unless model
|
290
|
+
if allow_child
|
291
|
+
unless model.respond_to?(:searchkick_klass)
|
292
|
+
raise Error, "#{class_name} is not a searchkick model"
|
293
|
+
end
|
294
|
+
else
|
295
|
+
unless Searchkick.models.include?(model)
|
296
|
+
raise Error, "#{class_name} is not a searchkick model"
|
297
|
+
end
|
298
|
+
end
|
299
|
+
model
|
236
300
|
end
|
237
301
|
|
238
302
|
# private
|
239
303
|
def self.indexer
|
240
|
-
Thread.current[:searchkick_indexer] ||=
|
304
|
+
Thread.current[:searchkick_indexer] ||= Indexer.new
|
241
305
|
end
|
242
306
|
|
243
307
|
# private
|
@@ -250,22 +314,9 @@ module Searchkick
|
|
250
314
|
Thread.current[:searchkick_callbacks_enabled] = value
|
251
315
|
end
|
252
316
|
|
253
|
-
# private
|
254
|
-
def self.signer_middleware_key
|
255
|
-
defined?(FaradayMiddleware::AwsSignersV4) ? :aws_signers_v4 : :aws_sigv4
|
256
|
-
end
|
257
|
-
|
258
317
|
# private
|
259
318
|
def self.signer_middleware_aws_params
|
260
|
-
|
261
|
-
{service: "es", region: "us-east-1"}.merge(aws_credentials)
|
262
|
-
else
|
263
|
-
{
|
264
|
-
credentials: aws_credentials[:credentials] || Aws::Credentials.new(aws_credentials[:access_key_id], aws_credentials[:secret_access_key]),
|
265
|
-
service_name: "es",
|
266
|
-
region: aws_credentials[:region] || "us-east-1"
|
267
|
-
}
|
268
|
-
end
|
319
|
+
{service: "es", region: "us-east-1"}.merge(aws_credentials)
|
269
320
|
end
|
270
321
|
|
271
322
|
# private
|
@@ -275,31 +326,55 @@ module Searchkick
|
|
275
326
|
def self.relation?(klass)
|
276
327
|
if klass.respond_to?(:current_scope)
|
277
328
|
!klass.current_scope.nil?
|
278
|
-
|
279
|
-
!Mongoid::Threaded.current_scope(klass).nil?
|
329
|
+
else
|
330
|
+
klass.is_a?(Mongoid::Criteria) || !Mongoid::Threaded.current_scope(klass).nil?
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
# private
|
335
|
+
def self.scope(model)
|
336
|
+
# safety check to make sure used properly in code
|
337
|
+
raise Error, "Cannot scope relation" if relation?(model)
|
338
|
+
|
339
|
+
if model.searchkick_options[:unscope]
|
340
|
+
model.unscoped
|
341
|
+
else
|
342
|
+
model
|
280
343
|
end
|
281
344
|
end
|
282
345
|
|
283
346
|
# private
|
284
347
|
def self.not_found_error?(e)
|
285
|
-
(defined?(
|
348
|
+
(defined?(Elastic::Transport) && e.is_a?(Elastic::Transport::Transport::Errors::NotFound)) ||
|
349
|
+
(defined?(Elasticsearch::Transport) && e.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound)) ||
|
286
350
|
(defined?(OpenSearch) && e.is_a?(OpenSearch::Transport::Transport::Errors::NotFound))
|
287
351
|
end
|
288
352
|
|
289
353
|
# private
|
290
354
|
def self.transport_error?(e)
|
291
|
-
(defined?(
|
355
|
+
(defined?(Elastic::Transport) && e.is_a?(Elastic::Transport::Transport::Error)) ||
|
356
|
+
(defined?(Elasticsearch::Transport) && e.is_a?(Elasticsearch::Transport::Transport::Error)) ||
|
292
357
|
(defined?(OpenSearch) && e.is_a?(OpenSearch::Transport::Transport::Error))
|
293
358
|
end
|
294
|
-
end
|
295
359
|
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
360
|
+
# private
|
361
|
+
def self.not_allowed_error?(e)
|
362
|
+
(defined?(Elastic::Transport) && e.is_a?(Elastic::Transport::Transport::Errors::MethodNotAllowed)) ||
|
363
|
+
(defined?(Elasticsearch::Transport) && e.is_a?(Elasticsearch::Transport::Transport::Errors::MethodNotAllowed)) ||
|
364
|
+
(defined?(OpenSearch) && e.is_a?(OpenSearch::Transport::Transport::Errors::MethodNotAllowed))
|
365
|
+
end
|
366
|
+
end
|
302
367
|
|
303
368
|
ActiveSupport.on_load(:active_record) do
|
304
369
|
extend Searchkick::Model
|
305
370
|
end
|
371
|
+
|
372
|
+
ActiveSupport.on_load(:mongoid) do
|
373
|
+
Mongoid::Document::ClassMethods.include Searchkick::Model
|
374
|
+
end
|
375
|
+
|
376
|
+
ActiveSupport.on_load(:action_controller) do
|
377
|
+
include Searchkick::ControllerRuntime
|
378
|
+
end
|
379
|
+
|
380
|
+
Searchkick::LogSubscriber.attach_to :searchkick
|
data/lib/tasks/searchkick.rake
CHANGED
@@ -4,9 +4,12 @@ namespace :searchkick do
|
|
4
4
|
class_name = ENV["CLASS"]
|
5
5
|
abort "USAGE: rake searchkick:reindex CLASS=Product" unless class_name
|
6
6
|
|
7
|
-
model =
|
8
|
-
|
9
|
-
|
7
|
+
model =
|
8
|
+
begin
|
9
|
+
Searchkick.load_model(class_name)
|
10
|
+
rescue Searchkick::Error => e
|
11
|
+
abort e.message
|
12
|
+
end
|
10
13
|
|
11
14
|
puts "Reindexing #{model.name}..."
|
12
15
|
model.reindex
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: searchkick
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 5.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Kane
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-02-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activemodel
|
@@ -16,34 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '5'
|
19
|
+
version: '5.2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '5'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: elasticsearch
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '6'
|
34
|
-
- - "<"
|
35
|
-
- !ruby/object:Gem::Version
|
36
|
-
version: '7.14'
|
37
|
-
type: :runtime
|
38
|
-
prerelease: false
|
39
|
-
version_requirements: !ruby/object:Gem::Requirement
|
40
|
-
requirements:
|
41
|
-
- - ">="
|
42
|
-
- !ruby/object:Gem::Version
|
43
|
-
version: '6'
|
44
|
-
- - "<"
|
45
|
-
- !ruby/object:Gem::Version
|
46
|
-
version: '7.14'
|
26
|
+
version: '5.2'
|
47
27
|
- !ruby/object:Gem::Dependency
|
48
28
|
name: hashie
|
49
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -68,13 +48,14 @@ files:
|
|
68
48
|
- LICENSE.txt
|
69
49
|
- README.md
|
70
50
|
- lib/searchkick.rb
|
71
|
-
- lib/searchkick/bulk_indexer.rb
|
72
51
|
- lib/searchkick/bulk_reindex_job.rb
|
52
|
+
- lib/searchkick/controller_runtime.rb
|
73
53
|
- lib/searchkick/hash_wrapper.rb
|
74
54
|
- lib/searchkick/index.rb
|
55
|
+
- lib/searchkick/index_cache.rb
|
75
56
|
- lib/searchkick/index_options.rb
|
76
57
|
- lib/searchkick/indexer.rb
|
77
|
-
- lib/searchkick/
|
58
|
+
- lib/searchkick/log_subscriber.rb
|
78
59
|
- lib/searchkick/middleware.rb
|
79
60
|
- lib/searchkick/model.rb
|
80
61
|
- lib/searchkick/multi_search.rb
|
@@ -86,6 +67,8 @@ files:
|
|
86
67
|
- lib/searchkick/record_indexer.rb
|
87
68
|
- lib/searchkick/reindex_queue.rb
|
88
69
|
- lib/searchkick/reindex_v2_job.rb
|
70
|
+
- lib/searchkick/relation.rb
|
71
|
+
- lib/searchkick/relation_indexer.rb
|
89
72
|
- lib/searchkick/results.rb
|
90
73
|
- lib/searchkick/version.rb
|
91
74
|
- lib/tasks/searchkick.rake
|
@@ -101,14 +84,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
101
84
|
requirements:
|
102
85
|
- - ">="
|
103
86
|
- !ruby/object:Gem::Version
|
104
|
-
version: '2.
|
87
|
+
version: '2.6'
|
105
88
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
106
89
|
requirements:
|
107
90
|
- - ">="
|
108
91
|
- !ruby/object:Gem::Version
|
109
92
|
version: '0'
|
110
93
|
requirements: []
|
111
|
-
rubygems_version: 3.
|
94
|
+
rubygems_version: 3.3.3
|
112
95
|
signing_key:
|
113
96
|
specification_version: 4
|
114
97
|
summary: Intelligent search made easy with Rails and Elasticsearch or OpenSearch
|
@@ -1,173 +0,0 @@
|
|
1
|
-
module Searchkick
|
2
|
-
class BulkIndexer
|
3
|
-
attr_reader :index
|
4
|
-
|
5
|
-
def initialize(index)
|
6
|
-
@index = index
|
7
|
-
end
|
8
|
-
|
9
|
-
def import_scope(relation, resume: false, method_name: nil, async: false, batch: false, batch_id: nil, full: false, scope: nil)
|
10
|
-
if scope
|
11
|
-
relation = relation.send(scope)
|
12
|
-
elsif relation.respond_to?(:search_import)
|
13
|
-
relation = relation.search_import
|
14
|
-
end
|
15
|
-
|
16
|
-
if batch
|
17
|
-
import_or_update relation.to_a, method_name, async
|
18
|
-
Searchkick.with_redis { |r| r.srem(batches_key, batch_id) } if batch_id
|
19
|
-
elsif full && async
|
20
|
-
full_reindex_async(relation)
|
21
|
-
elsif relation.respond_to?(:find_in_batches)
|
22
|
-
if resume
|
23
|
-
# use total docs instead of max id since there's not a great way
|
24
|
-
# to get the max _id without scripting since it's a string
|
25
|
-
|
26
|
-
# TODO use primary key and prefix with table name
|
27
|
-
relation = relation.where("id > ?", index.total_docs)
|
28
|
-
end
|
29
|
-
|
30
|
-
relation = relation.select("id").except(:includes, :preload) if async
|
31
|
-
|
32
|
-
relation.find_in_batches batch_size: batch_size do |items|
|
33
|
-
import_or_update items, method_name, async
|
34
|
-
end
|
35
|
-
else
|
36
|
-
each_batch(relation) do |items|
|
37
|
-
import_or_update items, method_name, async
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
def bulk_index(records)
|
43
|
-
Searchkick.indexer.queue(records.map { |r| RecordData.new(index, r).index_data })
|
44
|
-
end
|
45
|
-
|
46
|
-
def bulk_delete(records)
|
47
|
-
Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(index, r).delete_data })
|
48
|
-
end
|
49
|
-
|
50
|
-
def bulk_update(records, method_name)
|
51
|
-
Searchkick.indexer.queue(records.map { |r| RecordData.new(index, r).update_data(method_name) })
|
52
|
-
end
|
53
|
-
|
54
|
-
def batches_left
|
55
|
-
Searchkick.with_redis { |r| r.scard(batches_key) }
|
56
|
-
end
|
57
|
-
|
58
|
-
private
|
59
|
-
|
60
|
-
def import_or_update(records, method_name, async)
|
61
|
-
if records.any?
|
62
|
-
if async
|
63
|
-
Searchkick::BulkReindexJob.perform_later(
|
64
|
-
class_name: records.first.class.searchkick_options[:class_name],
|
65
|
-
record_ids: records.map(&:id),
|
66
|
-
index_name: index.name,
|
67
|
-
method_name: method_name ? method_name.to_s : nil
|
68
|
-
)
|
69
|
-
else
|
70
|
-
records = records.select(&:should_index?)
|
71
|
-
if records.any?
|
72
|
-
with_retries do
|
73
|
-
# call out to index for ActiveSupport notifications
|
74
|
-
if method_name
|
75
|
-
index.bulk_update(records, method_name)
|
76
|
-
else
|
77
|
-
index.bulk_index(records)
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
def full_reindex_async(scope)
|
86
|
-
if scope.respond_to?(:primary_key)
|
87
|
-
# TODO expire Redis key
|
88
|
-
primary_key = scope.primary_key
|
89
|
-
|
90
|
-
scope = scope.select(primary_key).except(:includes, :preload)
|
91
|
-
|
92
|
-
starting_id =
|
93
|
-
begin
|
94
|
-
scope.minimum(primary_key)
|
95
|
-
rescue ActiveRecord::StatementInvalid
|
96
|
-
false
|
97
|
-
end
|
98
|
-
|
99
|
-
if starting_id.nil?
|
100
|
-
# no records, do nothing
|
101
|
-
elsif starting_id.is_a?(Numeric)
|
102
|
-
max_id = scope.maximum(primary_key)
|
103
|
-
batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil
|
104
|
-
|
105
|
-
batches_count.times do |i|
|
106
|
-
batch_id = i + 1
|
107
|
-
min_id = starting_id + (i * batch_size)
|
108
|
-
bulk_reindex_job scope, batch_id, min_id: min_id, max_id: min_id + batch_size - 1
|
109
|
-
end
|
110
|
-
else
|
111
|
-
scope.find_in_batches(batch_size: batch_size).each_with_index do |batch, i|
|
112
|
-
batch_id = i + 1
|
113
|
-
|
114
|
-
bulk_reindex_job scope, batch_id, record_ids: batch.map { |record| record.id.to_s }
|
115
|
-
end
|
116
|
-
end
|
117
|
-
else
|
118
|
-
batch_id = 1
|
119
|
-
# TODO remove any eager loading
|
120
|
-
scope = scope.only(:_id) if scope.respond_to?(:only)
|
121
|
-
each_batch(scope) do |items|
|
122
|
-
bulk_reindex_job scope, batch_id, record_ids: items.map { |i| i.id.to_s }
|
123
|
-
batch_id += 1
|
124
|
-
end
|
125
|
-
end
|
126
|
-
end
|
127
|
-
|
128
|
-
def each_batch(scope)
|
129
|
-
# https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
|
130
|
-
# use cursor for Mongoid
|
131
|
-
items = []
|
132
|
-
scope.all.each do |item|
|
133
|
-
items << item
|
134
|
-
if items.length == batch_size
|
135
|
-
yield items
|
136
|
-
items = []
|
137
|
-
end
|
138
|
-
end
|
139
|
-
yield items if items.any?
|
140
|
-
end
|
141
|
-
|
142
|
-
def bulk_reindex_job(scope, batch_id, options)
|
143
|
-
Searchkick.with_redis { |r| r.sadd(batches_key, batch_id) }
|
144
|
-
Searchkick::BulkReindexJob.perform_later(**{
|
145
|
-
class_name: scope.searchkick_options[:class_name],
|
146
|
-
index_name: index.name,
|
147
|
-
batch_id: batch_id
|
148
|
-
}.merge(options))
|
149
|
-
end
|
150
|
-
|
151
|
-
def with_retries
|
152
|
-
retries = 0
|
153
|
-
|
154
|
-
begin
|
155
|
-
yield
|
156
|
-
rescue Faraday::ClientError => e
|
157
|
-
if retries < 1
|
158
|
-
retries += 1
|
159
|
-
retry
|
160
|
-
end
|
161
|
-
raise e
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
def batches_key
|
166
|
-
"searchkick:reindex:#{index.name}:batches"
|
167
|
-
end
|
168
|
-
|
169
|
-
def batch_size
|
170
|
-
@batch_size ||= index.options[:batch_size] || 1000
|
171
|
-
end
|
172
|
-
end
|
173
|
-
end
|