searchkick 2.3.2 → 5.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +377 -84
- data/LICENSE.txt +1 -1
- data/README.md +859 -602
- data/lib/searchkick/bulk_reindex_job.rb +13 -9
- data/lib/searchkick/controller_runtime.rb +40 -0
- data/lib/searchkick/hash_wrapper.rb +12 -0
- data/lib/searchkick/index.rb +281 -356
- data/lib/searchkick/index_cache.rb +30 -0
- data/lib/searchkick/index_options.rb +487 -281
- data/lib/searchkick/indexer.rb +15 -8
- data/lib/searchkick/log_subscriber.rb +57 -0
- data/lib/searchkick/middleware.rb +9 -2
- data/lib/searchkick/model.rb +72 -118
- data/lib/searchkick/multi_search.rb +9 -10
- data/lib/searchkick/process_batch_job.rb +12 -15
- data/lib/searchkick/process_queue_job.rb +22 -13
- data/lib/searchkick/query.rb +458 -217
- data/lib/searchkick/railtie.rb +7 -0
- data/lib/searchkick/record_data.rb +128 -0
- data/lib/searchkick/record_indexer.rb +164 -0
- data/lib/searchkick/reindex_queue.rb +51 -9
- data/lib/searchkick/reindex_v2_job.rb +10 -32
- data/lib/searchkick/relation.rb +247 -0
- data/lib/searchkick/relation_indexer.rb +155 -0
- data/lib/searchkick/results.rb +201 -82
- data/lib/searchkick/version.rb +1 -1
- data/lib/searchkick/where.rb +11 -0
- data/lib/searchkick.rb +269 -97
- data/lib/tasks/searchkick.rake +37 -0
- metadata +24 -178
- data/.gitignore +0 -22
- data/.travis.yml +0 -39
- data/Gemfile +0 -16
- data/Rakefile +0 -20
- data/benchmark/Gemfile +0 -23
- data/benchmark/benchmark.rb +0 -97
- data/lib/searchkick/logging.rb +0 -242
- data/lib/searchkick/tasks.rb +0 -33
- data/searchkick.gemspec +0 -28
- data/test/aggs_test.rb +0 -197
- data/test/autocomplete_test.rb +0 -75
- data/test/boost_test.rb +0 -202
- data/test/callbacks_test.rb +0 -59
- data/test/ci/before_install.sh +0 -17
- data/test/errors_test.rb +0 -19
- data/test/gemfiles/activerecord31.gemfile +0 -7
- data/test/gemfiles/activerecord32.gemfile +0 -7
- data/test/gemfiles/activerecord40.gemfile +0 -8
- data/test/gemfiles/activerecord41.gemfile +0 -8
- data/test/gemfiles/activerecord42.gemfile +0 -7
- data/test/gemfiles/activerecord50.gemfile +0 -7
- data/test/gemfiles/apartment.gemfile +0 -8
- data/test/gemfiles/cequel.gemfile +0 -8
- data/test/gemfiles/mongoid2.gemfile +0 -7
- data/test/gemfiles/mongoid3.gemfile +0 -6
- data/test/gemfiles/mongoid4.gemfile +0 -7
- data/test/gemfiles/mongoid5.gemfile +0 -7
- data/test/gemfiles/mongoid6.gemfile +0 -12
- data/test/gemfiles/nobrainer.gemfile +0 -8
- data/test/gemfiles/parallel_tests.gemfile +0 -8
- data/test/geo_shape_test.rb +0 -175
- data/test/highlight_test.rb +0 -78
- data/test/index_test.rb +0 -166
- data/test/inheritance_test.rb +0 -83
- data/test/marshal_test.rb +0 -8
- data/test/match_test.rb +0 -276
- data/test/misspellings_test.rb +0 -56
- data/test/model_test.rb +0 -42
- data/test/multi_search_test.rb +0 -36
- data/test/multi_tenancy_test.rb +0 -22
- data/test/order_test.rb +0 -46
- data/test/pagination_test.rb +0 -70
- data/test/partial_reindex_test.rb +0 -58
- data/test/query_test.rb +0 -35
- data/test/records_test.rb +0 -10
- data/test/reindex_test.rb +0 -64
- data/test/reindex_v2_job_test.rb +0 -32
- data/test/routing_test.rb +0 -23
- data/test/should_index_test.rb +0 -32
- data/test/similar_test.rb +0 -28
- data/test/sql_test.rb +0 -214
- data/test/suggest_test.rb +0 -95
- data/test/support/kaminari.yml +0 -21
- data/test/synonyms_test.rb +0 -67
- data/test/test_helper.rb +0 -567
- data/test/where_test.rb +0 -223
data/lib/searchkick/index.rb
CHANGED
@@ -1,7 +1,5 @@
|
|
1
1
|
module Searchkick
|
2
2
|
class Index
|
3
|
-
include IndexOptions
|
4
|
-
|
5
3
|
attr_reader :name, :options
|
6
4
|
|
7
5
|
def initialize(name, options = {})
|
@@ -10,12 +8,16 @@ module Searchkick
|
|
10
8
|
@klass_document_type = {} # cache
|
11
9
|
end
|
12
10
|
|
11
|
+
def index_options
|
12
|
+
IndexOptions.new(self).index_options
|
13
|
+
end
|
14
|
+
|
13
15
|
def create(body = {})
|
14
16
|
client.indices.create index: name, body: body
|
15
17
|
end
|
16
18
|
|
17
19
|
def delete
|
18
|
-
if
|
20
|
+
if alias_exists?
|
19
21
|
# can't call delete directly on aliases in ES 6
|
20
22
|
indices = client.indices.get_alias(name: name).keys
|
21
23
|
client.indices.delete index: indices
|
@@ -36,25 +38,45 @@ module Searchkick
|
|
36
38
|
client.indices.exists_alias name: name
|
37
39
|
end
|
38
40
|
|
41
|
+
# call to_h for consistent results between elasticsearch gem 7 and 8
|
42
|
+
# could do for all API calls, but just do for ones where return value is focus for now
|
39
43
|
def mapping
|
40
|
-
client.indices.get_mapping
|
44
|
+
client.indices.get_mapping(index: name).to_h
|
41
45
|
end
|
42
46
|
|
47
|
+
# call to_h for consistent results between elasticsearch gem 7 and 8
|
43
48
|
def settings
|
44
|
-
client.indices.get_settings
|
49
|
+
client.indices.get_settings(index: name).to_h
|
45
50
|
end
|
46
51
|
|
47
52
|
def refresh_interval
|
48
|
-
|
53
|
+
index_settings["refresh_interval"]
|
49
54
|
end
|
50
55
|
|
51
56
|
def update_settings(settings)
|
52
57
|
client.indices.put_settings index: name, body: settings
|
53
58
|
end
|
54
59
|
|
60
|
+
def tokens(text, options = {})
|
61
|
+
client.indices.analyze(body: {text: text}.merge(options), index: name)["tokens"].map { |t| t["token"] }
|
62
|
+
end
|
63
|
+
|
64
|
+
def total_docs
|
65
|
+
response =
|
66
|
+
client.search(
|
67
|
+
index: name,
|
68
|
+
body: {
|
69
|
+
query: {match_all: {}},
|
70
|
+
size: 0
|
71
|
+
}
|
72
|
+
)
|
73
|
+
|
74
|
+
Results.new(nil, response).total_count
|
75
|
+
end
|
76
|
+
|
55
77
|
def promote(new_name, update_refresh_interval: false)
|
56
78
|
if update_refresh_interval
|
57
|
-
new_index =
|
79
|
+
new_index = Index.new(new_name, @options)
|
58
80
|
settings = options[:settings] || {}
|
59
81
|
refresh_interval = (settings[:index] && settings[:index][:refresh_interval]) || "1s"
|
60
82
|
new_index.update_settings(index: {refresh_interval: refresh_interval})
|
@@ -63,7 +85,8 @@ module Searchkick
|
|
63
85
|
old_indices =
|
64
86
|
begin
|
65
87
|
client.indices.get_alias(name: name).keys
|
66
|
-
rescue
|
88
|
+
rescue => e
|
89
|
+
raise e unless Searchkick.not_found_error?(e)
|
67
90
|
{}
|
68
91
|
end
|
69
92
|
actions = old_indices.map { |old_name| {remove: {index: old_name, alias: name}} } + [{add: {index: new_name, alias: name}}]
|
@@ -71,176 +94,293 @@ module Searchkick
|
|
71
94
|
end
|
72
95
|
alias_method :swap, :promote
|
73
96
|
|
74
|
-
|
75
|
-
|
97
|
+
def retrieve(record)
|
98
|
+
record_data = RecordData.new(self, record).record_data
|
99
|
+
|
100
|
+
# remove underscore
|
101
|
+
get_options = record_data.to_h { |k, v| [k.to_s.sub(/\A_/, "").to_sym, v] }
|
102
|
+
|
103
|
+
client.get(get_options)["_source"]
|
104
|
+
end
|
105
|
+
|
106
|
+
def all_indices(unaliased: false)
|
107
|
+
indices =
|
108
|
+
begin
|
109
|
+
if client.indices.respond_to?(:get_alias)
|
110
|
+
client.indices.get_alias(index: "#{name}*")
|
111
|
+
else
|
112
|
+
client.indices.get_aliases
|
113
|
+
end
|
114
|
+
rescue => e
|
115
|
+
raise e unless Searchkick.not_found_error?(e)
|
116
|
+
{}
|
117
|
+
end
|
118
|
+
indices = indices.select { |_k, v| v.empty? || v["aliases"].empty? } if unaliased
|
119
|
+
indices.select { |k, _v| k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
|
120
|
+
end
|
121
|
+
|
122
|
+
# remove old indices that start w/ index_name
|
123
|
+
def clean_indices
|
124
|
+
indices = all_indices(unaliased: true)
|
125
|
+
indices.each do |index|
|
126
|
+
Index.new(index).delete
|
127
|
+
end
|
128
|
+
indices
|
129
|
+
end
|
76
130
|
|
77
131
|
def store(record)
|
78
|
-
|
132
|
+
notify(record, "Store") do
|
133
|
+
queue_index([record])
|
134
|
+
end
|
79
135
|
end
|
80
136
|
|
81
137
|
def remove(record)
|
82
|
-
|
138
|
+
notify(record, "Remove") do
|
139
|
+
queue_delete([record])
|
140
|
+
end
|
83
141
|
end
|
84
142
|
|
85
143
|
def update_record(record, method_name)
|
86
|
-
|
144
|
+
notify(record, "Update") do
|
145
|
+
queue_update([record], method_name)
|
146
|
+
end
|
87
147
|
end
|
88
148
|
|
89
149
|
def bulk_delete(records)
|
90
|
-
|
150
|
+
return if records.empty?
|
151
|
+
|
152
|
+
notify_bulk(records, "Delete") do
|
153
|
+
queue_delete(records)
|
154
|
+
end
|
91
155
|
end
|
92
156
|
|
93
157
|
def bulk_index(records)
|
94
|
-
|
158
|
+
return if records.empty?
|
159
|
+
|
160
|
+
notify_bulk(records, "Import") do
|
161
|
+
queue_index(records)
|
162
|
+
end
|
95
163
|
end
|
96
164
|
alias_method :import, :bulk_index
|
97
165
|
|
98
166
|
def bulk_update(records, method_name)
|
99
|
-
|
100
|
-
end
|
167
|
+
return if records.empty?
|
101
168
|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
_id: search_id(r),
|
106
|
-
_type: document_type(r)
|
107
|
-
}
|
108
|
-
data[:_routing] = r.search_routing if r.respond_to?(:search_routing)
|
109
|
-
data
|
110
|
-
end
|
111
|
-
|
112
|
-
def retrieve(record)
|
113
|
-
client.get(
|
114
|
-
index: name,
|
115
|
-
type: document_type(record),
|
116
|
-
id: search_id(record)
|
117
|
-
)["_source"]
|
169
|
+
notify_bulk(records, "Update") do
|
170
|
+
queue_update(records, method_name)
|
171
|
+
end
|
118
172
|
end
|
119
173
|
|
120
|
-
def
|
121
|
-
|
122
|
-
begin
|
123
|
-
remove(record)
|
124
|
-
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
125
|
-
# do nothing
|
126
|
-
end
|
127
|
-
else
|
128
|
-
store(record)
|
129
|
-
end
|
174
|
+
def search_id(record)
|
175
|
+
RecordData.new(self, record).search_id
|
130
176
|
end
|
131
177
|
|
132
|
-
def
|
133
|
-
|
134
|
-
if defined?(Searchkick::ReindexV2Job)
|
135
|
-
Searchkick::ReindexV2Job.perform_later(record.class.name, record.id.to_s)
|
136
|
-
else
|
137
|
-
raise Searchkick::Error, "Active Job not found"
|
138
|
-
end
|
139
|
-
else
|
140
|
-
reindex_record(record)
|
141
|
-
end
|
178
|
+
def document_type(record)
|
179
|
+
RecordData.new(self, record).document_type
|
142
180
|
end
|
143
181
|
|
144
182
|
def similar_record(record, **options)
|
145
|
-
like_text = retrieve(record).to_hash
|
146
|
-
.keep_if { |k, _| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
|
147
|
-
.values.compact.join(" ")
|
148
|
-
|
149
|
-
# TODO deep merge method
|
150
|
-
options[:where] ||= {}
|
151
|
-
options[:where][:_id] ||= {}
|
152
|
-
options[:where][:_id][:not] = record.id.to_s
|
153
183
|
options[:per_page] ||= 10
|
154
|
-
options[:similar] =
|
184
|
+
options[:similar] = [RecordData.new(self, record).record_data]
|
185
|
+
options[:models] ||= [record.class] unless options.key?(:model)
|
155
186
|
|
156
|
-
|
157
|
-
|
187
|
+
Searchkick.search("*", **options)
|
188
|
+
end
|
189
|
+
|
190
|
+
def reload_synonyms
|
191
|
+
if Searchkick.opensearch?
|
192
|
+
client.transport.perform_request "POST", "_plugins/_refresh_search_analyzers/#{CGI.escape(name)}"
|
193
|
+
else
|
194
|
+
raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
|
195
|
+
begin
|
196
|
+
client.transport.perform_request("GET", "#{CGI.escape(name)}/_reload_search_analyzers")
|
197
|
+
rescue => e
|
198
|
+
raise Error, "Requires non-OSS version of Elasticsearch" if Searchkick.not_allowed_error?(e)
|
199
|
+
raise e
|
200
|
+
end
|
201
|
+
end
|
158
202
|
end
|
159
203
|
|
160
204
|
# queue
|
161
205
|
|
162
206
|
def reindex_queue
|
163
|
-
|
207
|
+
ReindexQueue.new(name)
|
164
208
|
end
|
165
209
|
|
166
|
-
#
|
210
|
+
# reindex
|
211
|
+
|
212
|
+
# note: this is designed to be used internally
|
213
|
+
# so it does not check object matches index class
|
214
|
+
def reindex(object, method_name: nil, full: false, **options)
|
215
|
+
if object.is_a?(Array)
|
216
|
+
# note: purposefully skip full
|
217
|
+
return reindex_records(object, method_name: method_name, **options)
|
218
|
+
end
|
219
|
+
|
220
|
+
if !object.respond_to?(:searchkick_klass)
|
221
|
+
raise Error, "Cannot reindex object"
|
222
|
+
end
|
223
|
+
|
224
|
+
scoped = Searchkick.relation?(object)
|
225
|
+
# call searchkick_klass for inheritance
|
226
|
+
relation = scoped ? object.all : Searchkick.scope(object.searchkick_klass).all
|
227
|
+
|
228
|
+
refresh = options.fetch(:refresh, !scoped)
|
229
|
+
options.delete(:refresh)
|
167
230
|
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
231
|
+
if method_name || (scoped && !full)
|
232
|
+
mode = options.delete(:mode) || :inline
|
233
|
+
raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
|
234
|
+
|
235
|
+
# import only
|
236
|
+
import_scope(relation, method_name: method_name, mode: mode)
|
237
|
+
self.refresh if refresh
|
238
|
+
true
|
174
239
|
else
|
175
|
-
|
240
|
+
async = options.delete(:async)
|
241
|
+
if async
|
242
|
+
if async.is_a?(Hash) && async[:wait]
|
243
|
+
# TODO warn in 5.1
|
244
|
+
# Searchkick.warn "async option is deprecated - use mode: :async, wait: true instead"
|
245
|
+
options[:wait] = true unless options.key?(:wait)
|
246
|
+
else
|
247
|
+
# TODO warn in 5.1
|
248
|
+
# Searchkick.warn "async option is deprecated - use mode: :async instead"
|
249
|
+
end
|
250
|
+
options[:mode] ||= :async
|
251
|
+
end
|
252
|
+
|
253
|
+
full_reindex(relation, **options)
|
176
254
|
end
|
177
255
|
end
|
178
256
|
|
179
|
-
# reindex
|
180
|
-
|
181
257
|
def create_index(index_options: nil)
|
182
258
|
index_options ||= self.index_options
|
183
|
-
index =
|
259
|
+
index = Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
|
184
260
|
index.create(index_options)
|
185
261
|
index
|
186
262
|
end
|
187
263
|
|
188
|
-
def
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
264
|
+
def import_scope(relation, **options)
|
265
|
+
relation_indexer.reindex(relation, **options)
|
266
|
+
end
|
267
|
+
|
268
|
+
def batches_left
|
269
|
+
relation_indexer.batches_left
|
270
|
+
end
|
271
|
+
|
272
|
+
# private
|
273
|
+
def klass_document_type(klass, ignore_type = false)
|
274
|
+
@klass_document_type[[klass, ignore_type]] ||= begin
|
275
|
+
if !ignore_type && klass.searchkick_klass.searchkick_options[:_type]
|
276
|
+
type = klass.searchkick_klass.searchkick_options[:_type]
|
277
|
+
type = type.call if type.respond_to?(:call)
|
278
|
+
type
|
279
|
+
else
|
280
|
+
klass.model_name.to_s.underscore
|
194
281
|
end
|
195
|
-
|
196
|
-
indices.select { |k, _v| k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
|
282
|
+
end
|
197
283
|
end
|
198
284
|
|
199
|
-
#
|
200
|
-
def
|
201
|
-
|
202
|
-
|
203
|
-
|
285
|
+
# private
|
286
|
+
def conversions_fields
|
287
|
+
@conversions_fields ||= begin
|
288
|
+
conversions = Array(options[:conversions])
|
289
|
+
conversions.map(&:to_s) + conversions.map(&:to_sym)
|
204
290
|
end
|
205
|
-
indices
|
206
291
|
end
|
207
292
|
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
)
|
293
|
+
# private
|
294
|
+
def suggest_fields
|
295
|
+
@suggest_fields ||= Array(options[:suggest]).map(&:to_s)
|
296
|
+
end
|
297
|
+
|
298
|
+
# private
|
299
|
+
def locations_fields
|
300
|
+
@locations_fields ||= begin
|
301
|
+
locations = Array(options[:locations])
|
302
|
+
locations.map(&:to_s) + locations.map(&:to_sym)
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
# private
|
307
|
+
def uuid
|
308
|
+
index_settings["uuid"]
|
309
|
+
end
|
217
310
|
|
218
|
-
|
311
|
+
protected
|
312
|
+
|
313
|
+
def client
|
314
|
+
Searchkick.client
|
315
|
+
end
|
316
|
+
|
317
|
+
def queue_index(records)
|
318
|
+
Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).index_data })
|
319
|
+
end
|
320
|
+
|
321
|
+
def queue_delete(records)
|
322
|
+
Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data })
|
323
|
+
end
|
324
|
+
|
325
|
+
def queue_update(records, method_name)
|
326
|
+
Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) })
|
327
|
+
end
|
328
|
+
|
329
|
+
def relation_indexer
|
330
|
+
@relation_indexer ||= RelationIndexer.new(self)
|
331
|
+
end
|
332
|
+
|
333
|
+
def index_settings
|
334
|
+
settings.values.first["settings"]["index"]
|
335
|
+
end
|
336
|
+
|
337
|
+
def import_before_promotion(index, relation, **import_options)
|
338
|
+
index.import_scope(relation, **import_options)
|
339
|
+
end
|
340
|
+
|
341
|
+
def reindex_records(object, mode: nil, refresh: false, **options)
|
342
|
+
mode ||= Searchkick.callbacks_value || @options[:callbacks] || :inline
|
343
|
+
mode = :inline if mode == :bulk
|
344
|
+
|
345
|
+
result = RecordIndexer.new(self).reindex(object, mode: mode, full: false, **options)
|
346
|
+
self.refresh if refresh
|
347
|
+
result
|
219
348
|
end
|
220
349
|
|
221
350
|
# https://gist.github.com/jarosan/3124884
|
222
351
|
# http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
|
223
|
-
def
|
352
|
+
def full_reindex(relation, import: true, resume: false, retain: false, mode: nil, refresh_interval: nil, scope: nil, wait: nil)
|
353
|
+
raise ArgumentError, "wait only available in :async mode" if !wait.nil? && mode != :async
|
354
|
+
|
224
355
|
if resume
|
225
356
|
index_name = all_indices.sort.last
|
226
|
-
raise
|
227
|
-
index =
|
357
|
+
raise Error, "No index to resume" unless index_name
|
358
|
+
index = Index.new(index_name, @options)
|
228
359
|
else
|
229
360
|
clean_indices unless retain
|
230
361
|
|
231
|
-
index_options =
|
362
|
+
index_options = relation.searchkick_index_options
|
232
363
|
index_options.deep_merge!(settings: {index: {refresh_interval: refresh_interval}}) if refresh_interval
|
233
364
|
index = create_index(index_options: index_options)
|
234
365
|
end
|
235
366
|
|
367
|
+
import_options = {
|
368
|
+
mode: (mode || :inline),
|
369
|
+
full: true,
|
370
|
+
resume: resume,
|
371
|
+
scope: scope
|
372
|
+
}
|
373
|
+
|
374
|
+
uuid = index.uuid
|
375
|
+
|
236
376
|
# check if alias exists
|
237
377
|
alias_exists = alias_exists?
|
238
378
|
if alias_exists
|
239
|
-
|
240
|
-
index.import_scope(scope, resume: resume, async: async, full: true) if import
|
379
|
+
import_before_promotion(index, relation, **import_options) if import
|
241
380
|
|
242
381
|
# get existing indices to remove
|
243
|
-
unless async
|
382
|
+
unless mode == :async
|
383
|
+
check_uuid(uuid, index.uuid)
|
244
384
|
promote(index.name, update_refresh_interval: !refresh_interval.nil?)
|
245
385
|
clean_indices unless retain
|
246
386
|
end
|
@@ -249,11 +389,11 @@ module Searchkick
|
|
249
389
|
promote(index.name, update_refresh_interval: !refresh_interval.nil?)
|
250
390
|
|
251
391
|
# import after promotion
|
252
|
-
index.import_scope(
|
392
|
+
index.import_scope(relation, **import_options) if import
|
253
393
|
end
|
254
394
|
|
255
|
-
if async
|
256
|
-
if
|
395
|
+
if mode == :async
|
396
|
+
if wait
|
257
397
|
puts "Created index: #{index.name}"
|
258
398
|
puts "Jobs queued. Waiting..."
|
259
399
|
loop do
|
@@ -265,6 +405,7 @@ module Searchkick
|
|
265
405
|
# already promoted if alias didn't exist
|
266
406
|
if alias_exists
|
267
407
|
puts "Jobs complete. Promoting..."
|
408
|
+
check_uuid(uuid, index.uuid)
|
268
409
|
promote(index.name, update_refresh_interval: !refresh_interval.nil?)
|
269
410
|
end
|
270
411
|
clean_indices unless retain
|
@@ -276,267 +417,51 @@ module Searchkick
|
|
276
417
|
index.refresh
|
277
418
|
true
|
278
419
|
end
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
# use scope for import
|
283
|
-
scope = scope.search_import if scope.respond_to?(:search_import)
|
284
|
-
|
285
|
-
if batch
|
286
|
-
import_or_update scope.to_a, method_name, async
|
287
|
-
Searchkick.with_redis { |r| r.srem(batches_key, batch_id) } if batch_id
|
288
|
-
elsif full && async
|
289
|
-
full_reindex_async(scope)
|
290
|
-
elsif scope.respond_to?(:find_in_batches)
|
291
|
-
if resume
|
292
|
-
# use total docs instead of max id since there's not a great way
|
293
|
-
# to get the max _id without scripting since it's a string
|
294
|
-
|
295
|
-
# TODO use primary key and prefix with table name
|
296
|
-
scope = scope.where("id > ?", total_docs)
|
297
|
-
end
|
298
|
-
|
299
|
-
scope = scope.select("id").except(:includes, :preload) if async
|
300
|
-
|
301
|
-
scope.find_in_batches batch_size: batch_size do |items|
|
302
|
-
import_or_update items, method_name, async
|
303
|
-
end
|
304
|
-
else
|
305
|
-
each_batch(scope) do |items|
|
306
|
-
import_or_update items, method_name, async
|
307
|
-
end
|
308
|
-
end
|
309
|
-
end
|
310
|
-
|
311
|
-
def batches_left
|
312
|
-
Searchkick.with_redis { |r| r.scard(batches_key) }
|
313
|
-
end
|
314
|
-
|
315
|
-
# other
|
316
|
-
|
317
|
-
def tokens(text, options = {})
|
318
|
-
client.indices.analyze(body: {text: text}.merge(options), index: name)["tokens"].map { |t| t["token"] }
|
319
|
-
end
|
320
|
-
|
321
|
-
def klass_document_type(klass)
|
322
|
-
@klass_document_type[klass] ||= begin
|
323
|
-
if klass.respond_to?(:document_type)
|
324
|
-
klass.document_type
|
325
|
-
else
|
326
|
-
klass.model_name.to_s.underscore
|
327
|
-
end
|
420
|
+
rescue => e
|
421
|
+
if Searchkick.transport_error?(e) && (e.message.include?("No handler for type [text]") || e.message.include?("class java.util.ArrayList cannot be cast to class java.util.Map"))
|
422
|
+
raise UnsupportedVersionError
|
328
423
|
end
|
329
|
-
end
|
330
|
-
|
331
|
-
protected
|
332
424
|
|
333
|
-
|
334
|
-
Searchkick.client
|
425
|
+
raise e
|
335
426
|
end
|
336
427
|
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
428
|
+
# safety check
|
429
|
+
# still a chance for race condition since its called before promotion
|
430
|
+
# ideal is for user to disable automatic index creation
|
431
|
+
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation
|
432
|
+
def check_uuid(old_uuid, new_uuid)
|
433
|
+
if old_uuid != new_uuid
|
434
|
+
raise Error, "Safety check failed - only run one Model.reindex per model at a time"
|
342
435
|
end
|
343
436
|
end
|
344
437
|
|
345
|
-
def
|
346
|
-
|
347
|
-
|
348
|
-
end
|
349
|
-
|
350
|
-
EXCLUDED_ATTRIBUTES = ["_id", "_type"]
|
351
|
-
|
352
|
-
def search_data(record, method_name = nil)
|
353
|
-
partial_reindex = !method_name.nil?
|
354
|
-
options = record.class.searchkick_options
|
355
|
-
|
356
|
-
# remove _id since search_id is used instead
|
357
|
-
source = record.send(method_name || :search_data).each_with_object({}) { |(k, v), memo| memo[k.to_s] = v; memo }.except(*EXCLUDED_ATTRIBUTES)
|
358
|
-
|
359
|
-
# conversions
|
360
|
-
if options[:conversions]
|
361
|
-
Array(options[:conversions]).map(&:to_s).each do |conversions_field|
|
362
|
-
if source[conversions_field]
|
363
|
-
source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
|
364
|
-
end
|
365
|
-
end
|
366
|
-
end
|
367
|
-
|
368
|
-
# hack to prevent generator field doesn't exist error
|
369
|
-
if options[:suggest]
|
370
|
-
options[:suggest].map(&:to_s).each do |field|
|
371
|
-
source[field] = nil if !source[field] && !partial_reindex
|
372
|
-
end
|
373
|
-
end
|
374
|
-
|
375
|
-
# locations
|
376
|
-
if options[:locations]
|
377
|
-
options[:locations].map(&:to_s).each do |field|
|
378
|
-
if source[field]
|
379
|
-
if !source[field].is_a?(Hash) && (source[field].first.is_a?(Array) || source[field].first.is_a?(Hash))
|
380
|
-
# multiple locations
|
381
|
-
source[field] = source[field].map { |a| location_value(a) }
|
382
|
-
else
|
383
|
-
source[field] = location_value(source[field])
|
384
|
-
end
|
385
|
-
end
|
386
|
-
end
|
387
|
-
end
|
388
|
-
|
389
|
-
cast_big_decimal(source)
|
390
|
-
|
391
|
-
source
|
392
|
-
end
|
393
|
-
|
394
|
-
def location_value(value)
|
395
|
-
if value.is_a?(Array)
|
396
|
-
value.map(&:to_f).reverse
|
397
|
-
elsif value.is_a?(Hash)
|
398
|
-
{lat: value[:lat].to_f, lon: value[:lon].to_f}
|
399
|
-
else
|
400
|
-
value
|
401
|
-
end
|
402
|
-
end
|
403
|
-
|
404
|
-
# change all BigDecimal values to floats due to
|
405
|
-
# https://github.com/rails/rails/issues/6033
|
406
|
-
# possible loss of precision :/
|
407
|
-
def cast_big_decimal(obj)
|
408
|
-
case obj
|
409
|
-
when BigDecimal
|
410
|
-
obj.to_f
|
411
|
-
when Hash
|
412
|
-
obj.each do |k, v|
|
413
|
-
obj[k] = cast_big_decimal(v)
|
414
|
-
end
|
415
|
-
when Enumerable
|
416
|
-
obj.map do |v|
|
417
|
-
cast_big_decimal(v)
|
418
|
-
end
|
419
|
-
else
|
420
|
-
obj
|
421
|
-
end
|
422
|
-
end
|
423
|
-
|
424
|
-
def import_or_update(records, method_name, async)
|
425
|
-
if records.any?
|
426
|
-
if async
|
427
|
-
Searchkick::BulkReindexJob.perform_later(
|
428
|
-
class_name: records.first.class.name,
|
429
|
-
record_ids: records.map(&:id),
|
430
|
-
index_name: name,
|
431
|
-
method_name: method_name ? method_name.to_s : nil
|
432
|
-
)
|
433
|
-
else
|
434
|
-
records = records.select(&:should_index?)
|
435
|
-
if records.any?
|
436
|
-
with_retries do
|
437
|
-
method_name ? bulk_update(records, method_name) : import(records)
|
438
|
-
end
|
439
|
-
end
|
440
|
-
end
|
441
|
-
end
|
442
|
-
end
|
443
|
-
|
444
|
-
def full_reindex_async(scope)
|
445
|
-
if scope.respond_to?(:primary_key)
|
446
|
-
# TODO expire Redis key
|
447
|
-
primary_key = scope.primary_key
|
448
|
-
|
449
|
-
starting_id =
|
450
|
-
begin
|
451
|
-
scope.minimum(primary_key)
|
452
|
-
rescue ActiveRecord::StatementInvalid
|
453
|
-
false
|
454
|
-
end
|
455
|
-
|
456
|
-
if starting_id.nil?
|
457
|
-
# no records, do nothing
|
458
|
-
elsif starting_id.is_a?(Numeric)
|
459
|
-
max_id = scope.maximum(primary_key)
|
460
|
-
batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil
|
461
|
-
|
462
|
-
batches_count.times do |i|
|
463
|
-
batch_id = i + 1
|
464
|
-
min_id = starting_id + (i * batch_size)
|
465
|
-
bulk_reindex_job scope, batch_id, min_id: min_id, max_id: min_id + batch_size - 1
|
466
|
-
end
|
467
|
-
else
|
468
|
-
scope.find_in_batches(batch_size: batch_size).each_with_index do |batch, i|
|
469
|
-
batch_id = i + 1
|
470
|
-
|
471
|
-
bulk_reindex_job scope, batch_id, record_ids: batch.map { |record| record.id.to_s }
|
472
|
-
end
|
473
|
-
end
|
438
|
+
def notify(record, name)
|
439
|
+
if Searchkick.callbacks_value == :bulk
|
440
|
+
yield
|
474
441
|
else
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
end
|
483
|
-
end
|
484
|
-
|
485
|
-
def each_batch(scope)
|
486
|
-
# https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
|
487
|
-
# use cursor for Mongoid
|
488
|
-
items = []
|
489
|
-
scope.all.each do |item|
|
490
|
-
items << item
|
491
|
-
if items.length == batch_size
|
492
|
-
yield items
|
493
|
-
items = []
|
442
|
+
name = "#{record.class.searchkick_klass.name} #{name}" if record && record.class.searchkick_klass
|
443
|
+
event = {
|
444
|
+
name: name,
|
445
|
+
id: search_id(record)
|
446
|
+
}
|
447
|
+
ActiveSupport::Notifications.instrument("request.searchkick", event) do
|
448
|
+
yield
|
494
449
|
end
|
495
450
|
end
|
496
|
-
yield items if items.any?
|
497
451
|
end
|
498
452
|
|
499
|
-
def
|
500
|
-
Searchkick
|
501
|
-
class_name: scope.model_name.name,
|
502
|
-
index_name: name,
|
503
|
-
batch_id: batch_id
|
504
|
-
}.merge(options))
|
505
|
-
Searchkick.with_redis { |r| r.sadd(batches_key, batch_id) }
|
506
|
-
end
|
507
|
-
|
508
|
-
def batch_size
|
509
|
-
@batch_size ||= @options[:batch_size] || 1000
|
510
|
-
end
|
511
|
-
|
512
|
-
def with_retries
|
513
|
-
retries = 0
|
514
|
-
|
515
|
-
begin
|
453
|
+
def notify_bulk(records, name)
|
454
|
+
if Searchkick.callbacks_value == :bulk
|
516
455
|
yield
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
456
|
+
else
|
457
|
+
event = {
|
458
|
+
name: "#{records.first.class.searchkick_klass.name} #{name}",
|
459
|
+
count: records.size
|
460
|
+
}
|
461
|
+
ActiveSupport::Notifications.instrument("request.searchkick", event) do
|
462
|
+
yield
|
521
463
|
end
|
522
|
-
raise e
|
523
464
|
end
|
524
465
|
end
|
525
|
-
|
526
|
-
def bulk_index_helper(records)
|
527
|
-
Searchkick.indexer.queue(records.map { |r| {index: record_data(r).merge(data: search_data(r))} })
|
528
|
-
end
|
529
|
-
|
530
|
-
def bulk_delete_helper(records)
|
531
|
-
Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| {delete: record_data(r)} })
|
532
|
-
end
|
533
|
-
|
534
|
-
def bulk_update_helper(records, method_name)
|
535
|
-
Searchkick.indexer.queue(records.map { |r| {update: record_data(r).merge(data: {doc: search_data(r, method_name)})} })
|
536
|
-
end
|
537
|
-
|
538
|
-
def batches_key
|
539
|
-
"searchkick:reindex:#{name}:batches"
|
540
|
-
end
|
541
466
|
end
|
542
467
|
end
|