chewy 8.0.1 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +72 -0
- data/README.md +3 -18
- data/lib/chewy/errors.rb +3 -0
- data/lib/chewy/fields/root.rb +6 -4
- data/lib/chewy/index/actions.rb +13 -11
- data/lib/chewy/index/adapter/object.rb +17 -0
- data/lib/chewy/index/adapter/orm.rb +20 -0
- data/lib/chewy/index/compiled.rb +235 -0
- data/lib/chewy/index/crutch.rb +12 -2
- data/lib/chewy/index/import/bulk_builder.rb +8 -5
- data/lib/chewy/index/import/progressbar.rb +79 -0
- data/lib/chewy/index/import/routine.rb +3 -2
- data/lib/chewy/index/import.rb +87 -20
- data/lib/chewy/index/mapping.rb +1 -0
- data/lib/chewy/index/witchcraft.rb +30 -19
- data/lib/chewy/index/wrapper.rb +1 -1
- data/lib/chewy/index.rb +2 -0
- data/lib/chewy/minitest/helpers.rb +0 -1
- data/lib/chewy/multi_search.rb +1 -1
- data/lib/chewy/rake_helper.rb +19 -2
- data/lib/chewy/rspec/helpers.rb +0 -1
- data/lib/chewy/search/parameters/runtime_mappings.rb +14 -0
- data/lib/chewy/search/request.rb +20 -4
- data/lib/chewy/search/scrolling.rb +14 -6
- data/lib/chewy/stash.rb +10 -6
- data/lib/chewy/strategy/delayed_sidekiq/worker.rb +1 -1
- data/lib/chewy/version.rb +1 -1
- data/lib/chewy.rb +1 -0
- metadata +6 -6
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
module Chewy
|
|
2
|
+
class Index
|
|
3
|
+
module Import
|
|
4
|
+
# Thin wrapper around `ruby-progressbar` for import feedback.
|
|
5
|
+
#
|
|
6
|
+
# Unlike the original PR #787 implementation, this wrapper is only
|
|
7
|
+
# touched from the parent process: serial imports increment it directly,
|
|
8
|
+
# and parallel imports increment it via `Parallel`'s `finish:` callback
|
|
9
|
+
# (which runs in the parent under an internal mutex). The workers stay
|
|
10
|
+
# process-based, so there is no GVL contention as in PR #787 / #800.
|
|
11
|
+
#
|
|
12
|
+
# `Progressbar.build` returns a NULL object when the feature is disabled,
|
|
13
|
+
# so call sites do not need feature guards.
|
|
14
|
+
class Progressbar
|
|
15
|
+
NULL = Object.new
|
|
16
|
+
class << NULL
|
|
17
|
+
def increment(_); end
|
|
18
|
+
def total=(_); end
|
|
19
|
+
def finish; end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
BOUNDED_FORMAT = '%t |%B| %p%% %c/%C %e'.freeze
|
|
23
|
+
UNBOUNDED_FORMAT = '%t %c (%a)'.freeze
|
|
24
|
+
TITLE = 'Importing'.freeze
|
|
25
|
+
|
|
26
|
+
# @param enabled [Boolean, :unbounded] feature flag. `:unbounded` shows
|
|
27
|
+
# a spinner with no total (skip `import_count`).
|
|
28
|
+
# @param total [Integer, nil] expected total; ignored when `:unbounded`.
|
|
29
|
+
# @return [Progressbar, NULL]
|
|
30
|
+
def self.build(enabled, total)
|
|
31
|
+
return NULL unless enabled
|
|
32
|
+
|
|
33
|
+
unless '::ProgressBar'.safe_constantize
|
|
34
|
+
raise 'The `ruby-progressbar` gem is required for import progress, ' \
|
|
35
|
+
"please add `gem 'ruby-progressbar'` to your Gemfile"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
return new if enabled == :unbounded
|
|
39
|
+
|
|
40
|
+
new(normalize_total(total))
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Some ActiveRecord scopes (e.g., `.group(...)`) make `.count` return a
|
|
44
|
+
# Hash rather than an Integer. Coerce so we still get a usable total.
|
|
45
|
+
def self.normalize_total(total)
|
|
46
|
+
case total
|
|
47
|
+
when Hash then total.values.sum
|
|
48
|
+
when Integer then total
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
attr_reader :bar
|
|
53
|
+
|
|
54
|
+
def initialize(total = nil)
|
|
55
|
+
format = total ? BOUNDED_FORMAT : UNBOUNDED_FORMAT
|
|
56
|
+
@bar = ::ProgressBar.create(title: TITLE, total: total, format: format)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Clamps to total when bounded — action_objects may include :delete
|
|
60
|
+
# entries (parent-child re-indexing, delete_if scope) that aren't
|
|
61
|
+
# counted by `adapter.import_count`, which would otherwise raise
|
|
62
|
+
# ProgressBar::InvalidProgressError.
|
|
63
|
+
def increment(by)
|
|
64
|
+
target = bar.progress + by
|
|
65
|
+
target = [bar.total, target].min if bar.total
|
|
66
|
+
bar.progress = target
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def total=(value)
|
|
70
|
+
bar.total = value
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def finish
|
|
74
|
+
bar.finish unless bar.finished?
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
@@ -31,7 +31,7 @@ module Chewy
|
|
|
31
31
|
|
|
32
32
|
DEFAULT_OPTIONS = {
|
|
33
33
|
refresh: true,
|
|
34
|
-
update_fields:
|
|
34
|
+
update_fields: nil,
|
|
35
35
|
update_failover: true,
|
|
36
36
|
batch_size: Chewy::Index::Adapter::Base::BATCH_SIZE
|
|
37
37
|
}.freeze
|
|
@@ -56,6 +56,7 @@ module Chewy
|
|
|
56
56
|
{}
|
|
57
57
|
end
|
|
58
58
|
end
|
|
59
|
+
@context = @options[:context] || {}
|
|
59
60
|
@errors = []
|
|
60
61
|
@stats = {}
|
|
61
62
|
@leftovers = []
|
|
@@ -78,7 +79,7 @@ module Chewy
|
|
|
78
79
|
# @param delete [Array<Object>] any acceptable objects for deleting
|
|
79
80
|
# @return [true, false] the result of the request, true if no errors
|
|
80
81
|
def process(index: [], delete: [])
|
|
81
|
-
bulk_builder = BulkBuilder.new(@index, to_index: index, delete: delete, fields: @options[:update_fields])
|
|
82
|
+
bulk_builder = BulkBuilder.new(@index, to_index: index, delete: delete, fields: @options[:update_fields], context: @context)
|
|
82
83
|
bulk_body = bulk_builder.bulk_body
|
|
83
84
|
|
|
84
85
|
if @options[:journal]
|
data/lib/chewy/index/import.rb
CHANGED
|
@@ -2,6 +2,7 @@ require 'chewy/index/import/journal_builder'
|
|
|
2
2
|
require 'chewy/index/import/bulk_builder'
|
|
3
3
|
require 'chewy/index/import/bulk_request'
|
|
4
4
|
require 'chewy/index/import/routine'
|
|
5
|
+
require 'chewy/index/import/progressbar'
|
|
5
6
|
|
|
6
7
|
module Chewy
|
|
7
8
|
class Index
|
|
@@ -11,20 +12,22 @@ module Chewy
|
|
|
11
12
|
IMPORT_WORKER = lambda do |index, options, total, ids, iteration|
|
|
12
13
|
::Process.setproctitle("chewy [#{index}]: import data (#{iteration + 1}/#{total})")
|
|
13
14
|
routine = Routine.new(index, **options)
|
|
15
|
+
processed = 0
|
|
14
16
|
index.adapter.import(*ids, routine.options) do |action_objects|
|
|
15
17
|
routine.process(**action_objects)
|
|
18
|
+
processed += action_objects.sum { |_, v| v.size }
|
|
16
19
|
end
|
|
17
|
-
{errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
|
|
20
|
+
{errors: routine.errors, import: routine.stats, leftovers: routine.leftovers, processed: processed}
|
|
18
21
|
end
|
|
19
22
|
|
|
20
23
|
LEFTOVERS_WORKER = lambda do |index, options, total, body, iteration|
|
|
21
24
|
::Process.setproctitle("chewy [#{index}]: import leftovers (#{iteration + 1}/#{total})")
|
|
22
25
|
routine = Routine.new(index, **options)
|
|
23
26
|
routine.perform_bulk(body)
|
|
24
|
-
routine.errors
|
|
27
|
+
{errors: routine.errors}
|
|
25
28
|
end
|
|
26
29
|
|
|
27
|
-
module ClassMethods
|
|
30
|
+
module ClassMethods # rubocop:disable Metrics/ModuleLength
|
|
28
31
|
# @!method import(*collection, **options)
|
|
29
32
|
# Basically, one of the main methods for an index. Performs any objects import
|
|
30
33
|
# to the index. Does all the objects handling routines.
|
|
@@ -68,9 +71,15 @@ module Chewy
|
|
|
68
71
|
# @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default
|
|
69
72
|
# @option options [Boolean] direct_import skips object reloading in ORM adapter, `false` by default
|
|
70
73
|
# @option options [true, false] journal enables imported objects journaling, false by default
|
|
71
|
-
# @option options [Array<Symbol, String>] update_fields list of fields for
|
|
74
|
+
# @option options [Array<Symbol, String>] update_fields list of fields for partial import. `nil` (default) triggers full document reindex;
|
|
75
|
+
# an empty array (`[]`) is an explicit no-op (no fields updated).
|
|
72
76
|
# @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default
|
|
73
77
|
# @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
|
|
78
|
+
# @option options [true, false, :unbounded] progressbar shows an import progressbar
|
|
79
|
+
# on stderr. `true` precomputes the total via `adapter.import_count` (one extra
|
|
80
|
+
# count query); `:unbounded` shows a spinner without computing the total. Default
|
|
81
|
+
# `false`. Safe in parallel mode: the bar is incremented in the parent process via
|
|
82
|
+
# `Parallel`'s `finish:` callback, workers stay process-based.
|
|
74
83
|
# @return [true, false] false in case of errors
|
|
75
84
|
def import(*args)
|
|
76
85
|
intercept_import_using_strategy(*args).blank?
|
|
@@ -115,13 +124,26 @@ module Chewy
|
|
|
115
124
|
# @param crutches [Object] optional crutches object; if omitted - a crutch for the single passed object is created as a fallback
|
|
116
125
|
# @param fields [Array<Symbol>] and array of fields to restrict the generated document
|
|
117
126
|
# @return [Hash] a JSON-ready hash
|
|
118
|
-
def compose(object, crutches = nil, fields: [])
|
|
119
|
-
crutches ||= Chewy::Index::Crutch::Crutches.new self, [object]
|
|
127
|
+
def compose(object, crutches = nil, fields: [], context: {})
|
|
128
|
+
crutches ||= Chewy::Index::Crutch::Crutches.new self, [object], context
|
|
120
129
|
|
|
121
|
-
|
|
122
|
-
|
|
130
|
+
# Hot path: the default-fields compiled method has already
|
|
131
|
+
# been built, so skip the per-call availability check and
|
|
132
|
+
# dispatch directly. Bulk reindexing hits this branch for
|
|
133
|
+
# every object after the first.
|
|
134
|
+
return __chewy_compose__(object, crutches, context) if fields.empty? && @compiled_default_ready
|
|
135
|
+
|
|
136
|
+
if compiled_compose_available?(fields)
|
|
137
|
+
if fields.empty?
|
|
138
|
+
@compiled_default_ready = true
|
|
139
|
+
__chewy_compose__(object, crutches, context)
|
|
140
|
+
else
|
|
141
|
+
compiled_compose(object, crutches, context, fields)
|
|
142
|
+
end
|
|
143
|
+
elsif witchcraft? && root.children.present?
|
|
144
|
+
cauldron(fields: fields).brew(object, crutches, context)
|
|
123
145
|
else
|
|
124
|
-
root.compose(object, crutches, fields: fields)
|
|
146
|
+
root.compose(object, crutches, fields: fields, context: context)
|
|
125
147
|
end
|
|
126
148
|
end
|
|
127
149
|
|
|
@@ -175,46 +197,91 @@ module Chewy
|
|
|
175
197
|
end
|
|
176
198
|
|
|
177
199
|
def import_linear(objects, routine)
|
|
200
|
+
bar = build_progressbar(routine, objects)
|
|
178
201
|
ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
|
|
179
202
|
adapter.import(*objects, routine.options) do |action_objects|
|
|
180
203
|
routine.process(**action_objects)
|
|
204
|
+
bar.increment(action_objects.sum { |_, v| v.size })
|
|
181
205
|
end
|
|
182
206
|
routine.perform_bulk(routine.leftovers)
|
|
183
207
|
payload[:import] = routine.stats
|
|
184
208
|
payload[:errors] = payload_errors(routine.errors) if routine.errors.present?
|
|
185
209
|
payload[:errors]
|
|
186
210
|
end
|
|
211
|
+
ensure
|
|
212
|
+
bar&.finish
|
|
187
213
|
end
|
|
188
214
|
|
|
189
215
|
def import_parallel(objects, routine)
|
|
190
216
|
raise "The `parallel` gem is required for parallel import, please add `gem 'parallel'` to your Gemfile" unless '::Parallel'.safe_constantize
|
|
191
217
|
|
|
218
|
+
bar = build_progressbar(routine, objects)
|
|
192
219
|
ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
|
|
193
220
|
batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
|
|
194
221
|
|
|
195
222
|
::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
|
|
196
223
|
results = ::Parallel.map_with_index(
|
|
197
224
|
batches,
|
|
198
|
-
routine.parallel_options,
|
|
225
|
+
parallel_options_with_progress(routine.parallel_options, bar),
|
|
199
226
|
&IMPORT_WORKER.curry[self, routine.options, batches.size]
|
|
200
227
|
)
|
|
201
228
|
::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
|
|
202
229
|
errors, import, leftovers = process_parallel_import_results(results)
|
|
203
|
-
|
|
204
|
-
if leftovers.present?
|
|
205
|
-
batches = leftovers.each_slice(routine.options[:batch_size])
|
|
206
|
-
results = ::Parallel.map_with_index(
|
|
207
|
-
batches,
|
|
208
|
-
routine.parallel_options,
|
|
209
|
-
&LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
|
|
210
|
-
)
|
|
211
|
-
errors.concat(results.flatten(1))
|
|
212
|
-
end
|
|
230
|
+
errors.concat(process_parallel_leftovers(leftovers, routine)) if leftovers.present?
|
|
213
231
|
|
|
214
232
|
payload[:import] = import
|
|
215
233
|
payload[:errors] = payload_errors(errors) if errors.present?
|
|
216
234
|
payload[:errors]
|
|
217
235
|
end
|
|
236
|
+
ensure
|
|
237
|
+
bar&.finish
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def process_parallel_leftovers(leftovers, routine)
|
|
241
|
+
batches = leftovers.each_slice(routine.options[:batch_size]).to_a
|
|
242
|
+
results = ::Parallel.map_with_index(
|
|
243
|
+
batches,
|
|
244
|
+
routine.parallel_options,
|
|
245
|
+
&LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
|
|
246
|
+
)
|
|
247
|
+
results.flat_map { |r| r[:errors] }
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Builds Parallel options with a `finish:` callback that increments the
|
|
251
|
+
# progressbar after each worker batch returns. The callback runs in the
|
|
252
|
+
# parent (main) thread under Parallel's internal mutex (parallel-1.x),
|
|
253
|
+
# so workers stay process-based and there is no worker-side
|
|
254
|
+
# synchronization — the regression that triggered the PR #800 revert.
|
|
255
|
+
#
|
|
256
|
+
# If the caller already supplied a `finish:` callback in
|
|
257
|
+
# `parallel_options`, both run; user callback first, then the bar.
|
|
258
|
+
def parallel_options_with_progress(parallel_options, bar)
|
|
259
|
+
user_finish = parallel_options[:finish]
|
|
260
|
+
progress = lambda do |item, i, result|
|
|
261
|
+
user_finish&.call(item, i, result)
|
|
262
|
+
bar.increment(result[:processed]) if result.is_a?(Hash) && result[:processed]
|
|
263
|
+
end
|
|
264
|
+
parallel_options.merge(finish: progress)
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def build_progressbar(routine, objects)
|
|
268
|
+
enabled = routine.options[:progressbar]
|
|
269
|
+
total = enabled == true ? safe_import_count(objects) : nil
|
|
270
|
+
Progressbar.build(enabled, total)
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
# Returns nil when the adapter cannot or should not be counted:
|
|
274
|
+
# missing `import_count` on a custom adapter, a grouped scope that
|
|
275
|
+
# raises, or any unexpected count failure. A nil total makes
|
|
276
|
+
# `Progressbar.new` render a spinner instead of a bounded bar —
|
|
277
|
+
# avoids aborting the import just because the progressbar can't
|
|
278
|
+
# size itself.
|
|
279
|
+
def safe_import_count(objects)
|
|
280
|
+
return nil unless adapter.respond_to?(:import_count)
|
|
281
|
+
|
|
282
|
+
adapter.import_count(*objects)
|
|
283
|
+
rescue StandardError
|
|
284
|
+
nil
|
|
218
285
|
end
|
|
219
286
|
|
|
220
287
|
def process_parallel_import_results(results)
|
data/lib/chewy/index/mapping.rb
CHANGED
|
@@ -1,15 +1,3 @@
|
|
|
1
|
-
begin
|
|
2
|
-
require 'method_source'
|
|
3
|
-
begin
|
|
4
|
-
require 'prism'
|
|
5
|
-
rescue LoadError
|
|
6
|
-
require 'parser/current'
|
|
7
|
-
end
|
|
8
|
-
require 'unparser'
|
|
9
|
-
rescue LoadError
|
|
10
|
-
nil
|
|
11
|
-
end
|
|
12
|
-
|
|
13
1
|
module Chewy
|
|
14
2
|
class Index
|
|
15
3
|
module Witchcraft
|
|
@@ -19,8 +7,30 @@ module Chewy
|
|
|
19
7
|
class_attribute :_witchcraft, instance_reader: false, instance_writer: false
|
|
20
8
|
end
|
|
21
9
|
|
|
10
|
+
def self.load_dependencies!
|
|
11
|
+
return if @dependencies_loaded
|
|
12
|
+
|
|
13
|
+
require 'method_source'
|
|
14
|
+
begin
|
|
15
|
+
require 'prism'
|
|
16
|
+
rescue LoadError
|
|
17
|
+
require 'parser/current'
|
|
18
|
+
end
|
|
19
|
+
require 'unparser'
|
|
20
|
+
@dependencies_loaded = true
|
|
21
|
+
rescue LoadError
|
|
22
|
+
nil
|
|
23
|
+
end
|
|
24
|
+
|
|
22
25
|
module ClassMethods
|
|
23
26
|
def witchcraft!
|
|
27
|
+
warn(
|
|
28
|
+
'[DEPRECATION] Chewy::Index.witchcraft! is deprecated and will be removed in a future release. ' \
|
|
29
|
+
'The compiled compose path is now the default and delivers equivalent performance without ' \
|
|
30
|
+
"method_source/parser/unparser dependencies. Remove the `witchcraft!` call from #{name || self}.",
|
|
31
|
+
uplevel: 1
|
|
32
|
+
)
|
|
33
|
+
Witchcraft.load_dependencies!
|
|
24
34
|
self._witchcraft = true
|
|
25
35
|
check_requirements!
|
|
26
36
|
end
|
|
@@ -59,15 +69,15 @@ module Chewy
|
|
|
59
69
|
@fields = fields
|
|
60
70
|
end
|
|
61
71
|
|
|
62
|
-
def brew(object, crutches = nil)
|
|
63
|
-
alicorn.call(locals, object, crutches).as_json
|
|
72
|
+
def brew(object, crutches = nil, context = {})
|
|
73
|
+
alicorn.call(locals, object, crutches, context).as_json
|
|
64
74
|
end
|
|
65
75
|
|
|
66
76
|
private
|
|
67
77
|
|
|
68
78
|
def alicorn
|
|
69
79
|
@alicorn ||= singleton_class.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
70
|
-
-> (locals, object0, crutches) do
|
|
80
|
+
-> (locals, object0, crutches, context) do
|
|
71
81
|
#{composed_values(@index.root, 0)}
|
|
72
82
|
end
|
|
73
83
|
RUBY
|
|
@@ -171,8 +181,8 @@ module Chewy
|
|
|
171
181
|
end
|
|
172
182
|
end
|
|
173
183
|
|
|
174
|
-
def source_for(proc, nesting)
|
|
175
|
-
lambdas =
|
|
184
|
+
def source_for(proc, nesting) # rubocop:disable Metrics/AbcSize
|
|
185
|
+
lambdas = extract_lambdas(ast_from_proc(proc))
|
|
176
186
|
|
|
177
187
|
raise "No lambdas found, try to reformat your code:\n`#{proc.source}`" unless lambdas
|
|
178
188
|
|
|
@@ -189,6 +199,7 @@ module Chewy
|
|
|
189
199
|
source = replace_lvar(source, proc_params[n], :"object#{n}") if proc_params[n]
|
|
190
200
|
end
|
|
191
201
|
source = replace_lvar(source, proc_params[nesting + 1], :crutches) if proc_params[nesting + 1]
|
|
202
|
+
source = replace_lvar(source, proc_params[nesting + 2], :context) if proc_params[nesting + 2]
|
|
192
203
|
|
|
193
204
|
binding_variable_list(source).each do |variable|
|
|
194
205
|
locals.push(proc.binding.eval(variable.to_s))
|
|
@@ -207,13 +218,13 @@ module Chewy
|
|
|
207
218
|
end
|
|
208
219
|
end
|
|
209
220
|
|
|
210
|
-
def
|
|
221
|
+
def extract_lambdas(node)
|
|
211
222
|
return unless node.is_a?(Parser::AST::Node)
|
|
212
223
|
|
|
213
224
|
if node.type == :block && node.children[0].type == :send && node.children[0].to_a == [nil, :lambda]
|
|
214
225
|
[node.children[2]]
|
|
215
226
|
else
|
|
216
|
-
node.children.map { |child|
|
|
227
|
+
node.children.map { |child| extract_lambdas(child) }.flatten.compact
|
|
217
228
|
end
|
|
218
229
|
end
|
|
219
230
|
|
data/lib/chewy/index/wrapper.rb
CHANGED
data/lib/chewy/index.rb
CHANGED
|
@@ -11,6 +11,7 @@ require 'chewy/index/settings'
|
|
|
11
11
|
require 'chewy/index/specification'
|
|
12
12
|
require 'chewy/index/syncer'
|
|
13
13
|
require 'chewy/index/witchcraft'
|
|
14
|
+
require 'chewy/index/compiled'
|
|
14
15
|
require 'chewy/index/wrapper'
|
|
15
16
|
|
|
16
17
|
module Chewy
|
|
@@ -32,6 +33,7 @@ module Chewy
|
|
|
32
33
|
include Observe
|
|
33
34
|
include Crutch
|
|
34
35
|
include Witchcraft
|
|
36
|
+
include Compiled
|
|
35
37
|
include Wrapper
|
|
36
38
|
|
|
37
39
|
singleton_class.delegate :client, to: 'Chewy'
|
data/lib/chewy/multi_search.rb
CHANGED
|
@@ -11,7 +11,7 @@ module Chewy
|
|
|
11
11
|
# Instantiate a new MultiSearch instance.
|
|
12
12
|
#
|
|
13
13
|
# @param queries [Array<Chewy::Search::Request>]
|
|
14
|
-
# @option [Elasticsearch::
|
|
14
|
+
# @option [Elasticsearch::Client] :client (Chewy.client)
|
|
15
15
|
# The Elasticsearch client that should be used for issuing requests.
|
|
16
16
|
def initialize(queries, client: Chewy.client)
|
|
17
17
|
@client = client
|
data/lib/chewy/rake_helper.rb
CHANGED
|
@@ -21,6 +21,8 @@ module Chewy
|
|
|
21
21
|
|
|
22
22
|
DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze
|
|
23
23
|
FALSE_VALUES = %w[0 f false off].freeze
|
|
24
|
+
TRUE_VALUES = %w[1 t true on yes].freeze
|
|
25
|
+
UNBOUNDED_VALUES = %w[unbounded].freeze
|
|
24
26
|
|
|
25
27
|
class << self
|
|
26
28
|
# Performs zero-downtime reindexing of all documents for the specified indexes
|
|
@@ -105,7 +107,7 @@ module Chewy
|
|
|
105
107
|
indexes_from(only: only, except: except).each_with_object([]) do |index, updated_indexes|
|
|
106
108
|
if index.exists?
|
|
107
109
|
output.puts "Updating #{index}"
|
|
108
|
-
index.import(parallel: parallel)
|
|
110
|
+
index.import(parallel: parallel, progressbar: progressbar_option)
|
|
109
111
|
updated_indexes.push(index)
|
|
110
112
|
else
|
|
111
113
|
output.puts "Skipping #{index}, it does not exists (use rake chewy:reset[#{index.derivable_name}] to create and update it)"
|
|
@@ -336,7 +338,22 @@ module Chewy
|
|
|
336
338
|
|
|
337
339
|
def reset_one(index, output, parallel: false)
|
|
338
340
|
output.puts "Resetting #{index}"
|
|
339
|
-
index.reset!((Time.now.to_f * 1000).round, parallel: parallel, apply_journal: journal_exists
|
|
341
|
+
index.reset!((Time.now.to_f * 1000).round, parallel: parallel, apply_journal: journal_exists?, progressbar: progressbar_option)
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
def progressbar_option
|
|
345
|
+
value = ENV.fetch('PROGRESS', nil)
|
|
346
|
+
return false if value.nil? || value.empty?
|
|
347
|
+
|
|
348
|
+
case value.downcase
|
|
349
|
+
when *FALSE_VALUES then false
|
|
350
|
+
when *UNBOUNDED_VALUES then :unbounded
|
|
351
|
+
when *TRUE_VALUES then true
|
|
352
|
+
else
|
|
353
|
+
warn "PROGRESS=#{value.inspect} not recognized; treating as enabled. " \
|
|
354
|
+
"Use #{TRUE_VALUES.join('/')}, #{UNBOUNDED_VALUES.join('/')}, or #{FALSE_VALUES.join('/')}."
|
|
355
|
+
true
|
|
356
|
+
end
|
|
340
357
|
end
|
|
341
358
|
|
|
342
359
|
def warn_missing_index(output)
|
data/lib/chewy/rspec/helpers.rb
CHANGED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
module Chewy
|
|
2
|
+
module Search
|
|
3
|
+
class Parameters
|
|
4
|
+
# Just a standard hash storage. Nothing to see here.
|
|
5
|
+
#
|
|
6
|
+
# @see Chewy::Search::Parameters::HashStorage
|
|
7
|
+
# @see Chewy::Search::Request#runtime_mappings
|
|
8
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime-search-request.html
|
|
9
|
+
class RuntimeMappings < Storage
|
|
10
|
+
include HashStorage
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
data/lib/chewy/search/request.rb
CHANGED
|
@@ -18,14 +18,14 @@ module Chewy
|
|
|
18
18
|
include Scoping
|
|
19
19
|
include Scrolling
|
|
20
20
|
UNDEFINED = Class.new.freeze
|
|
21
|
-
EVERFIELDS = %w[_index
|
|
21
|
+
EVERFIELDS = %w[_index _id _routing].freeze
|
|
22
22
|
DELEGATED_METHODS = %i[
|
|
23
23
|
query filter post_filter knn order reorder docvalue_fields
|
|
24
24
|
track_scores track_total_hits request_cache explain version profile
|
|
25
25
|
search_type preference limit offset terminate_after
|
|
26
26
|
timeout min_score source stored_fields search_after
|
|
27
27
|
load script_fields suggest aggs aggregations collapse none
|
|
28
|
-
indices_boost rescore highlight total total_count
|
|
28
|
+
indices_boost rescore highlight runtime_mappings total total_count
|
|
29
29
|
total_entries indices types delete_all count exists?
|
|
30
30
|
exist? find pluck scroll_batches scroll_hits
|
|
31
31
|
scroll_results scroll_wrappers ignore_unavailable
|
|
@@ -656,7 +656,23 @@ module Chewy
|
|
|
656
656
|
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/highlighting.html
|
|
657
657
|
# @param value [Hash]
|
|
658
658
|
# @return [Chewy::Search::Request]
|
|
659
|
-
|
|
659
|
+
#
|
|
660
|
+
# @!method runtime_mappings(value)
|
|
661
|
+
# Add a `runtime_mappings` part to the request. Further
|
|
662
|
+
# call values are merged to the storage hash.
|
|
663
|
+
#
|
|
664
|
+
# @example
|
|
665
|
+
# PlacesIndex
|
|
666
|
+
# .runtime_mappings(field1: {type: "keyword", script: {lang: "painless", source: "emit('some script here')"}})
|
|
667
|
+
# .runtime_mappings(field2: {type: "keyword", script: {lang: "painless", source: "emit('some script here')"}})
|
|
668
|
+
# # => <PlacesIndex::Query {..., :body=>{:runtime_mappings=>{
|
|
669
|
+
# # "field1"=>{:type=>"keyword", :script=>{:lang=>"painless", :source=>"emit('some script here')"}},
|
|
670
|
+
# # "field2"=>{:type=>"keyword", :script=>{:lang=>"painless", :source=>"emit('some script here')"}}}}}>
|
|
671
|
+
# @see Chewy::Search::Parameters::RuntimeMappings
|
|
672
|
+
# @see https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime-search-request.html
|
|
673
|
+
# @param value [Hash]
|
|
674
|
+
# @return [Chewy::Search::Request]
|
|
675
|
+
%i[script_fields indices_boost rescore highlight runtime_mappings].each do |name|
|
|
660
676
|
define_method name do |value|
|
|
661
677
|
modify(name) { update!(value) }
|
|
662
678
|
end
|
|
@@ -936,7 +952,7 @@ module Chewy
|
|
|
936
952
|
|
|
937
953
|
# Returns and array of values for specified fields.
|
|
938
954
|
# Uses `source` to restrict the list of returned fields.
|
|
939
|
-
# Fields `_id`, `
|
|
955
|
+
# Fields `_id`, `_routing` and `_index` are also supported.
|
|
940
956
|
#
|
|
941
957
|
# @overload pluck(field)
|
|
942
958
|
# If single field is passed - it returns and array of values.
|
|
@@ -29,20 +29,28 @@ module Chewy
|
|
|
29
29
|
|
|
30
30
|
result = perform(size: batch_size, scroll: scroll)
|
|
31
31
|
total = [raw_limit_value, result.fetch('hits', {}).fetch('total', {}).fetch('value', 0)].compact.min
|
|
32
|
+
|
|
33
|
+
total_batches = total / batch_size
|
|
32
34
|
last_batch_size = total % batch_size
|
|
33
|
-
|
|
35
|
+
|
|
36
|
+
total_batches += 1 if last_batch_size != 0
|
|
37
|
+
|
|
34
38
|
scroll_id = nil
|
|
35
39
|
|
|
36
|
-
|
|
40
|
+
total_batches.times do |batch_counter|
|
|
41
|
+
last_run = total_batches - 1 == batch_counter
|
|
42
|
+
|
|
37
43
|
hits = result.fetch('hits', {}).fetch('hits', [])
|
|
38
|
-
|
|
39
|
-
|
|
44
|
+
hits = hits.first(last_batch_size) if last_run && last_batch_size != 0
|
|
45
|
+
|
|
46
|
+
raise Chewy::MissingHitsInScrollError if hits.empty?
|
|
47
|
+
|
|
40
48
|
yield(hits) if hits.present?
|
|
41
49
|
scroll_id = result['_scroll_id']
|
|
42
50
|
|
|
43
|
-
break if result['terminated_early']
|
|
51
|
+
break if result['terminated_early']
|
|
44
52
|
|
|
45
|
-
result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
|
|
53
|
+
result = perform_scroll(scroll: scroll, scroll_id: scroll_id) unless last_run
|
|
46
54
|
end
|
|
47
55
|
ensure
|
|
48
56
|
Chewy.client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
|
data/lib/chewy/stash.rb
CHANGED
|
@@ -38,17 +38,21 @@ module Chewy
|
|
|
38
38
|
|
|
39
39
|
# Selects all the journal entries for the specified indices.
|
|
40
40
|
#
|
|
41
|
+
# Uses a single `terms` filter rather than a chain of `bool.should`
|
|
42
|
+
# clauses so the query depth stays constant regardless of how many
|
|
43
|
+
# indices are passed. Avoids hitting the Elasticsearch
|
|
44
|
+
# `indices.query.bool.max_nested_depth` limit (default 30) when
|
|
45
|
+
# cleaning or applying journals across many indices.
|
|
46
|
+
#
|
|
41
47
|
# @param indices [Chewy::Index, Array<Chewy::Index>]
|
|
42
48
|
def self.for(*something)
|
|
43
49
|
something = something.flatten.compact
|
|
50
|
+
return all if something.empty?
|
|
51
|
+
|
|
44
52
|
indexes = something.flat_map { |s| Chewy.derive_name(s) }
|
|
45
|
-
return none if
|
|
53
|
+
return none if indexes.blank?
|
|
46
54
|
|
|
47
|
-
|
|
48
|
-
indexes.each do |index|
|
|
49
|
-
scope = scope.or(filter(term: {index_name: index.derivable_name}))
|
|
50
|
-
end
|
|
51
|
-
scope
|
|
55
|
+
filter(terms: {index_name: indexes.map(&:derivable_name).uniq})
|
|
52
56
|
end
|
|
53
57
|
|
|
54
58
|
default_import_options journal: false
|
|
@@ -13,7 +13,7 @@ module Chewy
|
|
|
13
13
|
local timechunks_key = prefix .. ":" .. type .. ":timechunks"
|
|
14
14
|
|
|
15
15
|
-- Get timechunk_keys with scores less than or equal to the specified score
|
|
16
|
-
local timechunk_keys = redis.call('
|
|
16
|
+
local timechunk_keys = redis.call('zrange', timechunks_key, '-inf', score, 'byscore')
|
|
17
17
|
|
|
18
18
|
-- Get all members from the sets associated with the timechunk_keys
|
|
19
19
|
local members = {}
|
data/lib/chewy/version.rb
CHANGED