chewy 8.0.1 → 8.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,79 @@
1
+ module Chewy
2
+ class Index
3
+ module Import
4
+ # Thin wrapper around `ruby-progressbar` for import feedback.
5
+ #
6
+ # Unlike the original PR #787 implementation, this wrapper is only
7
+ # touched from the parent process: serial imports increment it directly,
8
+ # and parallel imports increment it via `Parallel`'s `finish:` callback
9
+ # (which runs in the parent under an internal mutex). The workers stay
10
+ # process-based, so there is no GVL contention as in PR #787 / #800.
11
+ #
12
+ # `Progressbar.build` returns a NULL object when the feature is disabled,
13
+ # so call sites do not need feature guards.
14
+ class Progressbar
15
+ NULL = Object.new
16
+ class << NULL
17
+ def increment(_); end
18
+ def total=(_); end
19
+ def finish; end
20
+ end
21
+
22
+ BOUNDED_FORMAT = '%t |%B| %p%% %c/%C %e'.freeze
23
+ UNBOUNDED_FORMAT = '%t %c (%a)'.freeze
24
+ TITLE = 'Importing'.freeze
25
+
26
+ # @param enabled [Boolean, :unbounded] feature flag. `:unbounded` shows
27
+ # a spinner with no total (skip `import_count`).
28
+ # @param total [Integer, nil] expected total; ignored when `:unbounded`.
29
+ # @return [Progressbar, NULL]
30
+ def self.build(enabled, total)
31
+ return NULL unless enabled
32
+
33
+ unless '::ProgressBar'.safe_constantize
34
+ raise 'The `ruby-progressbar` gem is required for import progress, ' \
35
+ "please add `gem 'ruby-progressbar'` to your Gemfile"
36
+ end
37
+
38
+ return new if enabled == :unbounded
39
+
40
+ new(normalize_total(total))
41
+ end
42
+
43
+ # Some ActiveRecord scopes (e.g., `.group(...)`) make `.count` return a
44
+ # Hash rather than an Integer. Coerce so we still get a usable total.
45
+ def self.normalize_total(total)
46
+ case total
47
+ when Hash then total.values.sum
48
+ when Integer then total
49
+ end
50
+ end
51
+
52
+ attr_reader :bar
53
+
54
+ def initialize(total = nil)
55
+ format = total ? BOUNDED_FORMAT : UNBOUNDED_FORMAT
56
+ @bar = ::ProgressBar.create(title: TITLE, total: total, format: format)
57
+ end
58
+
59
+ # Clamps to total when bounded — action_objects may include :delete
60
+ # entries (parent-child re-indexing, delete_if scope) that aren't
61
+ # counted by `adapter.import_count`, which would otherwise raise
62
+ # ProgressBar::InvalidProgressError.
63
+ def increment(by)
64
+ target = bar.progress + by
65
+ target = [bar.total, target].min if bar.total
66
+ bar.progress = target
67
+ end
68
+
69
+ def total=(value)
70
+ bar.total = value
71
+ end
72
+
73
+ def finish
74
+ bar.finish unless bar.finished?
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
@@ -31,7 +31,7 @@ module Chewy
31
31
 
32
32
  DEFAULT_OPTIONS = {
33
33
  refresh: true,
34
- update_fields: [],
34
+ update_fields: nil,
35
35
  update_failover: true,
36
36
  batch_size: Chewy::Index::Adapter::Base::BATCH_SIZE
37
37
  }.freeze
@@ -56,6 +56,7 @@ module Chewy
56
56
  {}
57
57
  end
58
58
  end
59
+ @context = @options[:context] || {}
59
60
  @errors = []
60
61
  @stats = {}
61
62
  @leftovers = []
@@ -78,7 +79,7 @@ module Chewy
78
79
  # @param delete [Array<Object>] any acceptable objects for deleting
79
80
  # @return [true, false] the result of the request, true if no errors
80
81
  def process(index: [], delete: [])
81
- bulk_builder = BulkBuilder.new(@index, to_index: index, delete: delete, fields: @options[:update_fields])
82
+ bulk_builder = BulkBuilder.new(@index, to_index: index, delete: delete, fields: @options[:update_fields], context: @context)
82
83
  bulk_body = bulk_builder.bulk_body
83
84
 
84
85
  if @options[:journal]
@@ -2,6 +2,7 @@ require 'chewy/index/import/journal_builder'
2
2
  require 'chewy/index/import/bulk_builder'
3
3
  require 'chewy/index/import/bulk_request'
4
4
  require 'chewy/index/import/routine'
5
+ require 'chewy/index/import/progressbar'
5
6
 
6
7
  module Chewy
7
8
  class Index
@@ -11,20 +12,22 @@ module Chewy
11
12
  IMPORT_WORKER = lambda do |index, options, total, ids, iteration|
12
13
  ::Process.setproctitle("chewy [#{index}]: import data (#{iteration + 1}/#{total})")
13
14
  routine = Routine.new(index, **options)
15
+ processed = 0
14
16
  index.adapter.import(*ids, routine.options) do |action_objects|
15
17
  routine.process(**action_objects)
18
+ processed += action_objects.sum { |_, v| v.size }
16
19
  end
17
- {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
20
+ {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers, processed: processed}
18
21
  end
19
22
 
20
23
  LEFTOVERS_WORKER = lambda do |index, options, total, body, iteration|
21
24
  ::Process.setproctitle("chewy [#{index}]: import leftovers (#{iteration + 1}/#{total})")
22
25
  routine = Routine.new(index, **options)
23
26
  routine.perform_bulk(body)
24
- routine.errors
27
+ {errors: routine.errors}
25
28
  end
26
29
 
27
- module ClassMethods
30
+ module ClassMethods # rubocop:disable Metrics/ModuleLength
28
31
  # @!method import(*collection, **options)
29
32
  # Basically, one of the main methods for an index. Performs any objects import
30
33
  # to the index. Does all the objects handling routines.
@@ -68,9 +71,15 @@ module Chewy
68
71
  # @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default
69
72
  # @option options [Boolean] direct_import skips object reloading in ORM adapter, `false` by default
70
73
  # @option options [true, false] journal enables imported objects journaling, false by default
71
- # @option options [Array<Symbol, String>] update_fields list of fields for the partial import, empty by default
74
+ # @option options [Array<Symbol, String>] update_fields list of fields for partial import. `nil` (default) triggers full document reindex;
75
+ # an empty array (`[]`) is an explicit no-op (no fields updated).
72
76
  # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default
73
77
  # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
78
+ # @option options [true, false, :unbounded] progressbar shows an import progressbar
79
+ # on stderr. `true` precomputes the total via `adapter.import_count` (one extra
80
+ # count query); `:unbounded` shows a spinner without computing the total. Default
81
+ # `false`. Safe in parallel mode: the bar is incremented in the parent process via
82
+ # `Parallel`'s `finish:` callback, workers stay process-based.
74
83
  # @return [true, false] false in case of errors
75
84
  def import(*args)
76
85
  intercept_import_using_strategy(*args).blank?
@@ -115,13 +124,26 @@ module Chewy
115
124
  # @param crutches [Object] optional crutches object; if omitted - a crutch for the single passed object is created as a fallback
116
125
  # @param fields [Array<Symbol>] and array of fields to restrict the generated document
117
126
  # @return [Hash] a JSON-ready hash
118
- def compose(object, crutches = nil, fields: [])
119
- crutches ||= Chewy::Index::Crutch::Crutches.new self, [object]
127
+ def compose(object, crutches = nil, fields: [], context: {})
128
+ crutches ||= Chewy::Index::Crutch::Crutches.new self, [object], context
120
129
 
121
- if witchcraft? && root.children.present?
122
- cauldron(fields: fields).brew(object, crutches)
130
+ # Hot path: the default-fields compiled method has already
131
+ # been built, so skip the per-call availability check and
132
+ # dispatch directly. Bulk reindexing hits this branch for
133
+ # every object after the first.
134
+ return __chewy_compose__(object, crutches, context) if fields.empty? && @compiled_default_ready
135
+
136
+ if compiled_compose_available?(fields)
137
+ if fields.empty?
138
+ @compiled_default_ready = true
139
+ __chewy_compose__(object, crutches, context)
140
+ else
141
+ compiled_compose(object, crutches, context, fields)
142
+ end
143
+ elsif witchcraft? && root.children.present?
144
+ cauldron(fields: fields).brew(object, crutches, context)
123
145
  else
124
- root.compose(object, crutches, fields: fields)
146
+ root.compose(object, crutches, fields: fields, context: context)
125
147
  end
126
148
  end
127
149
 
@@ -175,46 +197,91 @@ module Chewy
175
197
  end
176
198
 
177
199
  def import_linear(objects, routine)
200
+ bar = build_progressbar(routine, objects)
178
201
  ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
179
202
  adapter.import(*objects, routine.options) do |action_objects|
180
203
  routine.process(**action_objects)
204
+ bar.increment(action_objects.sum { |_, v| v.size })
181
205
  end
182
206
  routine.perform_bulk(routine.leftovers)
183
207
  payload[:import] = routine.stats
184
208
  payload[:errors] = payload_errors(routine.errors) if routine.errors.present?
185
209
  payload[:errors]
186
210
  end
211
+ ensure
212
+ bar&.finish
187
213
  end
188
214
 
189
215
  def import_parallel(objects, routine)
190
216
  raise "The `parallel` gem is required for parallel import, please add `gem 'parallel'` to your Gemfile" unless '::Parallel'.safe_constantize
191
217
 
218
+ bar = build_progressbar(routine, objects)
192
219
  ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
193
220
  batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
194
221
 
195
222
  ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
196
223
  results = ::Parallel.map_with_index(
197
224
  batches,
198
- routine.parallel_options,
225
+ parallel_options_with_progress(routine.parallel_options, bar),
199
226
  &IMPORT_WORKER.curry[self, routine.options, batches.size]
200
227
  )
201
228
  ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
202
229
  errors, import, leftovers = process_parallel_import_results(results)
203
-
204
- if leftovers.present?
205
- batches = leftovers.each_slice(routine.options[:batch_size])
206
- results = ::Parallel.map_with_index(
207
- batches,
208
- routine.parallel_options,
209
- &LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
210
- )
211
- errors.concat(results.flatten(1))
212
- end
230
+ errors.concat(process_parallel_leftovers(leftovers, routine)) if leftovers.present?
213
231
 
214
232
  payload[:import] = import
215
233
  payload[:errors] = payload_errors(errors) if errors.present?
216
234
  payload[:errors]
217
235
  end
236
+ ensure
237
+ bar&.finish
238
+ end
239
+
240
+ def process_parallel_leftovers(leftovers, routine)
241
+ batches = leftovers.each_slice(routine.options[:batch_size]).to_a
242
+ results = ::Parallel.map_with_index(
243
+ batches,
244
+ routine.parallel_options,
245
+ &LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
246
+ )
247
+ results.flat_map { |r| r[:errors] }
248
+ end
249
+
250
+ # Builds Parallel options with a `finish:` callback that increments the
251
+ # progressbar after each worker batch returns. The callback runs in the
252
+ # parent (main) thread under Parallel's internal mutex (parallel-1.x),
253
+ # so workers stay process-based and there is no worker-side
254
+ # synchronization — the regression that triggered the PR #800 revert.
255
+ #
256
+ # If the caller already supplied a `finish:` callback in
257
+ # `parallel_options`, both run; user callback first, then the bar.
258
+ def parallel_options_with_progress(parallel_options, bar)
259
+ user_finish = parallel_options[:finish]
260
+ progress = lambda do |item, i, result|
261
+ user_finish&.call(item, i, result)
262
+ bar.increment(result[:processed]) if result.is_a?(Hash) && result[:processed]
263
+ end
264
+ parallel_options.merge(finish: progress)
265
+ end
266
+
267
+ def build_progressbar(routine, objects)
268
+ enabled = routine.options[:progressbar]
269
+ total = enabled == true ? safe_import_count(objects) : nil
270
+ Progressbar.build(enabled, total)
271
+ end
272
+
273
+ # Returns nil when the adapter cannot or should not be counted:
274
+ # missing `import_count` on a custom adapter, a grouped scope that
275
+ # raises, or any unexpected count failure. A nil total makes
276
+ # `Progressbar.new` render a spinner instead of a bounded bar —
277
+ # avoids aborting the import just because the progressbar can't
278
+ # size itself.
279
+ def safe_import_count(objects)
280
+ return nil unless adapter.respond_to?(:import_count)
281
+
282
+ adapter.import_count(*objects)
283
+ rescue StandardError
284
+ nil
218
285
  end
219
286
 
220
287
  def process_parallel_import_results(results)
@@ -122,6 +122,7 @@ module Chewy
122
122
  # end
123
123
  #
124
124
  def field(*args, **options, &block)
125
+ invalidate_compiled_compose! if respond_to?(:invalidate_compiled_compose!)
125
126
  if args.size > 1
126
127
  args.map { |name| field(name, **options) }
127
128
  else
@@ -1,15 +1,3 @@
1
- begin
2
- require 'method_source'
3
- begin
4
- require 'prism'
5
- rescue LoadError
6
- require 'parser/current'
7
- end
8
- require 'unparser'
9
- rescue LoadError
10
- nil
11
- end
12
-
13
1
  module Chewy
14
2
  class Index
15
3
  module Witchcraft
@@ -19,8 +7,30 @@ module Chewy
19
7
  class_attribute :_witchcraft, instance_reader: false, instance_writer: false
20
8
  end
21
9
 
10
+ def self.load_dependencies!
11
+ return if @dependencies_loaded
12
+
13
+ require 'method_source'
14
+ begin
15
+ require 'prism'
16
+ rescue LoadError
17
+ require 'parser/current'
18
+ end
19
+ require 'unparser'
20
+ @dependencies_loaded = true
21
+ rescue LoadError
22
+ nil
23
+ end
24
+
22
25
  module ClassMethods
23
26
  def witchcraft!
27
+ warn(
28
+ '[DEPRECATION] Chewy::Index.witchcraft! is deprecated and will be removed in a future release. ' \
29
+ 'The compiled compose path is now the default and delivers equivalent performance without ' \
30
+ "method_source/parser/unparser dependencies. Remove the `witchcraft!` call from #{name || self}.",
31
+ uplevel: 1
32
+ )
33
+ Witchcraft.load_dependencies!
24
34
  self._witchcraft = true
25
35
  check_requirements!
26
36
  end
@@ -59,15 +69,15 @@ module Chewy
59
69
  @fields = fields
60
70
  end
61
71
 
62
- def brew(object, crutches = nil)
63
- alicorn.call(locals, object, crutches).as_json
72
+ def brew(object, crutches = nil, context = {})
73
+ alicorn.call(locals, object, crutches, context).as_json
64
74
  end
65
75
 
66
76
  private
67
77
 
68
78
  def alicorn
69
79
  @alicorn ||= singleton_class.class_eval <<-RUBY, __FILE__, __LINE__ + 1
70
- -> (locals, object0, crutches) do
80
+ -> (locals, object0, crutches, context) do
71
81
  #{composed_values(@index.root, 0)}
72
82
  end
73
83
  RUBY
@@ -171,8 +181,8 @@ module Chewy
171
181
  end
172
182
  end
173
183
 
174
- def source_for(proc, nesting)
175
- lambdas = exctract_lambdas(ast_from_proc(proc))
184
+ def source_for(proc, nesting) # rubocop:disable Metrics/AbcSize
185
+ lambdas = extract_lambdas(ast_from_proc(proc))
176
186
 
177
187
  raise "No lambdas found, try to reformat your code:\n`#{proc.source}`" unless lambdas
178
188
 
@@ -189,6 +199,7 @@ module Chewy
189
199
  source = replace_lvar(source, proc_params[n], :"object#{n}") if proc_params[n]
190
200
  end
191
201
  source = replace_lvar(source, proc_params[nesting + 1], :crutches) if proc_params[nesting + 1]
202
+ source = replace_lvar(source, proc_params[nesting + 2], :context) if proc_params[nesting + 2]
192
203
 
193
204
  binding_variable_list(source).each do |variable|
194
205
  locals.push(proc.binding.eval(variable.to_s))
@@ -207,13 +218,13 @@ module Chewy
207
218
  end
208
219
  end
209
220
 
210
- def exctract_lambdas(node)
221
+ def extract_lambdas(node)
211
222
  return unless node.is_a?(Parser::AST::Node)
212
223
 
213
224
  if node.type == :block && node.children[0].type == :send && node.children[0].to_a == [nil, :lambda]
214
225
  [node.children[2]]
215
226
  else
216
- node.children.map { |child| exctract_lambdas(child) }.flatten.compact
227
+ node.children.map { |child| extract_lambdas(child) }.flatten.compact
217
228
  end
218
229
  end
219
230
 
@@ -39,7 +39,7 @@ module Chewy
39
39
  end
40
40
  end
41
41
 
42
- %w[_id _type _index].each do |name|
42
+ %w[_id _index].each do |name|
43
43
  define_method name do
44
44
  _data[name]
45
45
  end
data/lib/chewy/index.rb CHANGED
@@ -11,6 +11,7 @@ require 'chewy/index/settings'
11
11
  require 'chewy/index/specification'
12
12
  require 'chewy/index/syncer'
13
13
  require 'chewy/index/witchcraft'
14
+ require 'chewy/index/compiled'
14
15
  require 'chewy/index/wrapper'
15
16
 
16
17
  module Chewy
@@ -32,6 +33,7 @@ module Chewy
32
33
  include Observe
33
34
  include Crutch
34
35
  include Witchcraft
36
+ include Compiled
35
37
  include Wrapper
36
38
 
37
39
  singleton_class.delegate :client, to: 'Chewy'
@@ -96,7 +96,6 @@ module Chewy
96
96
  'hits' => hits.each_with_index.map do |hit, i|
97
97
  {
98
98
  '_index' => index.index_name,
99
- '_type' => '_doc',
100
99
  '_id' => hit[:id] || (i + 1).to_s,
101
100
  '_score' => 3.14,
102
101
  '_source' => hit
@@ -11,7 +11,7 @@ module Chewy
11
11
  # Instantiate a new MultiSearch instance.
12
12
  #
13
13
  # @param queries [Array<Chewy::Search::Request>]
14
- # @option [Elasticsearch::Transport::Client] :client (Chewy.client)
14
+ # @option [Elasticsearch::Client] :client (Chewy.client)
15
15
  # The Elasticsearch client that should be used for issuing requests.
16
16
  def initialize(queries, client: Chewy.client)
17
17
  @client = client
@@ -21,6 +21,8 @@ module Chewy
21
21
 
22
22
  DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze
23
23
  FALSE_VALUES = %w[0 f false off].freeze
24
+ TRUE_VALUES = %w[1 t true on yes].freeze
25
+ UNBOUNDED_VALUES = %w[unbounded].freeze
24
26
 
25
27
  class << self
26
28
  # Performs zero-downtime reindexing of all documents for the specified indexes
@@ -105,7 +107,7 @@ module Chewy
105
107
  indexes_from(only: only, except: except).each_with_object([]) do |index, updated_indexes|
106
108
  if index.exists?
107
109
  output.puts "Updating #{index}"
108
- index.import(parallel: parallel)
110
+ index.import(parallel: parallel, progressbar: progressbar_option)
109
111
  updated_indexes.push(index)
110
112
  else
111
113
  output.puts "Skipping #{index}, it does not exists (use rake chewy:reset[#{index.derivable_name}] to create and update it)"
@@ -336,7 +338,22 @@ module Chewy
336
338
 
337
339
  def reset_one(index, output, parallel: false)
338
340
  output.puts "Resetting #{index}"
339
- index.reset!((Time.now.to_f * 1000).round, parallel: parallel, apply_journal: journal_exists?)
341
+ index.reset!((Time.now.to_f * 1000).round, parallel: parallel, apply_journal: journal_exists?, progressbar: progressbar_option)
342
+ end
343
+
344
+ def progressbar_option
345
+ value = ENV.fetch('PROGRESS', nil)
346
+ return false if value.nil? || value.empty?
347
+
348
+ case value.downcase
349
+ when *FALSE_VALUES then false
350
+ when *UNBOUNDED_VALUES then :unbounded
351
+ when *TRUE_VALUES then true
352
+ else
353
+ warn "PROGRESS=#{value.inspect} not recognized; treating as enabled. " \
354
+ "Use #{TRUE_VALUES.join('/')}, #{UNBOUNDED_VALUES.join('/')}, or #{FALSE_VALUES.join('/')}."
355
+ true
356
+ end
340
357
  end
341
358
 
342
359
  def warn_missing_index(output)
@@ -39,7 +39,6 @@ module Chewy
39
39
  'hits' => hits.each_with_index.map do |hit, i|
40
40
  {
41
41
  '_index' => index.index_name,
42
- '_type' => '_doc',
43
42
  '_id' => (i + 1).to_s,
44
43
  '_score' => 3.14,
45
44
  '_source' => hit
@@ -0,0 +1,14 @@
1
+ module Chewy
2
+ module Search
3
+ class Parameters
4
+ # Just a standard hash storage. Nothing to see here.
5
+ #
6
+ # @see Chewy::Search::Parameters::HashStorage
7
+ # @see Chewy::Search::Request#runtime_mappings
8
+ # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime-search-request.html
9
+ class RuntimeMappings < Storage
10
+ include HashStorage
11
+ end
12
+ end
13
+ end
14
+ end
@@ -18,14 +18,14 @@ module Chewy
18
18
  include Scoping
19
19
  include Scrolling
20
20
  UNDEFINED = Class.new.freeze
21
- EVERFIELDS = %w[_index _type _id _parent _routing].freeze
21
+ EVERFIELDS = %w[_index _id _routing].freeze
22
22
  DELEGATED_METHODS = %i[
23
23
  query filter post_filter knn order reorder docvalue_fields
24
24
  track_scores track_total_hits request_cache explain version profile
25
25
  search_type preference limit offset terminate_after
26
26
  timeout min_score source stored_fields search_after
27
27
  load script_fields suggest aggs aggregations collapse none
28
- indices_boost rescore highlight total total_count
28
+ indices_boost rescore highlight runtime_mappings total total_count
29
29
  total_entries indices types delete_all count exists?
30
30
  exist? find pluck scroll_batches scroll_hits
31
31
  scroll_results scroll_wrappers ignore_unavailable
@@ -656,7 +656,23 @@ module Chewy
656
656
  # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/highlighting.html
657
657
  # @param value [Hash]
658
658
  # @return [Chewy::Search::Request]
659
- %i[script_fields indices_boost rescore highlight].each do |name|
659
+ #
660
+ # @!method runtime_mappings(value)
661
+ # Add a `runtime_mappings` part to the request. Further
662
+ # call values are merged to the storage hash.
663
+ #
664
+ # @example
665
+ # PlacesIndex
666
+ # .runtime_mappings(field1: {type: "keyword", script: {lang: "painless", source: "emit('some script here')"}})
667
+ # .runtime_mappings(field2: {type: "keyword", script: {lang: "painless", source: "emit('some script here')"}})
668
+ # # => <PlacesIndex::Query {..., :body=>{:runtime_mappings=>{
669
+ # # "field1"=>{:type=>"keyword", :script=>{:lang=>"painless", :source=>"emit('some script here')"}},
670
+ # # "field2"=>{:type=>"keyword", :script=>{:lang=>"painless", :source=>"emit('some script here')"}}}}}>
671
+ # @see Chewy::Search::Parameters::RuntimeMappings
672
+ # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime-search-request.html
673
+ # @param value [Hash]
674
+ # @return [Chewy::Search::Request]
675
+ %i[script_fields indices_boost rescore highlight runtime_mappings].each do |name|
660
676
  define_method name do |value|
661
677
  modify(name) { update!(value) }
662
678
  end
@@ -936,7 +952,7 @@ module Chewy
936
952
 
937
953
  # Returns and array of values for specified fields.
938
954
  # Uses `source` to restrict the list of returned fields.
939
- # Fields `_id`, `_type`, `_routing` and `_index` are also supported.
955
+ # Fields `_id`, `_routing` and `_index` are also supported.
940
956
  #
941
957
  # @overload pluck(field)
942
958
  # If single field is passed - it returns and array of values.
@@ -29,20 +29,28 @@ module Chewy
29
29
 
30
30
  result = perform(size: batch_size, scroll: scroll)
31
31
  total = [raw_limit_value, result.fetch('hits', {}).fetch('total', {}).fetch('value', 0)].compact.min
32
+
33
+ total_batches = total / batch_size
32
34
  last_batch_size = total % batch_size
33
- fetched = 0
35
+
36
+ total_batches += 1 if last_batch_size != 0
37
+
34
38
  scroll_id = nil
35
39
 
36
- loop do
40
+ total_batches.times do |batch_counter|
41
+ last_run = total_batches - 1 == batch_counter
42
+
37
43
  hits = result.fetch('hits', {}).fetch('hits', [])
38
- fetched += hits.size
39
- hits = hits.first(last_batch_size) if last_batch_size != 0 && fetched >= total
44
+ hits = hits.first(last_batch_size) if last_run && last_batch_size != 0
45
+
46
+ raise Chewy::MissingHitsInScrollError if hits.empty?
47
+
40
48
  yield(hits) if hits.present?
41
49
  scroll_id = result['_scroll_id']
42
50
 
43
- break if result['terminated_early'] || fetched >= total
51
+ break if result['terminated_early']
44
52
 
45
- result = perform_scroll(scroll: scroll, scroll_id: scroll_id)
53
+ result = perform_scroll(scroll: scroll, scroll_id: scroll_id) unless last_run
46
54
  end
47
55
  ensure
48
56
  Chewy.client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
data/lib/chewy/stash.rb CHANGED
@@ -38,17 +38,21 @@ module Chewy
38
38
 
39
39
  # Selects all the journal entries for the specified indices.
40
40
  #
41
+ # Uses a single `terms` filter rather than a chain of `bool.should`
42
+ # clauses so the query depth stays constant regardless of how many
43
+ # indices are passed. Avoids hitting the Elasticsearch
44
+ # `indices.query.bool.max_nested_depth` limit (default 30) when
45
+ # cleaning or applying journals across many indices.
46
+ #
41
47
  # @param indices [Chewy::Index, Array<Chewy::Index>]
42
48
  def self.for(*something)
43
49
  something = something.flatten.compact
50
+ return all if something.empty?
51
+
44
52
  indexes = something.flat_map { |s| Chewy.derive_name(s) }
45
- return none if something.present? && indexes.blank?
53
+ return none if indexes.blank?
46
54
 
47
- scope = all
48
- indexes.each do |index|
49
- scope = scope.or(filter(term: {index_name: index.derivable_name}))
50
- end
51
- scope
55
+ filter(terms: {index_name: indexes.map(&:derivable_name).uniq})
52
56
  end
53
57
 
54
58
  default_import_options journal: false
@@ -13,7 +13,7 @@ module Chewy
13
13
  local timechunks_key = prefix .. ":" .. type .. ":timechunks"
14
14
 
15
15
  -- Get timechunk_keys with scores less than or equal to the specified score
16
- local timechunk_keys = redis.call('zrangebyscore', timechunks_key, '-inf', score)
16
+ local timechunk_keys = redis.call('zrange', timechunks_key, '-inf', score, 'byscore')
17
17
 
18
18
  -- Get all members from the sets associated with the timechunk_keys
19
19
  local members = {}
data/lib/chewy/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Chewy
2
- VERSION = '8.0.1'.freeze
2
+ VERSION = '8.3.0'.freeze
3
3
  end
data/lib/chewy.rb CHANGED
@@ -31,6 +31,7 @@ end
31
31
  try_require 'kaminari'
32
32
  try_require 'kaminari/core'
33
33
  try_require 'parallel'
34
+ try_require 'ruby-progressbar'
34
35
 
35
36
  ActiveSupport.on_load(:active_record) do
36
37
  try_require 'kaminari/activerecord'