chewy 8.1.0 → 8.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 702dbe73f6b523156348491049ed540490d755d66c3242a2422c4da92090346c
-  data.tar.gz: 730beaf500e9d4feeafc5033bd028abc67662f9ae3299a95aa5d3f323f6ee011
+  metadata.gz: 4797e925ff275cb6b742ef6dd7be3a03d6033f73c93453305d08383c11252c00
+  data.tar.gz: 46b165cfeda3df1558221bc668a7749b1c98ec8a937e4956f710dfbe93934cb1
 SHA512:
-  metadata.gz: 87105fad14e8959fabbad411e3b22af813b6261c0cff1a45f14205819e5564f03d936acf6fcd909ac152d98325c2619628c3732fa1eb628d6edb42a5b9670a95
-  data.tar.gz: b9f693b6e4180e9c2023456439e1de74c86dff2a453f50be369c50f30f7b419eab75173f2071707cf922457dc69fb9fc26cf0e82ad79034e3eea79212091a791
+  metadata.gz: 6451084c18ae23ee908f0ba6c00cffa2cc5abcec29ecc4c911f7e8340081f4b0b89c66d91f2c9dda988c412bcd5a995705ff219d00147c4d52209749f375ff84
+  data.tar.gz: b2dd7dda7bae223e450165ff35be42ea814b57d8f06cd0546970a0aed3bed113f290ccc233724248a0cdce3af7b9ef8ac60d2e0e155b1dc6f19376375ccb1990

data/CHANGELOG.md

@@ -8,6 +8,23 @@
 
 ### Changes
 
+## 8.2.0 (2026-05-29)
+
+### New Features
+
+* Add `progressbar:` option to `import`/`import!` and a `PROGRESS=1` rake env toggle for `chewy:reset` / `chewy:update`. The bar is opt-in (default `false`), supports a `:unbounded` spinner mode that skips the `import_count` query, and is safe in parallel mode — workers stay process-based, the bar is incremented in the parent via `Parallel`'s `finish:` callback. Reintroduces the feature originally added in [#787](https://github.com/toptal/chewy/pull/787) and reverted in [#800](https://github.com/toptal/chewy/pull/800) without the GVL regression.
+
+### Bug Fixes
+
+* Fix race condition during `reset!` where the unsuffixed concrete index could be recreated by a concurrent process between the delete and the alias creation, causing the alias creation to fail with an index/alias name collision.
+  The delete + alias-add are now performed in a single atomic `_aliases` cluster state update via the `remove_index` action.
+  No public API or layout change.
+* [#992](https://github.com/toptal/chewy/issues/992): `import(update_fields: [])` is now a no-op (zero fields to update) instead of triggering a full document reindex. The default of `update_fields` is now `nil` (still performs a full reindex). Behavior change: any caller passing an explicit empty array previously got a silent full reimport; they will now skip the update.
+
+### Changes
+
+* [#1024](https://github.com/toptal/chewy/pull/1024): Replace deprecated `ZRANGEBYSCORE` with `ZRANGE ... BYSCORE` in the `delayed_sidekiq` worker Lua script. `ZRANGEBYSCORE` has been deprecated in Redis since 6.2.0.
+
 ## 8.1.0 (2026-05-28)
 
 ### New Features

data/lib/chewy/index/actions.rb

@@ -7,12 +7,13 @@ module Chewy
       extend ActiveSupport::Concern
 
       module ClassMethods
-        # Checks index existance. Returns true or false
+        # Checks index existance. Supports suffixes. Returns true or false
         #
         #   UsersIndex.exists? #=> true
+        #   UsersIndex.exists?('11-2024') #=> false
         #
-        def exists?
-          client.indices.exists(index: index_name)
+        def exists?(suffix = nil)
+          client.indices.exists(index: index_name(suffix: suffix))
         end
 
         # Creates index and applies mappings and settings.
@@ -163,16 +164,17 @@ module Chewy
             ))
             original_index_settings suffixed_name
 
-            delete if indexes.blank?
-            client.indices.update_aliases body: {actions: [
-              *indexes.map do |index|
-                {remove: {index: index, alias: general_name}}
-              end,
-              {add: {index: suffixed_name, alias: general_name}}
-            ]}
+            actions = indexes.map { |index| {remove: {index: index, alias: general_name}} }
+            actions << {add: {index: suffixed_name, alias: general_name}}
+            if indexes.blank? && exists?
+              index_names = client.indices.get_alias(index: general_name).keys
+              actions << {remove_index: {index: index_names.join(',')}}
+            end
+
+            client.indices.update_aliases body: {actions: actions}
             client.indices.delete index: indexes if indexes.present?
 
-            self.journal.apply(start_time, **import_options) if apply_journal
+            self.journal.apply(start_time, **import_options.except(:progressbar)) if apply_journal
             result
           else
             purge!

data/lib/chewy/index/adapter/object.rb

@@ -146,6 +146,23 @@ module Chewy
           collection.each_slice(options[:batch_size], &block)
         end
 
+        # Returns the count of objects that would be imported. Used by the
+        # progressbar feature to set the total. Mirrors {#import_args} input
+        # handling but does not enumerate batches.
+        #
+        # @return [Integer]
+        def import_count(*args)
+          args = args.dup
+          args.extract_options!
+          collection = if args.empty? && @target.respond_to?(import_all_method)
+            @target.send(import_all_method)
+          else
+            args.flatten(1).compact
+          end
+
+          collection.count
+        end
+
         # This method is used internally by the request DSL when the
         # collection of ORM/ODM objects is requested.
         #

data/lib/chewy/index/adapter/orm.rb

@@ -96,6 +96,26 @@ module Chewy
         end
         alias_method :import_references, :import_fields
 
+        # Returns the count of records that would be imported. Used by the
+        # progressbar feature to set the total. Accepts the same shapes as
+        # {#import}: nothing (uses default scope), a relation, or an array
+        # of ids/objects.
+        #
+        # @return [Integer]
+        def import_count(*args)
+          args = args.dup
+          args.extract_options!
+          collection = if args.empty?
+            default_scope
+          elsif args.first.is_a?(relation_class)
+            args.first
+          else
+            args.flatten.compact
+          end
+
+          collection.count
+        end
+
         def load(ids, **options)
           scope = all_scope_where_ids_in(ids)
           additional_scope = options[options[:_index].to_sym].try(:[], :scope) || options[:scope]

data/lib/chewy/index/import/bulk_builder.rb

@@ -13,11 +13,11 @@ module Chewy
         # @param to_index [Array<Object>] objects to index
         # @param delete [Array<Object>] objects or ids to delete
         # @param fields [Array<Symbol, String>] and array of fields for documents update
-        def initialize(index, to_index: [], delete: [], fields: [], context: {})
+        def initialize(index, to_index: [], delete: [], fields: nil, context: {})
           @index = index
           @to_index = to_index
           @delete = delete
-          @fields = fields.map!(&:to_sym)
+          @fields = fields&.map(&:to_sym)
           @context = context
         end
 
@@ -47,6 +47,8 @@ module Chewy
         end
 
         def index_entry(object)
+          return [] if @fields&.empty?
+
           entry = {}
           entry[:_id] = index_object_ids[object] if index_object_ids[object]
           entry[:routing] = routing(object) if join_field?
@@ -264,7 +266,7 @@ module Chewy
         def parent_changed?(data, old_parent)
           return false unless old_parent
           return false unless join_field?
-          return false unless @fields.include?(join_field.to_sym)
+          return false unless @fields&.include?(join_field.to_sym)
           return false unless data.key?(join_field)
 
           # The join field value can be a hash, e.g.:

data/lib/chewy/index/import/progressbar.rb

@@ -0,0 +1,79 @@
+module Chewy
+  class Index
+    module Import
+      # Thin wrapper around `ruby-progressbar` for import feedback.
+      #
+      # Unlike the original PR #787 implementation, this wrapper is only
+      # touched from the parent process: serial imports increment it directly,
+      # and parallel imports increment it via `Parallel`'s `finish:` callback
+      # (which runs in the parent under an internal mutex). The workers stay
+      # process-based, so there is no GVL contention as in PR #787 / #800.
+      #
+      # `Progressbar.build` returns a NULL object when the feature is disabled,
+      # so call sites do not need feature guards.
+      class Progressbar
+        NULL = Object.new
+        class << NULL
+          def increment(_); end
+          def total=(_); end
+          def finish; end
+        end
+
+        BOUNDED_FORMAT = '%t |%B| %p%% %c/%C %e'.freeze
+        UNBOUNDED_FORMAT = '%t %c (%a)'.freeze
+        TITLE = 'Importing'.freeze
+
+        # @param enabled [Boolean, :unbounded] feature flag. `:unbounded` shows
+        #   a spinner with no total (skip `import_count`).
+        # @param total [Integer, nil] expected total; ignored when `:unbounded`.
+        # @return [Progressbar, NULL]
+        def self.build(enabled, total)
+          return NULL unless enabled
+
+          unless '::ProgressBar'.safe_constantize
+            raise 'The `ruby-progressbar` gem is required for import progress, ' \
+                  "please add `gem 'ruby-progressbar'` to your Gemfile"
+          end
+
+          return new if enabled == :unbounded
+
+          new(normalize_total(total))
+        end
+
+        # Some ActiveRecord scopes (e.g., `.group(...)`) make `.count` return a
+        # Hash rather than an Integer. Coerce so we still get a usable total.
+        def self.normalize_total(total)
+          case total
+          when Hash then total.values.sum
+          when Integer then total
+          end
+        end
+
+        attr_reader :bar
+
+        def initialize(total = nil)
+          format = total ? BOUNDED_FORMAT : UNBOUNDED_FORMAT
+          @bar = ::ProgressBar.create(title: TITLE, total: total, format: format)
+        end
+
+        # Clamps to total when bounded — action_objects may include :delete
+        # entries (parent-child re-indexing, delete_if scope) that aren't
+        # counted by `adapter.import_count`, which would otherwise raise
+        # ProgressBar::InvalidProgressError.
+        def increment(by)
+          target = bar.progress + by
+          target = [bar.total, target].min if bar.total
+          bar.progress = target
+        end
+
+        def total=(value)
+          bar.total = value
+        end
+
+        def finish
+          bar.finish unless bar.finished?
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/import/routine.rb

@@ -31,7 +31,7 @@ module Chewy
 
         DEFAULT_OPTIONS = {
           refresh: true,
-          update_fields: [],
+          update_fields: nil,
           update_failover: true,
           batch_size: Chewy::Index::Adapter::Base::BATCH_SIZE
         }.freeze

data/lib/chewy/index/import.rb

@@ -2,6 +2,7 @@ require 'chewy/index/import/journal_builder'
 require 'chewy/index/import/bulk_builder'
 require 'chewy/index/import/bulk_request'
 require 'chewy/index/import/routine'
+require 'chewy/index/import/progressbar'
 
 module Chewy
   class Index
@@ -11,20 +12,22 @@ module Chewy
       IMPORT_WORKER = lambda do |index, options, total, ids, iteration|
         ::Process.setproctitle("chewy [#{index}]: import data (#{iteration + 1}/#{total})")
         routine = Routine.new(index, **options)
+        processed = 0
         index.adapter.import(*ids, routine.options) do |action_objects|
           routine.process(**action_objects)
+          processed += action_objects.sum { |_, v| v.size }
         end
-        {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
+        {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers, processed: processed}
       end
 
       LEFTOVERS_WORKER = lambda do |index, options, total, body, iteration|
         ::Process.setproctitle("chewy [#{index}]: import leftovers (#{iteration + 1}/#{total})")
         routine = Routine.new(index, **options)
         routine.perform_bulk(body)
-        routine.errors
+        {errors: routine.errors}
       end
 
-      module ClassMethods
+      module ClassMethods # rubocop:disable Metrics/ModuleLength
         # @!method import(*collection, **options)
         # Basically, one of the main methods for an index. Performs any objects import
         # to the index. Does all the objects handling routines.
@@ -68,9 +71,15 @@ module Chewy
         # @option options [Integer] batch_size passed to the adapter import method, used to split imported objects in chunks, 1000 by default
         # @option options [Boolean] direct_import skips object reloading in ORM adapter, `false` by default
         # @option options [true, false] journal enables imported objects journaling, false by default
-        # @option options [Array<Symbol, String>] update_fields list of fields for the partial import, empty by default
+        # @option options [Array<Symbol, String>] update_fields list of fields for partial import. `nil` (default) triggers full document reindex;
+        #   an empty array (`[]`) is an explicit no-op (no fields updated).
         # @option options [true, false] update_failover enables full objects reimport in cases of partial update errors, `true` by default
         # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
+        # @option options [true, false, :unbounded] progressbar shows an import progressbar
+        #   on stderr. `true` precomputes the total via `adapter.import_count` (one extra
+        #   count query); `:unbounded` shows a spinner without computing the total. Default
+        #   `false`. Safe in parallel mode: the bar is incremented in the parent process via
+        #   `Parallel`'s `finish:` callback, workers stay process-based.
         # @return [true, false] false in case of errors
         def import(*args)
           intercept_import_using_strategy(*args).blank?
@@ -175,46 +184,91 @@ module Chewy
         end
 
         def import_linear(objects, routine)
+          bar = build_progressbar(routine, objects)
           ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
             adapter.import(*objects, routine.options) do |action_objects|
               routine.process(**action_objects)
+              bar.increment(action_objects.sum { |_, v| v.size })
             end
             routine.perform_bulk(routine.leftovers)
             payload[:import] = routine.stats
             payload[:errors] = payload_errors(routine.errors) if routine.errors.present?
             payload[:errors]
           end
+        ensure
+          bar&.finish
         end
 
         def import_parallel(objects, routine)
           raise "The `parallel` gem is required for parallel import, please add `gem 'parallel'` to your Gemfile" unless '::Parallel'.safe_constantize
 
+          bar = build_progressbar(routine, objects)
           ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
             batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
 
             ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
             results = ::Parallel.map_with_index(
               batches,
-              routine.parallel_options,
+              parallel_options_with_progress(routine.parallel_options, bar),
               &IMPORT_WORKER.curry[self, routine.options, batches.size]
             )
             ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
             errors, import, leftovers = process_parallel_import_results(results)
-
-            if leftovers.present?
-              batches = leftovers.each_slice(routine.options[:batch_size])
-              results = ::Parallel.map_with_index(
-                batches,
-                routine.parallel_options,
-                &LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
-              )
-              errors.concat(results.flatten(1))
-            end
+            errors.concat(process_parallel_leftovers(leftovers, routine)) if leftovers.present?
 
             payload[:import] = import
             payload[:errors] = payload_errors(errors) if errors.present?
             payload[:errors]
           end
+        ensure
+          bar&.finish
+        end
+
+        def process_parallel_leftovers(leftovers, routine)
+          batches = leftovers.each_slice(routine.options[:batch_size]).to_a
+          results = ::Parallel.map_with_index(
+            batches,
+            routine.parallel_options,
+            &LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
+          )
+          results.flat_map { |r| r[:errors] }
+        end
+
+        # Builds Parallel options with a `finish:` callback that increments the
+        # progressbar after each worker batch returns. The callback runs in the
+        # parent (main) thread under Parallel's internal mutex (parallel-1.x),
+        # so workers stay process-based and there is no worker-side
+        # synchronization — the regression that triggered the PR #800 revert.
+        #
+        # If the caller already supplied a `finish:` callback in
+        # `parallel_options`, both run; user callback first, then the bar.
+        def parallel_options_with_progress(parallel_options, bar)
+          user_finish = parallel_options[:finish]
+          progress = lambda do |item, i, result|
+            user_finish&.call(item, i, result)
+            bar.increment(result[:processed]) if result.is_a?(Hash) && result[:processed]
+          end
+          parallel_options.merge(finish: progress)
+        end
+
+        def build_progressbar(routine, objects)
+          enabled = routine.options[:progressbar]
+          total = enabled == true ? safe_import_count(objects) : nil
+          Progressbar.build(enabled, total)
+        end
+
+        # Returns nil when the adapter cannot or should not be counted:
+        # missing `import_count` on a custom adapter, a grouped scope that
+        # raises, or any unexpected count failure. A nil total makes
+        # `Progressbar.new` render a spinner instead of a bounded bar —
+        # avoids aborting the import just because the progressbar can't
+        # size itself.
+        def safe_import_count(objects)
+          return nil unless adapter.respond_to?(:import_count)
+
+          adapter.import_count(*objects)
+        rescue StandardError
+          nil
         end
 
         def process_parallel_import_results(results)

data/lib/chewy/rake_helper.rb

@@ -21,6 +21,8 @@ module Chewy
 
     DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze
     FALSE_VALUES = %w[0 f false off].freeze
+    TRUE_VALUES = %w[1 t true on yes].freeze
+    UNBOUNDED_VALUES = %w[unbounded].freeze
 
     class << self
       # Performs zero-downtime reindexing of all documents for the specified indexes
@@ -105,7 +107,7 @@ module Chewy
           indexes_from(only: only, except: except).each_with_object([]) do |index, updated_indexes|
             if index.exists?
               output.puts "Updating #{index}"
-              index.import(parallel: parallel)
+              index.import(parallel: parallel, progressbar: progressbar_option)
               updated_indexes.push(index)
             else
               output.puts "Skipping #{index}, it does not exists (use rake chewy:reset[#{index.derivable_name}] to create and update it)"
@@ -336,7 +338,22 @@ module Chewy
 
       def reset_one(index, output, parallel: false)
         output.puts "Resetting #{index}"
-        index.reset!((Time.now.to_f * 1000).round, parallel: parallel, apply_journal: journal_exists?)
+        index.reset!((Time.now.to_f * 1000).round, parallel: parallel, apply_journal: journal_exists?, progressbar: progressbar_option)
+      end
+
+      def progressbar_option
+        value = ENV.fetch('PROGRESS', nil)
+        return false if value.nil? || value.empty?
+
+        case value.downcase
+        when *FALSE_VALUES then false
+        when *UNBOUNDED_VALUES then :unbounded
+        when *TRUE_VALUES then true
+        else
+          warn "PROGRESS=#{value.inspect} not recognized; treating as enabled. " \
+               "Use #{TRUE_VALUES.join('/')}, #{UNBOUNDED_VALUES.join('/')}, or #{FALSE_VALUES.join('/')}."
+          true
+        end
       end
 
       def warn_missing_index(output)

data/lib/chewy/strategy/delayed_sidekiq/worker.rb

@@ -13,7 +13,7 @@ module Chewy
           local timechunks_key = prefix .. ":" .. type .. ":timechunks"
 
           -- Get timechunk_keys with scores less than or equal to the specified score
-          local timechunk_keys = redis.call('zrangebyscore', timechunks_key, '-inf', score)
+          local timechunk_keys = redis.call('zrange', timechunks_key, '-inf', score, 'byscore')
 
           -- Get all members from the sets associated with the timechunk_keys
           local members = {}

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '8.1.0'.freeze
+  VERSION = '8.2.0'.freeze
 end

data/lib/chewy.rb

@@ -31,6 +31,7 @@ end
 try_require 'kaminari'
 try_require 'kaminari/core'
 try_require 'parallel'
+try_require 'ruby-progressbar'
 
 ActiveSupport.on_load(:active_record) do
   try_require 'kaminari/activerecord'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chewy
 version: !ruby/object:Gem::Version
-  version: 8.1.0
+  version: 8.2.0
 platform: ruby
 authors:
 - Toptal, LLC
@@ -88,6 +88,7 @@ files:
 - lib/chewy/index/import/bulk_builder.rb
 - lib/chewy/index/import/bulk_request.rb
 - lib/chewy/index/import/journal_builder.rb
+- lib/chewy/index/import/progressbar.rb
 - lib/chewy/index/import/routine.rb
 - lib/chewy/index/mapping.rb
 - lib/chewy/index/observe.rb