chewy 7.2.6 → 7.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96329a4913f6308f4c7435cd50b962a36dc63b728b50567ff53ce2cb8011086f
-  data.tar.gz: a25b93955769a85fdf5de68a6d284060a981c2e3f6e10c929b0cd49816c19720
+  metadata.gz: fcd795c985f120c412c315bc0b87dd65e7b55a302aeae1bffbc5c71beb910652
+  data.tar.gz: ad70c5ff382f8405146da2be5643d9ac397d60fbbf16842a509a950cc90acbd2
 SHA512:
-  metadata.gz: 66a3dfa94536e8d7b262a6a0ccfe32ff76ca7431850722eef21b14a11b3d162ad0346c3a469293696ce79916c44db7fb17b56f462fd4083c619b2dde09fa6c29
-  data.tar.gz: 6e15e9fcb103c64a5565add1e5200caa4bf1018d3fc7e9ac79a8db6829a5bb1699ac77910e838d59df2f6f195d019a88c22987d51f1b02ebdbb8f37844cce8d3
+  metadata.gz: 0f4ac34a7c84c9fc6dd2743ab77982b634bddb0ceda0116b2a2bfa9a0b902655b375bc215857002041159a5cd0ce52e9c409342f06ce7a41f0667d3c74c07ab4
+  data.tar.gz: fa68110ccfc6acd942c54b279cfd99e36b980dcb408bd1508430598757c3efd57e8dc1eaa60c2bf77156eb9acba77c700715ac9676af0a36eacb600de730b1b5

data/.github/CODEOWNERS

@@ -1 +1 @@
-.github/workflows @toptal/rogue-one
+.github/workflows @toptal/platform-sre

data/.github/workflows/ruby.yml

@@ -1,6 +1,14 @@
 name: CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [master]
+  pull_request:
+    types: [
+      synchronize, # PR was updated
+      opened, # PR was open
+      reopened # PR was reopened
+    ]
 
 jobs:
   ruby-2:
@@ -34,7 +42,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', 3.1 ]
+        ruby: [ '3.0', '3.1', '3.2' ]
         gemfile: [ rails.6.1.activerecord, rails.7.0.activerecord ]
     name: ${{ matrix.ruby }}-${{ matrix.gemfile }}
 

data/CHANGELOG.md

@@ -8,6 +8,31 @@
 
 ### Bugs Fixed
 
+## 7.3.0 (2023-04-03)
+
+### New Features
+
+* [#869](https://github.com/toptal/chewy/pull/869): New strategy - `delayed_sidekiq`. Allow passing `strategy: :delayed_sidekiq` option to `SomeIndex.import([1, ...], strategy: :delayed_sidekiq)`. The strategy is compatible with `update_fields` option as well. ([@skcc321][])
+* [#879](https://github.com/toptal/chewy/pull/879): Configure CI to check for ruby 3.2 compatibility. ([@konalegi][])
+
+### Changes
+
+### Bugs Fixed
+
+* [#856](https://github.com/toptal/chewy/pull/856): Fix return value of subscribed_task_stats used in rake tasks. ([@fabiormoura][])
+
+## 7.2.7 (2022-11-15)
+
+### New Features
+
+* [#857](https://github.com/toptal/chewy/pull/857): Allow passing `wait_for_completion`, `request_per_second` and `scroll_size` options to `chewy:journal:clean` rake task and `delete_all` query builder method. ([@konalegi][])([@barthez][])
+
+### Changes
+
+### Bugs Fixed
+
+* [#863](https://github.com/toptal/chewy/pull/863): Fix `crutches` call doesn't respect `update_fields` option. ([@skcc321][])
+
 ## 7.2.6 (2022-06-13)
 
 ### New Features

data/README.md

@@ -5,7 +5,7 @@
 
 # Chewy
 
-Chewy is an ODM (Object Document Mapper), built on top of the [the official Elasticsearch client](https://github.com/elastic/elasticsearch-ruby).
+Chewy is an ODM (Object Document Mapper), built on top of [the official Elasticsearch client](https://github.com/elastic/elasticsearch-ruby).
 
 ## Why Chewy?
 
@@ -458,7 +458,7 @@ field :hierarchy_link, type: :join, relations: {question: %i[answer comment], an
 ```
 assuming you have `comment_type` and `commented_id` fields in your model.
 
-Note that when you reindex a parent, it's children and grandchildren will be reindexed as well.
+Note that when you reindex a parent, its children and grandchildren will be reindexed as well.
 This may require additional queries to the primary database and to elastisearch.
 
 Also note that the join field doesn't support crutches (it should be a field directly defined on the model).
@@ -525,7 +525,7 @@ So Chewy Crutches™ technology is able to increase your indexing performance in
 
 ### Witchcraft™ technology
 
-One more experimental technology to increase import performance. As far as you know, chewy defines value proc for every imported field in mapping, so at the import time each of this procs is executed on imported object to extract result document to import. It would be great for performance to use one huge whole-document-returning proc instead. So basically the idea or Witchcraft™ technology is to compile a single document-returning proc from the index definition.
+One more experimental technology to increase import performance. As far as you know, chewy defines value proc for every imported field in mapping, so at the import time each of these procs is executed on imported object to extract result document to import. It would be great for performance to use one huge whole-document-returning proc instead. So basically the idea or Witchcraft™ technology is to compile a single document-returning proc from the index definition.
 
 ```ruby
 index_scope Product
@@ -569,7 +569,7 @@ Obviously not every type of definition might be compiled. There are some restric
   end
   ```
 
-However, it is quite possible that your index definition will be supported by Witchcraft™ technology out of the box in the most of the cases.
+However, it is quite possible that your index definition will be supported by Witchcraft™ technology out of the box in most of the cases.
 
 ### Raw Import
 
@@ -675,7 +675,9 @@ end
 
 You may be wondering why do you need it? The answer is simple: not to lose the data.
 
-Imagine that you reset your index in a zero-downtime manner (to separate index), and at the meantime somebody keeps updating the data frequently (to old index). So all these actions will be written to the journal index and you'll be able to apply them after index reset using the `Chewy::Journal` interface.
+Imagine that you reset your index in a zero-downtime manner (to separate index), and in the meantime somebody keeps updating the data frequently (to old index). So all these actions will be written to the journal index and you'll be able to apply them after index reset using the `Chewy::Journal` interface.
+
+When enabled, journal can grow to enormous size, consider setting up cron job that would clean it occasionally using [`chewy:journal:clean` rake task](#chewyjournal).
 
 ### Index manipulation
 
@@ -772,6 +774,80 @@ The default queue name is `chewy`, you can customize it in settings: `sidekiq.qu
 Chewy.settings[:sidekiq] = {queue: :low}
 ```
 
+#### `:delayed_sidekiq`
+
+It accumulates ids of records to be reindexed during the latency window in redis and then does the reindexing of all accumulated records at once.
+The strategy is very useful in case of frequently mutated records.
+It supports `update_fields` option, so it will try to select just enough data from the DB
+
+There are three options that can be defined in the index:
+```ruby
+class CitiesIndex...
+  strategy_config delayed_sidekiq: {
+    latency: 3,
+    margin: 2,
+    ttl: 60 * 60 * 24,
+    reindex_wrapper: ->(&reindex) {
+      ActiveRecord::Base.connected_to(role: :reading) { reindex.call }
+    }
+    # latency - will prevent scheduling identical jobs
+    # margin - main purpose is to cover db replication lag by the margin
+    # ttl - a chunk expiration time (in seconds)
+    # reindex_wrapper - lambda that accepts block to wrap that reindex process AR connection block.
+  }
+
+  ...
+end
+```
+
+Also you can define defaults in the `initializers/chewy.rb`
+```ruby
+Chewy.settings = {
+  strategy_config: {
+    delayed_sidekiq: {
+      latency: 3,
+      margin: 2,
+      ttl: 60 * 60 * 24,
+      reindex_wrapper: ->(&reindex) {
+        ActiveRecord::Base.connected_to(role: :reading) { reindex.call }
+      }
+    }
+  }
+}
+
+```
+or in `config/chewy.yml`
+```ruby
+  strategy_config:
+    delayed_sidekiq:
+      latency: 3
+      margin: 2
+      ttl: <%= 60 * 60 * 24 %>
+      # reindex_wrapper setting is not possible here!!! use the initializer instead
+```
+
+You can use the strategy identically to other strategies
+```ruby
+Chewy.strategy(:delayed_sidekiq) do
+  City.popular.map(&:do_some_update_action!)
+end
+```
+
+The default queue name is `chewy`, you can customize it in settings: `sidekiq.queue_name`
+```
+Chewy.settings[:sidekiq] = {queue: :low}
+```
+
+Explicit call of the reindex using `:delayed_sidekiq strategy`
+```ruby
+CitiesIndex.import([1, 2, 3], strategy: :delayed_sidekiq)
+```
+
+Explicit call of the reindex using `:delayed_sidekiq` strategy with `:update_fields` support
+```ruby
+CitiesIndex.import([1, 2, 3], update_fields: [:name], strategy: :delayed_sidekiq)
+```
+
 #### `:active_job`
 
 This does the same thing as `:atomic`, but using ActiveJob. This will inherit the ActiveJob configuration settings including the `active_job.queue_adapter` setting for the environment. Patch `Chewy::Strategy::ActiveJob::Worker` for index updates improving.
@@ -886,7 +962,7 @@ Chewy has notifying the following events:
     {index: 30, delete: 5}
     ```
 
-  * `payload[:errors]`: might not exists. Contains grouped errors with objects ids list:
+  * `payload[:errors]`: might not exist. Contains grouped errors with objects ids list:
 
     ```ruby
     {index: {
@@ -1018,7 +1094,7 @@ Request DSL also provides additional scope actions, like `delete_all`, `exists?`
 
 #### Pagination
 
-The request DSL supports pagination with `Kaminari`. An extension is enabled on initializtion if `Kaminari` is available. See [Chewy::Search](lib/chewy/search.rb) and [Chewy::Search::Pagination::Kaminari](lib/chewy/search/pagination/kaminari.rb) for details.
+The request DSL supports pagination with `Kaminari`. An extension is enabled on initialization if `Kaminari` is available. See [Chewy::Search](lib/chewy/search.rb) and [Chewy::Search::Pagination::Kaminari](lib/chewy/search/pagination/kaminari.rb) for details.
 
 #### Named scopes
 
@@ -1144,6 +1220,17 @@ rake chewy:journal:apply["$(date -v-1H -u +%FT%TZ)"] # apply journaled changes f
 rake chewy:journal:apply["$(date -v-1H -u +%FT%TZ)",users] # apply journaled changes for the past hour on UsersIndex only
 ```
 
+When the size of the journal becomes very large, the classical way of deletion would be obstructive and resource consuming. Fortunately, Chewy internally uses [delete-by-query](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/docs-delete-by-query.html#docs-delete-by-query-task-api) ES function which supports async execution with batching and [throttling](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html#docs-delete-by-query-throttle).
+
+The available options, which can be set by ENV variables, are listed below:
+* `WAIT_FOR_COMPLETION` - a boolean flag. It controls async execution. It waits by default. When set to `false` (`0`, `f`, `false` or `off` in any case spelling is accepted as `false`), Elasticsearch performs some preflight checks, launches the request, and returns a task reference you can use to cancel the task or get its status.
+* `REQUESTS_PER_SECOND` - float. The throttle for this request in sub-requests per second. No throttling is enforced by default.
+* `SCROLL_SIZE` - integer. The number of documents to be deleted in single sub-request. The default batch size is 1000.
+
+```bash
+rake chewy:journal:clean WAIT_FOR_COMPLETION=false REQUESTS_PER_SECOND=10 SCROLL_SIZE=5000
+```
+
 ### RSpec integration
 
 Just add `require 'chewy/rspec'` to your spec_helper.rb and you will get additional features:

data/chewy.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec| # rubocop:disable Metrics/BlockLength
 
   spec.add_development_dependency 'database_cleaner'
   spec.add_development_dependency 'elasticsearch-extensions'
+  spec.add_development_dependency 'mock_redis'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec', '>= 3.7.0'
   spec.add_development_dependency 'rspec-collection_matchers'

data/lib/chewy/index/import/bulk_builder.rb

@@ -48,12 +48,11 @@ module Chewy
         def index_entry(object)
           entry = {}
           entry[:_id] = index_object_ids[object] if index_object_ids[object]
+          entry[:routing] = routing(object) if join_field?
 
-          data = data_for(object)
           parent = cache(entry[:_id])
-
-          entry[:routing] = routing(object) if join_field?
-          if parent_changed?(data, parent)
+          data = data_for(object) if parent.present?
+          if parent.present? && parent_changed?(data, parent)
             reindex_entries(object, data) + reindex_descendants(object)
           elsif @fields.present?
             return [] unless entry[:_id]
@@ -61,7 +60,7 @@ module Chewy
             entry[:data] = {doc: data_for(object, fields: @fields)}
             [{update: entry}]
           else
-            entry[:data] = data
+            entry[:data] = data || data_for(object)
             [{index: entry}]
           end
         end

data/lib/chewy/index/import.rb

@@ -73,7 +73,7 @@ module Chewy
         # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
         # @return [true, false] false in case of errors
         ruby2_keywords def import(*args)
-          import_routine(*args).blank?
+          intercept_import_using_strategy(*args).blank?
         end
 
         # @!method import!(*collection, **options)
@@ -84,7 +84,8 @@ module Chewy
         #
         # @raise [Chewy::ImportFailed] in case of errors
         ruby2_keywords def import!(*args)
-          errors = import_routine(*args)
+          errors = intercept_import_using_strategy(*args)
+
           raise Chewy::ImportFailed.new(self, errors) if errors.present?
 
           true
@@ -126,6 +127,32 @@ module Chewy
 
       private
 
+        def intercept_import_using_strategy(*args)
+          args_clone = args.deep_dup
+          options = args_clone.extract_options!
+          strategy = options.delete(:strategy)
+
+          return import_routine(*args) if strategy.blank?
+
+          ids = args_clone.flatten
+          return {} if ids.blank?
+          return {argument: {"#{strategy} supports ids only!" => ids}} unless ids.all? do |id|
+            id.respond_to?(:to_i)
+          end
+
+          case strategy
+          when :delayed_sidekiq
+            begin
+              Chewy::Strategy::DelayedSidekiq::Scheduler.new(self, ids, options).postpone
+              {} # success. errors handling convention
+            rescue StandardError => e
+              {scheduler: {e.message => ids}}
+            end
+          else
+            {argument: {"unsupported strategy: '#{strategy}'" => ids}}
+          end
+        end
+
         def import_routine(*args)
           return if !args.first.nil? && empty_objects_or_scope?(args.first)
 

data/lib/chewy/index.rb

@@ -20,6 +20,10 @@ module Chewy
       pipeline raw_import refresh replication
     ].freeze
 
+    STRATEGY_OPTIONS = {
+      delayed_sidekiq: %i[latency margin ttl reindex_wrapper]
+    }.freeze
+
     include Search
     include Actions
     include Aliases
@@ -221,6 +225,27 @@ module Chewy
         params.assert_valid_keys(IMPORT_OPTIONS_KEYS)
         self._default_import_options = _default_import_options.merge(params)
       end
+
+      def strategy_config(params = {})
+        @strategy_config ||= begin
+          config_struct = Struct.new(*STRATEGY_OPTIONS.keys).new
+
+          STRATEGY_OPTIONS.each_with_object(config_struct) do |(strategy, options), res|
+            res[strategy] = case strategy
+            when :delayed_sidekiq
+              Struct.new(*STRATEGY_OPTIONS[strategy]).new.tap do |config|
+                options.each do |option|
+                  config[option] = params.dig(strategy, option) || Chewy.configuration.dig(:strategy_config, strategy, option)
+                end
+
+                config[:reindex_wrapper] ||= ->(&reindex) { reindex.call } # default wrapper
+              end
+            else
+              raise NotImplementedError, "Unsupported strategy: '#{strategy}'"
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/chewy/journal.rb

@@ -43,8 +43,12 @@ module Chewy
     #
     # @param until_time [Time, DateTime] time to clean up until it
     # @return [Hash] delete_by_query ES API call result
-    def clean(until_time = nil)
-      Chewy::Stash::Journal.clean(until_time, only: @only)
+    def clean(until_time = nil, delete_by_query_options: {})
+      Chewy::Stash::Journal.clean(
+        until_time,
+        only: @only,
+        delete_by_query_options: delete_by_query_options.merge(refresh: false)
+      )
     end
 
   private

data/lib/chewy/rake_helper.rb

@@ -19,6 +19,9 @@ module Chewy
       output.puts "  Applying journal to #{targets}, #{count} entries, stage #{payload[:stage]}"
     end
 
+    DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze
+    FALSE_VALUES = %w[0 f false off].freeze
+
     class << self
       # Performs zero-downtime reindexing of all documents for the specified indexes
       #
@@ -162,7 +165,7 @@ module Chewy
 
         subscribed_task_stats(output) do
           output.puts "Applying journal entries created after #{time}"
-          count = Chewy::Journal.new(indexes_from(only: only, except: except)).apply(time)
+          count = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).apply(time)
           output.puts 'No journal entries were created after the specified time' if count.zero?
         end
       end
@@ -181,12 +184,16 @@ module Chewy
       # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param output [IO] output io for logging
       # @return [Array<Chewy::Index>] indexes that were actually updated
-      def journal_clean(time: nil, only: nil, except: nil, output: $stdout)
+      def journal_clean(time: nil, only: nil, except: nil, delete_by_query_options: {}, output: $stdout)
         subscribed_task_stats(output) do
           output.puts "Cleaning journal entries created before #{time}" if time
-          response = Chewy::Journal.new(indexes_from(only: only, except: except)).clean(time)
-          count = response['deleted'] || response['_indices']['_all']['deleted']
-          output.puts "Cleaned up #{count} journal entries"
+          response = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).clean(time, delete_by_query_options: delete_by_query_options)
+          if response.key?('task')
+            output.puts "Task to cleanup the journal has been created, #{response['task']}"
+          else
+            count = response['deleted'] || response['_indices']['_all']['deleted']
+            output.puts "Cleaned up #{count} journal entries"
+          end
         end
       end
 
@@ -228,6 +235,26 @@ module Chewy
         end
       end
 
+      # Reads options that are required to run journal cleanup asynchronously from ENV hash
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
+      #
+      # @example
+      #   Chewy::RakeHelper.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => 'false','REQUESTS_PER_SECOND' => '10','SCROLL_SIZE' => '5000'})
+      #   # => { wait_for_completion: false, requests_per_second: 10.0, scroll_size: 5000 }
+      #
+      def delete_by_query_options_from_env(env)
+        env
+          .slice(*DELETE_BY_QUERY_OPTIONS)
+          .transform_keys { |k| k.downcase.to_sym }
+          .to_h do |key, value|
+            case key
+            when :wait_for_completion then [key, !FALSE_VALUES.include?(value.downcase)]
+            when :requests_per_second then [key, value.to_f]
+            when :scroll_size then [key, value.to_i]
+            end
+          end
+      end
+
       def normalize_indexes(*identifiers)
         identifiers.flatten(1).map { |identifier| normalize_index(identifier) }
       end
@@ -243,11 +270,18 @@ module Chewy
         ActiveSupport::Notifications.subscribed(JOURNAL_CALLBACK.curry[output], 'apply_journal.chewy') do
           ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy', &block)
         end
+      ensure
         output.puts "Total: #{human_duration(Time.now - start)}"
       end
 
     private
 
+      def journal_indexes_from(only: nil, except: nil)
+        return if Array.wrap(only).empty? && Array.wrap(except).empty?
+
+        indexes_from(only: only, except: except)
+      end
+
       def indexes_from(only: nil, except: nil)
         indexes = if only.present?
           normalize_indexes(Array.wrap(only))

data/lib/chewy/search/request.rb

@@ -962,10 +962,22 @@ module Chewy
       #
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
       # @note The result hash is different for different API used.
-      # @param refresh [true, false] field names
+      # @param refresh [true, false] Refreshes all shards involved in the delete by query
+      # @param wait_for_completion [true, false] wait for request completion or run it asynchronously
+      #    and return task reference at `.tasks/task/${taskId}`.
+      # @param requests_per_second [Float] The throttle for this request in sub-requests per second
+      # @param scroll_size [Integer] Size of the scroll request that powers the operation
+
       # @return [Hash] the result of query execution
-      def delete_all(refresh: true)
-        request_body = only(WHERE_STORAGES).render.merge(refresh: refresh)
+      def delete_all(refresh: true, wait_for_completion: nil, requests_per_second: nil, scroll_size: nil)
+        request_body = only(WHERE_STORAGES).render.merge(
+          {
+            refresh: refresh,
+            wait_for_completion: wait_for_completion,
+            requests_per_second: requests_per_second,
+            scroll_size: scroll_size
+          }.compact
+        )
         ActiveSupport::Notifications.instrument 'delete_query.chewy', notification_payload(request: request_body) do
           request_body[:body] = {query: {match_all: {}}} if request_body[:body].empty?
           Chewy.client.delete_by_query(request_body)

data/lib/chewy/stash.rb

@@ -28,12 +28,12 @@ module Chewy
       # Cleans up all the journal entries until the specified time. If nothing is
       # specified - cleans up everything.
       #
-      # @param since_time [Time, DateTime] the time top boundary
+      # @param until_time [Time, DateTime] Clean everything before that date
       # @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
-      def self.clean(until_time = nil, only: [])
+      def self.clean(until_time = nil, only: [], delete_by_query_options: {})
         scope = self.for(only)
         scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
-        scope.delete_all
+        scope.delete_all(**delete_by_query_options)
       end
 
       # Selects all the journal entries for the specified indices.

data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require_relative '../../index'
+
+# The class is responsible for accumulating in redis [type, ids]
+# that were requested to be reindexed during `latency` seconds.
+# The reindex job is going to be scheduled after a `latency` seconds.
+# that job is going to read accumulated [type, ids] from the redis
+# and reindex all them at once.
+module Chewy
+  class Strategy
+    class DelayedSidekiq
+      require_relative 'worker'
+
+      class Scheduler
+        DEFAULT_TTL = 60 * 60 * 24 # in seconds
+        DEFAULT_LATENCY = 10
+        DEFAULT_MARGIN = 2
+        DEFAULT_QUEUE = 'chewy'
+        KEY_PREFIX = 'chewy:delayed_sidekiq'
+        FALLBACK_FIELDS = 'all'
+        FIELDS_IDS_SEPARATOR = ';'
+        IDS_SEPARATOR = ','
+
+        def initialize(type, ids, options = {})
+          @type = type
+          @ids = ids
+          @options = options
+        end
+
+        # the diagram:
+        #
+        #  inputs:
+        #  latency == 2
+        #  reindex_time = Time.current
+        #
+        #  Parallel OR Sequential triggers of reindex:          |  What is going on in reindex store (Redis):
+        #  --------------------------------------------------------------------------------------------------
+        #                                                       |
+        #  process 1 (reindex_time):                            |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1]
+        #    Schedule.new(CitiesIndex, [1]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & schedule a DelayedSidekiq::Worker at 1679347869 (at + 3)
+        #                                                       |    it will zpop chewy:delayed_sidekiq:timechunks up to 1679347866 score and reindex all ids with zpoped keys
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347866
+        #                                                       |
+        #                                                       |
+        #  process 2 (reindex_time):                            |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2]
+        #    Schedule.new(CitiesIndex, [2]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & do not schedule a new worker
+        #                                                       |
+        #                                                       |
+        #  process 1 (reindex_time + (latency - 1).seconds):    |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
+        #    Schedule.new(CitiesIndex, [3]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & do not schedule a new worker
+        #                                                       |
+        #                                                       |
+        #  process 2 (reindex_time + (latency + 1).seconds):    |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
+        #    Schedule.new(CitiesIndex, [4]).postpone            |  chewy:delayed_sidekiq:CitiesIndex:1679347868 = [4]
+        #                                                       |  chewy:delayed_sidekiq:timechunks = [
+        #                                                       |    { score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}
+        #                                                       |    { score: 1679347868, "chewy:delayed_sidekiq:CitiesIndex:1679347868"}
+        #                                                       |  ]
+        #                                                       |  & schedule a DelayedSidekiq::Worker at 1679347871 (at + 3)
+        #                                                       |    it will zpop chewy:delayed_sidekiq:timechunks up to 1679347868 score and reindex all ids with zpoped keys
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347866 (in case of failed previous reindex),
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347868
+        def postpone
+          ::Sidekiq.redis do |redis|
+            # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
+            if redis.respond_to?(:sadd?)
+              redis.sadd?(timechunk_key, serialize_data)
+            else
+              redis.sadd(timechunk_key, serialize_data)
+            end
+
+            redis.expire(timechunk_key, ttl)
+
+            unless redis.zrank(timechunks_key, timechunk_key)
+              redis.zadd(timechunks_key, at, timechunk_key)
+              redis.expire(timechunks_key, ttl)
+
+              ::Sidekiq::Client.push(
+                'queue' => sidekiq_queue,
+                'at' => at + margin,
+                'class' => Chewy::Strategy::DelayedSidekiq::Worker,
+                'args' => [type_name, at]
+              )
+            end
+          end
+        end
+
+      private
+
+        attr_reader :type, :ids, :options
+
+        # this method returns predictable value that jumps by latency value
+        # another words each latency seconds it return the same value
+        def at
+          @at ||= begin
+            schedule_at = latency.seconds.from_now.to_f
+
+            (schedule_at - (schedule_at % latency)).to_i
+          end
+        end
+
+        def fields
+          options[:update_fields].presence || [FALLBACK_FIELDS]
+        end
+
+        def timechunks_key
+          "#{KEY_PREFIX}:#{type_name}:timechunks"
+        end
+
+        def timechunk_key
+          "#{KEY_PREFIX}:#{type_name}:#{at}"
+        end
+
+        def serialize_data
+          [ids.join(IDS_SEPARATOR), fields.join(IDS_SEPARATOR)].join(FIELDS_IDS_SEPARATOR)
+        end
+
+        def type_name
+          type.name
+        end
+
+        def latency
+          strategy_config.latency || DEFAULT_LATENCY
+        end
+
+        def margin
+          strategy_config.margin || DEFAULT_MARGIN
+        end
+
+        def ttl
+          strategy_config.ttl || DEFAULT_TTL
+        end
+
+        def sidekiq_queue
+          Chewy.settings.dig(:sidekiq, :queue) || DEFAULT_QUEUE
+        end
+
+        def strategy_config
+          type.strategy_config.delayed_sidekiq
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/delayed_sidekiq/worker.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class DelayedSidekiq
+      class Worker
+        include ::Sidekiq::Worker
+
+        def perform(type, score, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+
+          ::Sidekiq.redis do |redis|
+            timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks"
+            timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score)
+            members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact
+
+            # extract ids and fields & do the reset of records
+            ids, fields = extract_ids_and_fields(members)
+            options[:update_fields] = fields if fields
+
+            index = type.constantize
+            index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
+              options.any? ? index.import!(ids, **options) : index.import!(ids)
+            end
+
+            redis.del(timechunk_keys)
+            redis.zremrangebyscore(timechunks_key, -1, score)
+          end
+        end
+
+      private
+
+        def extract_ids_and_fields(members)
+          ids = []
+          fields = []
+
+          members.each do |member|
+            member_ids, member_fields = member.split(Scheduler::FIELDS_IDS_SEPARATOR).map do |v|
+              v.split(Scheduler::IDS_SEPARATOR)
+            end
+            ids |= member_ids
+            fields |= member_fields
+          end
+
+          fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS)
+
+          [ids.map(&:to_i), fields]
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/delayed_sidekiq.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class DelayedSidekiq < Sidekiq
+      require_relative 'delayed_sidekiq/scheduler'
+
+      def leave
+        @stash.each do |type, ids|
+          next if ids.empty?
+
+          DelayedSidekiq::Scheduler.new(type, ids).postpone
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy.rb

@@ -8,6 +8,7 @@ begin
   require 'sidekiq'
   require 'chewy/strategy/sidekiq'
   require 'chewy/strategy/lazy_sidekiq'
+  require 'chewy/strategy/delayed_sidekiq'
 rescue LoadError
   nil
 end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.2.6'.freeze
+  VERSION = '7.3.0'.freeze
 end

data/lib/tasks/chewy.rake

@@ -94,7 +94,13 @@ namespace :chewy do
 
     desc 'Removes journal records created before the specified timestamp for the specified indexes/types or all of them'
     task clean: :environment do |_task, args|
-      Chewy::RakeHelper.journal_clean(**parse_journal_args(args.extras))
+      delete_options = Chewy::RakeHelper.delete_by_query_options_from_env(ENV)
+      Chewy::RakeHelper.journal_clean(
+        [
+          parse_journal_args(args.extras),
+          {delete_by_query_options: delete_options}
+        ].reduce({}, :merge)
+      )
     end
   end
 end

data/spec/chewy/index/import/bulk_builder_spec.rb

@@ -62,6 +62,8 @@ describe Chewy::Index::Import::BulkBuilder do
         let(:to_index) { cities.first(2) }
         let(:delete) { [cities.last] }
         specify do
+          expect(subject).to receive(:data_for).with(cities.first).and_call_original
+          expect(subject).to receive(:data_for).with(cities.second).and_call_original
           expect(subject.bulk_body).to eq([
             {index: {_id: 1, data: {'name' => 'City17', 'rating' => 42}}},
             {index: {_id: 2, data: {'name' => 'City18', 'rating' => 42}}},
@@ -72,6 +74,8 @@ describe Chewy::Index::Import::BulkBuilder do
         context ':fields' do
           let(:fields) { %w[name] }
           specify do
+            expect(subject).to receive(:data_for).with(cities.first, fields: [:name]).and_call_original
+            expect(subject).to receive(:data_for).with(cities.second, fields: [:name]).and_call_original
             expect(subject.bulk_body).to eq([
               {update: {_id: 1, data: {doc: {'name' => 'City17'}}}},
               {update: {_id: 2, data: {doc: {'name' => 'City18'}}}},

data/spec/chewy/rake_helper_spec.rb

@@ -426,6 +426,33 @@ Total: \\d+s\\Z
       described_class.journal_clean(except: CitiesIndex, output: output)
       expect(output.string).to match(Regexp.new(<<-OUTPUT, Regexp::MULTILINE))
 \\ACleaned up 1 journal entries
+Total: \\d+s\\Z
+      OUTPUT
+    end
+
+    it 'executes asynchronously' do
+      output = StringIO.new
+      expect(Chewy.client).to receive(:delete_by_query).with(
+        {
+          body: {query: {match_all: {}}},
+          index: ['chewy_journal'],
+          refresh: false,
+          requests_per_second: 10.0,
+          scroll_size: 200,
+          wait_for_completion: false
+        }
+      ).and_call_original
+      described_class.journal_clean(
+        output: output,
+        delete_by_query_options: {
+          wait_for_completion: false,
+          requests_per_second: 10.0,
+          scroll_size: 200
+        }
+      )
+
+      expect(output.string).to match(Regexp.new(<<-OUTPUT, Regexp::MULTILINE))
+\\ATask to cleanup the journal has been created, [^\\n]*
 Total: \\d+s\\Z
       OUTPUT
     end
@@ -502,4 +529,52 @@ Total: \\d+s\\Z
       end
     end
   end
+
+  describe '.delete_by_query_options_from_env' do
+    subject(:options) { described_class.delete_by_query_options_from_env(env) }
+    let(:env) do
+      {
+        'WAIT_FOR_COMPLETION' => 'false',
+        'REQUESTS_PER_SECOND' => '10',
+        'SCROLL_SIZE' => '5000'
+      }
+    end
+
+    it 'parses the options' do
+      expect(options).to eq(
+        wait_for_completion: false,
+        requests_per_second: 10.0,
+        scroll_size: 5000
+      )
+    end
+
+    context 'with different boolean values' do
+      it 'parses the option correctly' do
+        %w[1 t true TRUE on ON].each do |v|
+          expect(described_class.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => v}))
+            .to eq(wait_for_completion: true)
+        end
+
+        %w[0 f false FALSE off OFF].each do |v|
+          expect(described_class.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => v}))
+            .to eq(wait_for_completion: false)
+        end
+      end
+    end
+
+    context 'with other env' do
+      let(:env) { {'SOME_ENV' => '123', 'REQUESTS_PER_SECOND' => '15'} }
+
+      it 'parses only the options' do
+        expect(options).to eq(requests_per_second: 15.0)
+      end
+    end
+  end
+
+  describe '.subscribed_task_stats' do
+    specify do
+      block_output = described_class.subscribed_task_stats(StringIO.new) { 'expected output' }
+      expect(block_output).to eq('expected output')
+    end
+  end
 end

data/spec/chewy/search/request_spec.rb

@@ -817,6 +817,31 @@ describe Chewy::Search::Request do
           request: {index: ['products'], body: {query: {match: {name: 'name3'}}}, refresh: false}
         )
       end
+
+      it 'delete records asynchronously' do
+        outer_payload = nil
+        ActiveSupport::Notifications.subscribe('delete_query.chewy') do |_name, _start, _finish, _id, payload|
+          outer_payload = payload
+        end
+        subject.query(match: {name: 'name3'}).delete_all(
+          refresh: false,
+          wait_for_completion: false,
+          requests_per_second: 10.0,
+          scroll_size: 2000
+        )
+        expect(outer_payload).to eq(
+          index: ProductsIndex,
+          indexes: [ProductsIndex],
+          request: {
+            index: ['products'],
+            body: {query: {match: {name: 'name3'}}},
+            refresh: false,
+            wait_for_completion: false,
+            requests_per_second: 10.0,
+            scroll_size: 2000
+          }
+        )
+      end
     end
 
     describe '#response=' do

data/spec/chewy/strategy/delayed_sidekiq_spec.rb

@@ -0,0 +1,190 @@
+require 'spec_helper'
+
+if defined?(Sidekiq)
+  require 'sidekiq/testing'
+  require 'mock_redis'
+
+  describe Chewy::Strategy::DelayedSidekiq do
+    around do |example|
+      Chewy.strategy(:bypass) { example.run }
+    end
+
+    before do
+      redis = MockRedis.new
+      allow(Sidekiq).to receive(:redis).and_yield(redis)
+      Sidekiq::Worker.clear_all
+    end
+
+    before do
+      stub_model(:city) do
+        update_index('cities') { self }
+      end
+
+      stub_index(:cities) do
+        index_scope City
+      end
+    end
+
+    let(:city) { City.create!(name: 'hello') }
+    let(:other_city) { City.create!(name: 'world') }
+
+    it 'does not trigger immediate reindex due to it`s async nature' do
+      expect { [city, other_city].map(&:save!) }
+        .not_to update_index(CitiesIndex, strategy: :delayed_sidekiq)
+    end
+
+    it "respects 'refresh: false' options" do
+      allow(Chewy).to receive(:disable_refresh_async).and_return(true)
+      expect(CitiesIndex).to receive(:import!).with([city.id, other_city.id], refresh: false)
+      scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id, other_city.id])
+      scheduler.postpone
+      Chewy::Strategy::DelayedSidekiq::Worker.drain
+    end
+
+    context 'with default config' do
+      it 'does schedule a job that triggers reindex with default options' do
+        Timecop.freeze do
+          expect(Sidekiq::Client).to receive(:push).with(
+            hash_including(
+              'queue' => 'chewy',
+              'at' => (Time.current.to_i.ceil(-1) + 2.seconds).to_i,
+              'class' => Chewy::Strategy::DelayedSidekiq::Worker,
+              'args' => ['CitiesIndex', an_instance_of(Integer)]
+            )
+          ).and_call_original
+
+          expect($stdout).not_to receive(:puts)
+
+          Sidekiq::Testing.inline! do
+            expect { [city, other_city].map(&:save!) }
+              .to update_index(CitiesIndex, strategy: :delayed_sidekiq)
+              .and_reindex(city, other_city).only
+          end
+        end
+      end
+    end
+
+    context 'with custom config' do
+      before do
+        CitiesIndex.strategy_config(
+          delayed_sidekiq: {
+            reindex_wrapper: lambda { |&reindex|
+              puts 'hello'
+              reindex.call
+            },
+            margin: 5,
+            latency: 60
+          }
+        )
+      end
+
+      it 'respects :strategy_config options' do
+        Timecop.freeze do
+          expect(Sidekiq::Client).to receive(:push).with(
+            hash_including(
+              'queue' => 'chewy',
+              'at' => (60.seconds.from_now.change(sec: 0) + 5.seconds).to_i,
+              'class' => Chewy::Strategy::DelayedSidekiq::Worker,
+              'args' => ['CitiesIndex', an_instance_of(Integer)]
+            )
+          ).and_call_original
+
+          expect($stdout).to receive(:puts).with('hello') # check that reindex_wrapper works
+
+          Sidekiq::Testing.inline! do
+            expect { [city, other_city].map(&:save!) }
+              .to update_index(CitiesIndex, strategy: :delayed_sidekiq)
+              .and_reindex(city, other_city).only
+          end
+        end
+      end
+    end
+
+    context 'two reindex call within the timewindow' do
+      it 'accumulates all ids does the reindex one time' do
+        Timecop.freeze do
+          expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once
+          scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id])
+          scheduler.postpone
+          scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
+          scheduler.postpone
+          Chewy::Strategy::DelayedSidekiq::Worker.drain
+        end
+      end
+
+      context 'one call with update_fields another one without update_fields' do
+        it 'does reindex of all fields' do
+          Timecop.freeze do
+            expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once
+            scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
+            scheduler.postpone
+            scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
+            scheduler.postpone
+            Chewy::Strategy::DelayedSidekiq::Worker.drain
+          end
+        end
+      end
+
+      context 'both calls with different update fields' do
+        it 'deos reindex with union of fields' do
+          Timecop.freeze do
+            expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id], update_fields: %w[description name]).once
+            scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
+            scheduler.postpone
+            scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description'])
+            scheduler.postpone
+            Chewy::Strategy::DelayedSidekiq::Worker.drain
+          end
+        end
+      end
+    end
+
+    context 'two calls within different timewindows' do
+      it 'does two separate reindexes' do
+        Timecop.freeze do
+          expect(CitiesIndex).to receive(:import!).with([city.id]).once
+          expect(CitiesIndex).to receive(:import!).with([other_city.id]).once
+          Timecop.travel(20.seconds.ago) do
+            scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id])
+            scheduler.postpone
+          end
+          scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
+          scheduler.postpone
+          Chewy::Strategy::DelayedSidekiq::Worker.drain
+        end
+      end
+    end
+
+    context 'first call has update_fields' do
+      it 'does first reindex with the expected update_fields and second without update_fields' do
+        Timecop.freeze do
+          expect(CitiesIndex).to receive(:import!).with([city.id], update_fields: ['name']).once
+          expect(CitiesIndex).to receive(:import!).with([other_city.id]).once
+          Timecop.travel(20.seconds.ago) do
+            scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
+            scheduler.postpone
+          end
+          scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
+          scheduler.postpone
+          Chewy::Strategy::DelayedSidekiq::Worker.drain
+        end
+      end
+    end
+
+    context 'both calls have update_fields option' do
+      it 'does both reindexes with their expected update_fields option' do
+        Timecop.freeze do
+          expect(CitiesIndex).to receive(:import!).with([city.id], update_fields: ['name']).once
+          expect(CitiesIndex).to receive(:import!).with([other_city.id], update_fields: ['description']).once
+          Timecop.travel(20.seconds.ago) do
+            scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
+            scheduler.postpone
+          end
+          scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description'])
+          scheduler.postpone
+          Chewy::Strategy::DelayedSidekiq::Worker.drain
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: chewy
 version: !ruby/object:Gem::Version
-  version: 7.2.6
+  version: 7.3.0
 platform: ruby
 authors:
 - Toptal, LLC
 - pyromaniac
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-13 00:00:00.000000000 Z
+date: 2023-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: database_cleaner
@@ -39,6 +39,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mock_redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -344,6 +358,9 @@ files:
 - lib/chewy/strategy/atomic_no_refresh.rb
 - lib/chewy/strategy/base.rb
 - lib/chewy/strategy/bypass.rb
+- lib/chewy/strategy/delayed_sidekiq.rb
+- lib/chewy/strategy/delayed_sidekiq/scheduler.rb
+- lib/chewy/strategy/delayed_sidekiq/worker.rb
 - lib/chewy/strategy/lazy_sidekiq.rb
 - lib/chewy/strategy/sidekiq.rb
 - lib/chewy/strategy/urgent.rb
@@ -437,6 +454,7 @@ files:
 - spec/chewy/strategy/active_job_spec.rb
 - spec/chewy/strategy/atomic_no_refresh_spec.rb
 - spec/chewy/strategy/atomic_spec.rb
+- spec/chewy/strategy/delayed_sidekiq_spec.rb
 - spec/chewy/strategy/lazy_sidekiq_spec.rb
 - spec/chewy/strategy/sidekiq_spec.rb
 - spec/chewy/strategy_spec.rb
@@ -449,7 +467,7 @@ homepage: https://github.com/toptal/chewy
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -464,8 +482,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.29
-signing_key: 
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: Elasticsearch ODM client wrapper
 test_files:
@@ -554,6 +572,7 @@ test_files:
 - spec/chewy/strategy/active_job_spec.rb
 - spec/chewy/strategy/atomic_no_refresh_spec.rb
 - spec/chewy/strategy/atomic_spec.rb
+- spec/chewy/strategy/delayed_sidekiq_spec.rb
 - spec/chewy/strategy/lazy_sidekiq_spec.rb
 - spec/chewy/strategy/sidekiq_spec.rb
 - spec/chewy/strategy_spec.rb