chewy 7.2.1 → 7.6.0

data/lib/chewy/index/import.rb

@@ -2,19 +2,17 @@ require 'chewy/index/import/journal_builder'
 require 'chewy/index/import/bulk_builder'
 require 'chewy/index/import/bulk_request'
 require 'chewy/index/import/routine'
-require 'chewy/index/import/thread_safe_progress_bar'
 
 module Chewy
   class Index
     module Import
       extend ActiveSupport::Concern
 
-      IMPORT_WORKER = lambda do |index, options, total, progress_bar, ids, iteration|
+      IMPORT_WORKER = lambda do |index, options, total, ids, iteration|
         ::Process.setproctitle("chewy [#{index}]: import data (#{iteration + 1}/#{total})")
         routine = Routine.new(index, **options)
         index.adapter.import(*ids, routine.options) do |action_objects|
           routine.process(**action_objects)
-          progress_bar.increment(action_objects.map { |_, v| v.size }.sum) if routine.options[:progressbar]
         end
         {errors: routine.errors, import: routine.stats, leftovers: routine.leftovers}
       end
@@ -38,8 +36,7 @@ module Chewy
         # passed objects from the index if they are not in the default scope
         # or marked for destruction.
         #
-        # It handles parent-child relationships: if the object parent_id has been
-        # changed it destroys the object and recreates it from scratch.
+        # It handles parent-child relationships with a join field reindexing children when the parent is reindexed.
         #
         # Performs journaling if enabled: it stores all the ids of the imported
         # objects to a specialized index. It is possible to replay particular import
@@ -76,7 +73,7 @@ module Chewy
         # @option options [true, Integer, Hash] parallel enables parallel import processing with the Parallel gem, accepts the number of workers or any Parallel gem acceptable options
         # @return [true, false] false in case of errors
         def import(*args)
-          import_routine(*args).blank?
+          intercept_import_using_strategy(*args).blank?
         end
 
         # @!method import!(*collection, **options)
@@ -87,7 +84,8 @@ module Chewy
         #
         # @raise [Chewy::ImportFailed] in case of errors
         def import!(*args)
-          errors = import_routine(*args)
+          errors = intercept_import_using_strategy(*args)
+
           raise Chewy::ImportFailed.new(self, errors) if errors.present?
 
           true
@@ -129,6 +127,32 @@ module Chewy
 
       private
 
+        def intercept_import_using_strategy(*args)
+          args_clone = args.deep_dup
+          options = args_clone.extract_options!
+          strategy = options.delete(:strategy)
+
+          return import_routine(*args) if strategy.blank?
+
+          ids = args_clone.flatten
+          return {} if ids.blank?
+          return {argument: {"#{strategy} supports ids only!" => ids}} unless ids.all? do |id|
+            id.respond_to?(:to_i)
+          end
+
+          case strategy
+          when :delayed_sidekiq
+            begin
+              Chewy::Strategy::DelayedSidekiq::Scheduler.new(self, ids, options).postpone
+              {} # success. errors handling convention
+            rescue StandardError => e
+              {scheduler: {e.message => ids}}
+            end
+          else
+            {argument: {"unsupported strategy: '#{strategy}'" => ids}}
+          end
+        end
+
         def import_routine(*args)
           return if !args.first.nil? && empty_objects_or_scope?(args.first)
 
@@ -152,10 +176,8 @@ module Chewy
 
         def import_linear(objects, routine)
           ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
-            progress_bar = ThreadSafeProgressBar.new(routine.options[:progressbar]) { adapter.import_count(objects) }
             adapter.import(*objects, routine.options) do |action_objects|
               routine.process(**action_objects)
-              progress_bar.increment(action_objects.map { |_, v| v.size }.sum) if routine.options[:progressbar]
             end
             routine.perform_bulk(routine.leftovers)
             payload[:import] = routine.stats
@@ -170,20 +192,24 @@ module Chewy
           ActiveSupport::Notifications.instrument 'import_objects.chewy', index: self do |payload|
             batches = adapter.import_references(*objects, routine.options.slice(:batch_size)).to_a
 
-            progress_bar = ThreadSafeProgressBar.new(routine.options[:progressbar]) { adapter.import_count(objects) }
             ::ActiveRecord::Base.connection.close if defined?(::ActiveRecord::Base)
-
-            results = ::Parallel.map_with_index(batches, routine.parallel_options) do |ids, index|
-              progress_bar.wait_until_ready
-              ActiveRecord::Base.connection_pool.with_connection do
-                IMPORT_WORKER.call(self, routine.options, total, progress_bar, ids, index)
-              end
-            end
-
+            results = ::Parallel.map_with_index(
+              batches,
+              routine.parallel_options,
+              &IMPORT_WORKER.curry[self, routine.options, batches.size]
+            )
             ::ActiveRecord::Base.connection.reconnect! if defined?(::ActiveRecord::Base)
             errors, import, leftovers = process_parallel_import_results(results)
 
-            execute_leftovers(leftovers, routine, self, errors)
+            if leftovers.present?
+              batches = leftovers.each_slice(routine.options[:batch_size])
+              results = ::Parallel.map_with_index(
+                batches,
+                routine.parallel_options,
+                &LEFTOVERS_WORKER.curry[self, routine.options, batches.size]
+              )
+              errors.concat(results.flatten(1))
+            end
 
             payload[:import] = import
             payload[:errors] = payload_errors(errors) if errors.present?
@@ -191,18 +217,6 @@ module Chewy
           end
         end
 
-        def execute_leftovers(leftovers, routine, self_object, errors)
-          return unless leftovers.present?
-
-          batches = leftovers.each_slice(routine.options[:batch_size])
-          results = ::Parallel.map_with_index(
-            batches,
-            routine.parallel_options,
-            &LEFTOVERS_WORKER.curry[self_object, routine.options, batches.size]
-          )
-          errors.concat(results.flatten(1))
-        end
-
         def process_parallel_import_results(results)
           results.each_with_object([[], {}, []]) do |r, (e, i, l)|
             e.concat(r[:errors])

data/lib/chewy/index/mapping.rb

@@ -170,8 +170,8 @@ module Chewy
         #   template /tit.+/, type: 'text', mapping_hash # "match_mapping_type" as an optional second argument
         #   template template42: {match: 'hello*', mapping: {type: 'object'}} # or even pass a template as is
         #
-        def template(*args)
-          root.dynamic_template(*args)
+        def template(*args, **options)
+          root.dynamic_template(*args, **options)
         end
         alias_method :dynamic_template, :template
 

data/lib/chewy/index/observe/active_record_methods.rb

@@ -0,0 +1,87 @@
+module Chewy
+  class Index
+    module Observe
+      module Helpers
+        def update_proc(index_name, *args, &block)
+          options = args.extract_options!
+          method = args.first
+
+          proc do
+            reference = if index_name.is_a?(Proc)
+              if index_name.arity.zero?
+                instance_exec(&index_name)
+              else
+                index_name.call(self)
+              end
+            else
+              index_name
+            end
+
+            index = Chewy.derive_name(reference)
+
+            next if Chewy.strategy.current.name == :bypass
+
+            backreference = if method && method.to_s == 'self'
+              self
+            elsif method
+              send(method)
+            else
+              instance_eval(&block)
+            end
+
+            index.update_index(backreference, options)
+          end
+        end
+
+        def extract_callback_options!(args)
+          options = args.extract_options!
+          result = options.each_key.with_object({}) do |key, hash|
+            hash[key] = options.delete(key) if %i[if unless].include?(key)
+          end
+          args.push(options) unless options.empty?
+          result
+        end
+      end
+
+      extend Helpers
+
+      module ActiveRecordMethods
+        extend ActiveSupport::Concern
+
+        def run_chewy_callbacks
+          chewy_callbacks.each { |callback| callback.call(self) }
+        end
+
+        def update_chewy_indices
+          Chewy.strategy.current.update_chewy_indices(self)
+        end
+
+        included do
+          class_attribute :chewy_callbacks, default: []
+        end
+
+        class_methods do
+          def initialize_chewy_callbacks
+            if Chewy.use_after_commit_callbacks
+              after_commit :update_chewy_indices, on: %i[create update]
+              after_commit :run_chewy_callbacks, on: :destroy
+            else
+              after_save :update_chewy_indices
+              after_destroy :run_chewy_callbacks
+            end
+          end
+
+          def update_index(type_name, *args, &block)
+            callback_options = Observe.extract_callback_options!(args)
+            update_proc = Observe.update_proc(type_name, *args, &block)
+            callback = Chewy::Index::Observe::Callback.new(update_proc, callback_options)
+
+            initialize_chewy_callbacks if chewy_callbacks.empty?
+
+            self.chewy_callbacks = chewy_callbacks.dup << callback
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/observe/callback.rb

@@ -0,0 +1,34 @@
+module Chewy
+  class Index
+    module Observe
+      class Callback
+        def initialize(executable, filters = {})
+          @executable = executable
+          @if_filter = filters[:if]
+          @unless_filter = filters[:unless]
+        end
+
+        def call(context)
+          return if !@if_filter.nil? && !eval_filter(@if_filter, context)
+          return if !@unless_filter.nil? && eval_filter(@unless_filter, context)
+
+          eval_proc(@executable, context)
+        end
+
+      private
+
+        def eval_filter(filter, context)
+          case filter
+          when Symbol then context.send(filter)
+          when Proc then eval_proc(filter, context)
+          else filter
+          end
+        end
+
+        def eval_proc(executable, context)
+          executable.arity.zero? ? context.instance_exec(&executable) : executable.call(context)
+        end
+      end
+    end
+  end
+end

data/lib/chewy/index/observe.rb

@@ -1,66 +1,11 @@
+require 'chewy/index/observe/callback'
+require 'chewy/index/observe/active_record_methods'
+
 module Chewy
   class Index
     module Observe
       extend ActiveSupport::Concern
 
-      module Helpers
-        def update_proc(index_name, *args, &block)
-          options = args.extract_options!
-          method = args.first
-
-          proc do
-            reference = if index_name.is_a?(Proc)
-              if index_name.arity.zero?
-                instance_exec(&index_name)
-              else
-                index_name.call(self)
-              end
-            else
-              index_name
-            end
-
-            index = Chewy.derive_name(reference)
-
-            next if Chewy.strategy.current.name == :bypass
-
-            backreference = if method && method.to_s == 'self'
-              self
-            elsif method
-              send(method)
-            else
-              instance_eval(&block)
-            end
-
-            index.update_index(backreference, options)
-          end
-        end
-
-        def extract_callback_options!(args)
-          options = args.extract_options!
-          result = options.each_key.with_object({}) do |key, hash|
-            hash[key] = options.delete(key) if %i[if unless].include?(key)
-          end
-          args.push(options) unless options.empty?
-          result
-        end
-      end
-
-      extend Helpers
-
-      module ActiveRecordMethods
-        def update_index(type_name, *args, &block)
-          callback_options = Observe.extract_callback_options!(args)
-          update_proc = Observe.update_proc(type_name, *args, &block)
-
-          if Chewy.use_after_commit_callbacks
-            after_commit(**callback_options, &update_proc)
-          else
-            after_save(**callback_options, &update_proc)
-            after_destroy(**callback_options, &update_proc)
-          end
-        end
-      end
-
       module ClassMethods
         def update_index(objects, options = {})
           Chewy.strategy.current.update(self, objects, options)

data/lib/chewy/index/syncer.rb

@@ -27,7 +27,7 @@ module Chewy
     # @see Chewy::Index::Actions::ClassMethods#sync
     class Syncer
       DEFAULT_SYNC_BATCH_SIZE = 20_000
-      ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/.freeze
+      ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/
       OUTDATED_IDS_WORKER = lambda do |outdated_sync_field_type, source_data_hash, index, total, index_data|
         ::Process.setproctitle("chewy [#{index}]: sync outdated calculation (#{::Parallel.worker_number + 1}/#{total})") if index
         index_data.each_with_object([]) do |(id, index_sync_value), result|

data/lib/chewy/index.rb

@@ -20,6 +20,10 @@ module Chewy
       pipeline raw_import refresh replication
     ].freeze
 
+    STRATEGY_OPTIONS = {
+      delayed_sidekiq: %i[latency margin ttl reindex_wrapper]
+    }.freeze
+
     include Search
     include Actions
     include Aliases
@@ -221,6 +225,27 @@ module Chewy
         params.assert_valid_keys(IMPORT_OPTIONS_KEYS)
         self._default_import_options = _default_import_options.merge(params)
       end
+
+      def strategy_config(params = {})
+        @strategy_config ||= begin
+          config_struct = Struct.new(*STRATEGY_OPTIONS.keys).new
+
+          STRATEGY_OPTIONS.each_with_object(config_struct) do |(strategy, options), res|
+            res[strategy] = case strategy
+            when :delayed_sidekiq
+              Struct.new(*STRATEGY_OPTIONS[strategy]).new.tap do |config|
+                options.each do |option|
+                  config[option] = params.dig(strategy, option) || Chewy.configuration.dig(:strategy_config, strategy, option)
+                end
+
+                config[:reindex_wrapper] ||= ->(&reindex) { reindex.call } # default wrapper
+              end
+            else
+              raise NotImplementedError, "Unsupported strategy: '#{strategy}'"
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/chewy/journal.rb

@@ -16,14 +16,17 @@ module Chewy
     # specified indexes.
     #
     # @param since_time [Time, DateTime] timestamp from which changes will be applied
-    # @param retries [Integer] maximum number of attempts to make journal empty, 10 by default
+    # @param fetch_limit [Int] amount of entries to be fetched on each cycle
     # @return [Integer] the amount of journal entries found
-    def apply(since_time, retries: 10, **import_options)
+    def apply(since_time, fetch_limit: 10, **import_options)
       stage = 1
       since_time -= 1
       count = 0
-      while stage <= retries
-        entries = Chewy::Stash::Journal.entries(since_time, only: @only).to_a.presence or break
+
+      total_count = entries(since_time, fetch_limit).total_count
+
+      while count < total_count
+        entries = entries(since_time, fetch_limit).to_a.presence or break
         count += entries.size
         groups = reference_groups(entries)
         ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
@@ -40,12 +43,20 @@ module Chewy
     #
     # @param until_time [Time, DateTime] time to clean up until it
     # @return [Hash] delete_by_query ES API call result
-    def clean(until_time = nil)
-      Chewy::Stash::Journal.clean(until_time, only: @only)
+    def clean(until_time = nil, delete_by_query_options: {})
+      Chewy::Stash::Journal.clean(
+        until_time,
+        only: @only,
+        delete_by_query_options: delete_by_query_options.merge(refresh: false)
+      )
     end
 
   private
 
+    def entries(since_time, fetch_limit)
+      Chewy::Stash::Journal.entries(since_time, only: @only).order(:created_at).limit(fetch_limit)
+    end
+
     def reference_groups(entries)
       entries.group_by(&:index_name)
         .transform_keys { |index_name| Chewy.derive_name(index_name) }

data/lib/chewy/log_subscriber.rb

@@ -24,7 +24,11 @@ module Chewy
 
       subject = payload[:type].presence || payload[:index]
       action = "#{subject} #{action} (#{event.duration.round(1)}ms)"
-      action = color(action, GREEN, true)
+      action = if ActiveSupport.version >= Gem::Version.new('7.1')
+        color(action, GREEN, bold: true)
+      else
+        color(action, GREEN, true)
+      end
 
       debug("  #{action} #{description}")
     end

data/lib/chewy/minitest/helpers.rb

@@ -42,10 +42,87 @@ module Chewy
       # Run indexing for the database changes during the block provided.
       # By default, indexing is run at the end of the block.
       # @param strategy [Symbol] the Chewy index update strategy see Chewy docs.
+      #
       def run_indexing(strategy: :atomic, &block)
         Chewy.strategy strategy, &block
       end
 
+      # Mock Elasticsearch response
+      # Simple usage - just pass index, expected raw response
+      # and block with the query.
+      #
+      # @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
+      # @param raw_response [Hash] hash with response.
+      #
+      def mock_elasticsearch_response(index, raw_response)
+        mocked_request = Chewy::Search::Request.new(index)
+
+        original_new = Chewy::Search::Request.method(:new)
+
+        Chewy::Search::Request.define_singleton_method(:new) { |*_args| mocked_request }
+
+        original_perform = mocked_request.method(:perform)
+        mocked_request.define_singleton_method(:perform) { raw_response }
+
+        yield
+      ensure
+        mocked_request.define_singleton_method(:perform, original_perform)
+        Chewy::Search::Request.define_singleton_method(:new, original_new)
+      end
+
+      # Mock Elasticsearch response with defined sources
+      # Simple usage - just pass index, expected sources
+      # and block with the query.
+      #
+      # @param index [Chewy::Index] the index to watch, eg EntitiesIndex.
+      # @param hits [Hash] hash with sources.
+      #
+      def mock_elasticsearch_response_sources(index, hits, &block)
+        raw_response = {
+          'took' => 4,
+          'timed_out' => false,
+          '_shards' => {
+            'total' => 1,
+            'successful' => 1,
+            'skipped' => 0,
+            'failed' => 0
+          },
+          'hits' => {
+            'total' => {
+              'value' => hits.count,
+              'relation' => 'eq'
+            },
+            'max_score' => 1.0,
+            'hits' => hits.each_with_index.map do |hit, i|
+              {
+                '_index' => index.index_name,
+                '_type' => '_doc',
+                '_id' => hit[:id] || (i + 1).to_s,
+                '_score' => 3.14,
+                '_source' => hit
+              }
+            end
+          }
+        }
+
+        mock_elasticsearch_response(index, raw_response, &block)
+      end
+
+      # Check the assertion that actual Elasticsearch query is rendered
+      # to the expected query
+      #
+      # @param query [::Query] the actual Elasticsearch query.
+      # @param expected_query [Hash] expected query.
+      #
+      # @return [Boolean]
+      #   True - in the case when actual Elasticsearch query is rendered to the expected query.
+      #   False - in the opposite case.
+      #
+      def assert_elasticsearch_query(query, expected_query)
+        actual_query = query.render
+        assert_equal expected_query, actual_query, "got #{actual_query.inspect} instead of expected query."
+      end
+
       module ClassMethods
         # Declare that all tests in this file require real indexing, always.
         # In my completely unscientific experiments, this roughly doubled test runtime.

data/lib/chewy/minitest/search_index_receiver.rb

@@ -6,6 +6,8 @@
 # The class will capture the data from the *param on the Chewy::Index.bulk method and
 # aggregate the data for test analysis.
 class SearchIndexReceiver
+  MUTATION_FOR_CLASS = Struct.new(:indexes, :deletes, keyword_init: true)
+
   def initialize
     @mutations = {}
   end
@@ -71,6 +73,6 @@ private
   # @param index [Chewy::Index] the index to fetch.
   # @return [#indexes, #deletes] an object with a list of indexes and a list of deletes.
   def mutation_for(index)
-    @mutations[index] ||= OpenStruct.new(indexes: [], deletes: [])
+    @mutations[index] ||= MUTATION_FOR_CLASS.new(indexes: [], deletes: [])
   end
 end