chewy 7.2.4 → 7.6.0

data/lib/chewy/journal.rb

@@ -16,14 +16,17 @@ module Chewy
     # specified indexes.
     #
     # @param since_time [Time, DateTime] timestamp from which changes will be applied
-    # @param retries [Integer] maximum number of attempts to make journal empty, 10 by default
+    # @param fetch_limit [Int] amount of entries to be fetched on each cycle
     # @return [Integer] the amount of journal entries found
-    def apply(since_time, retries: 10, **import_options)
+    def apply(since_time, fetch_limit: 10, **import_options)
       stage = 1
       since_time -= 1
       count = 0
-      while stage <= retries
-        entries = Chewy::Stash::Journal.entries(since_time, only: @only).to_a.presence or break
+
+      total_count = entries(since_time, fetch_limit).total_count
+
+      while count < total_count
+        entries = entries(since_time, fetch_limit).to_a.presence or break
         count += entries.size
         groups = reference_groups(entries)
         ActiveSupport::Notifications.instrument 'apply_journal.chewy', stage: stage, groups: groups
@@ -40,12 +43,20 @@ module Chewy
     #
     # @param until_time [Time, DateTime] time to clean up until it
     # @return [Hash] delete_by_query ES API call result
-    def clean(until_time = nil)
-      Chewy::Stash::Journal.clean(until_time, only: @only)
+    def clean(until_time = nil, delete_by_query_options: {})
+      Chewy::Stash::Journal.clean(
+        until_time,
+        only: @only,
+        delete_by_query_options: delete_by_query_options.merge(refresh: false)
+      )
     end
 
   private
 
+    def entries(since_time, fetch_limit)
+      Chewy::Stash::Journal.entries(since_time, only: @only).order(:created_at).limit(fetch_limit)
+    end
+
     def reference_groups(entries)
       entries.group_by(&:index_name)
         .transform_keys { |index_name| Chewy.derive_name(index_name) }

data/lib/chewy/log_subscriber.rb

@@ -24,7 +24,11 @@ module Chewy
 
       subject = payload[:type].presence || payload[:index]
       action = "#{subject} #{action} (#{event.duration.round(1)}ms)"
-      action = color(action, GREEN, true)
+      action = if ActiveSupport.version >= Gem::Version.new('7.1')
+        color(action, GREEN, bold: true)
+      else
+        color(action, GREEN, true)
+      end
 
       debug("  #{action} #{description}")
     end

data/lib/chewy/minitest/helpers.rb

@@ -97,7 +97,7 @@ module Chewy
               {
                 '_index' => index.index_name,
                 '_type' => '_doc',
-                '_id' => (i + 1).to_s,
+                '_id' => hit[:id] || (i + 1).to_s,
                 '_score' => 3.14,
                 '_source' => hit
               }

data/lib/chewy/minitest/search_index_receiver.rb

@@ -6,6 +6,8 @@
 # The class will capture the data from the *param on the Chewy::Index.bulk method and
 # aggregate the data for test analysis.
 class SearchIndexReceiver
+  MUTATION_FOR_CLASS = Struct.new(:indexes, :deletes, keyword_init: true)
+
   def initialize
     @mutations = {}
   end
@@ -71,6 +73,6 @@ private
   # @param index [Chewy::Index] the index to fetch.
   # @return [#indexes, #deletes] an object with a list of indexes and a list of deletes.
   def mutation_for(index)
-    @mutations[index] ||= OpenStruct.new(indexes: [], deletes: [])
+    @mutations[index] ||= MUTATION_FOR_CLASS.new(indexes: [], deletes: [])
   end
 end

data/lib/chewy/rake_helper.rb

@@ -19,6 +19,9 @@ module Chewy
       output.puts "  Applying journal to #{targets}, #{count} entries, stage #{payload[:stage]}"
     end
 
+    DELETE_BY_QUERY_OPTIONS = %w[WAIT_FOR_COMPLETION REQUESTS_PER_SECOND SCROLL_SIZE].freeze
+    FALSE_VALUES = %w[0 f false off].freeze
+
     class << self
       # Performs zero-downtime reindexing of all documents for the specified indexes
       #
@@ -162,7 +165,7 @@ module Chewy
 
         subscribed_task_stats(output) do
           output.puts "Applying journal entries created after #{time}"
-          count = Chewy::Journal.new(indexes_from(only: only, except: except)).apply(time)
+          count = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).apply(time)
           output.puts 'No journal entries were created after the specified time' if count.zero?
         end
       end
@@ -181,12 +184,29 @@ module Chewy
       # @param except [Array<Chewy::Index, String>, Chewy::Index, String] indexes to exclude from processing
       # @param output [IO] output io for logging
       # @return [Array<Chewy::Index>] indexes that were actually updated
-      def journal_clean(time: nil, only: nil, except: nil, output: $stdout)
+      def journal_clean(time: nil, only: nil, except: nil, delete_by_query_options: {}, output: $stdout)
         subscribed_task_stats(output) do
           output.puts "Cleaning journal entries created before #{time}" if time
-          response = Chewy::Journal.new(indexes_from(only: only, except: except)).clean(time)
-          count = response['deleted'] || response['_indices']['_all']['deleted']
-          output.puts "Cleaned up #{count} journal entries"
+          response = Chewy::Journal.new(journal_indexes_from(only: only, except: except)).clean(time, delete_by_query_options: delete_by_query_options)
+          if response.key?('task')
+            output.puts "Task to cleanup the journal has been created, #{response['task']}"
+          else
+            count = response['deleted'] || response['_indices']['_all']['deleted']
+            output.puts "Cleaned up #{count} journal entries"
+          end
+        end
+      end
+
+      # Creates journal index.
+      #
+      # @example
+      #   Chewy::RakeHelper.journal_create # creates journal
+      #
+      # @param output [IO] output io for logging
+      # @return Chewy::Index Returns instance of chewy index
+      def journal_create(output: $stdout)
+        subscribed_task_stats(output) do
+          Chewy::Stash::Journal.create!
         end
       end
 
@@ -228,6 +248,44 @@ module Chewy
         end
       end
 
+      # Reads options that are required to run journal cleanup asynchronously from ENV hash
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
+      #
+      # @example
+      #   Chewy::RakeHelper.delete_by_query_options_from_env({'WAIT_FOR_COMPLETION' => 'false','REQUESTS_PER_SECOND' => '10','SCROLL_SIZE' => '5000'})
+      #   # => { wait_for_completion: false, requests_per_second: 10.0, scroll_size: 5000 }
+      #
+      def delete_by_query_options_from_env(env)
+        env
+          .slice(*DELETE_BY_QUERY_OPTIONS)
+          .transform_keys { |k| k.downcase.to_sym }
+          .to_h do |key, value|
+            case key
+            when :wait_for_completion then [key, !FALSE_VALUES.include?(value.downcase)]
+            when :requests_per_second then [key, value.to_f]
+            when :scroll_size then [key, value.to_i]
+            end
+          end
+      end
+
+      def create_missing_indexes!(output: $stdout, env: ENV)
+        subscribed_task_stats(output) do
+          Chewy.eager_load!
+          all_indexes = Chewy::Index.descendants
+          all_indexes -= [Chewy::Stash::Journal] unless Chewy.configuration[:journal]
+          all_indexes.each do |index|
+            if index.exists?
+              output.puts "#{index.name} already exists, skipping" if env['VERBOSE']
+              next
+            end
+
+            index.create!
+
+            output.puts "#{index.name} index successfully created"
+          end
+        end
+      end
+
       def normalize_indexes(*identifiers)
         identifiers.flatten(1).map { |identifier| normalize_index(identifier) }
       end
@@ -243,11 +301,18 @@ module Chewy
         ActiveSupport::Notifications.subscribed(JOURNAL_CALLBACK.curry[output], 'apply_journal.chewy') do
           ActiveSupport::Notifications.subscribed(IMPORT_CALLBACK.curry[output], 'import_objects.chewy', &block)
         end
+      ensure
         output.puts "Total: #{human_duration(Time.now - start)}"
       end
 
     private
 
+      def journal_indexes_from(only: nil, except: nil)
+        return if Array.wrap(only).empty? && Array.wrap(except).empty?
+
+        indexes_from(only: only, except: except)
+      end
+
       def indexes_from(only: nil, except: nil)
         indexes = if only.present?
           normalize_indexes(Array.wrap(only))
@@ -255,11 +320,7 @@ module Chewy
           all_indexes
         end
 
-        indexes = if except.present?
-          indexes - normalize_indexes(Array.wrap(except))
-        else
-          indexes
-        end
+        indexes -= normalize_indexes(Array.wrap(except)) if except.present?
 
         indexes.sort_by(&:derivable_name)
       end
@@ -282,9 +343,9 @@ module Chewy
         return if journal_exists?
 
         output.puts "############################################################\n" \
-                    "WARN: You are risking to lose some changes during the reset.\n" \
-                    "      Please consider enabling journaling.\n" \
-                    "      See https://github.com/toptal/chewy#journaling\n" \
+                    "WARN: You are risking to lose some changes during the reset.\n      " \
+                    "Please consider enabling journaling.\n      " \
+                    "See https://github.com/toptal/chewy#journaling\n" \
                     '############################################################'
       end
 

data/lib/chewy/rspec/update_index.rb

@@ -88,6 +88,11 @@ RSpec::Matchers.define :update_index do |index_name, options = {}| # rubocop:dis
     @only = true
   end
 
+  # Expect import to be called with refresh=false parameter
+  chain(:no_refresh) do
+    @no_refresh = true
+  end
+
   def supports_block_expectations?
     true
   end
@@ -100,11 +105,13 @@ RSpec::Matchers.define :update_index do |index_name, options = {}| # rubocop:dis
 
     index = Chewy.derive_name(index_name)
     if defined?(Mocha) && RSpec.configuration.mock_framework.to_s == 'RSpec::Core::MockingAdapters::Mocha'
-      Chewy::Index::Import::BulkRequest.stubs(:new).with(index, any_parameters).returns(mock_bulk_request)
+      params_matcher = @no_refresh ? has_entry(refresh: false) : any_parameters
+      Chewy::Index::Import::BulkRequest.stubs(:new).with(index, params_matcher).returns(mock_bulk_request)
     else
-      mocked_already = ::RSpec::Mocks.space.proxy_for(Chewy::Index::Import::BulkRequest).method_double_if_exists_for_message(:new)
+      mocked_already = RSpec::Mocks.space.proxy_for(Chewy::Index::Import::BulkRequest).method_double_if_exists_for_message(:new)
       allow(Chewy::Index::Import::BulkRequest).to receive(:new).and_call_original unless mocked_already
-      allow(Chewy::Index::Import::BulkRequest).to receive(:new).with(index, any_args).and_return(mock_bulk_request)
+      params_matcher = @no_refresh ? hash_including(refresh: false) : any_args
+      allow(Chewy::Index::Import::BulkRequest).to receive(:new).with(index, params_matcher).and_return(mock_bulk_request)
     end
 
     Chewy.strategy(options[:strategy] || :atomic) { block.call }
@@ -146,7 +153,7 @@ RSpec::Matchers.define :update_index do |index_name, options = {}| # rubocop:dis
     output = ''
 
     if mock_bulk_request.updates.none?
-      output << "Expected index `#{index_name}` to be updated, but it was not\n"
+      output << "Expected index `#{index_name}` to be updated#{' with no refresh' if @no_refresh}, but it was not\n"
     elsif @missed_reindex.present? || @missed_delete.present?
       message = "Expected index `#{index_name}` "
       message << [
@@ -213,7 +220,7 @@ RSpec::Matchers.define :update_index do |index_name, options = {}| # rubocop:dis
     expected_count = options[:times] || options[:count]
     expected_attributes = (options[:with] || options[:attributes] || {}).deep_symbolize_keys
 
-    args.flatten.map do |document|
+    args.flatten.to_h do |document|
       id = document.respond_to?(:id) ? document.id.to_s : document.to_s
       [id, {
         document: document,
@@ -222,7 +229,7 @@ RSpec::Matchers.define :update_index do |index_name, options = {}| # rubocop:dis
         real_count: 0,
         real_attributes: {}
       }]
-    end.to_h
+    end
   end
 
   def compare_attributes(expected, real)

data/lib/chewy/runtime/version.rb

@@ -5,7 +5,7 @@ module Chewy
       attr_reader :major, :minor, :patch
 
       def initialize(version)
-        @major, @minor, @patch = *(version.to_s.split('.', 3) + [0] * 3).first(3).map(&:to_i)
+        @major, @minor, @patch = *(version.to_s.split('.', 3) + ([0] * 3)).first(3).map(&:to_i)
       end
 
       def to_s

data/lib/chewy/search/parameters/collapse.rb

@@ -0,0 +1,16 @@
+require 'chewy/search/parameters/storage'
+
+module Chewy
+  module Search
+    class Parameters
+      # Just a standard hash storage. Nothing to see here.
+      #
+      # @see Chewy::Search::Parameters::HashStorage
+      # @see Chewy::Search::Request#collapse
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/collapse-search-results.html
+      class Collapse < Storage
+        include HashStorage
+      end
+    end
+  end
+end

data/lib/chewy/search/parameters/indices.rb

@@ -17,7 +17,7 @@ module Chewy
         # @param other [Chewy::Search::Parameters::Storage] any storage instance
         # @return [true, false] the result of comparison
         def ==(other)
-          super || other.class == self.class && other.render == render
+          super || (other.class == self.class && other.render == render)
         end
 
         # Just adds indices to indices.

data/lib/chewy/search/parameters/knn.rb

@@ -0,0 +1,16 @@
+require 'chewy/search/parameters/storage'
+
+module Chewy
+  module Search
+    class Parameters
+      # Just a standard hash storage. Nothing to see here.
+      #
+      # @see Chewy::Search::Parameters::HashStorage
+      # @see Chewy::Search::Request#knn
+      # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html
+      class Knn < Storage
+        include HashStorage
+      end
+    end
+  end
+end

data/lib/chewy/search/parameters/storage.rb

@@ -35,7 +35,7 @@ module Chewy
         # @param other [Chewy::Search::Parameters::Storage] any storage instance
         # @return [true, false] the result of comparision
         def ==(other)
-          super || other.class == self.class && other.value == value
+          super || (other.class == self.class && other.value == value)
         end
 
         # Replaces current value with normalized provided one. Doesn't

data/lib/chewy/search/parameters.rb

@@ -1,5 +1,5 @@
-Dir.glob(File.join(File.dirname(__FILE__), 'parameters', 'concerns', '*.rb')).sort.each { |f| require f }
-Dir.glob(File.join(File.dirname(__FILE__), 'parameters', '*.rb')).sort.each { |f| require f }
+Dir.glob(File.join(File.dirname(__FILE__), 'parameters', 'concerns', '*.rb')).each { |f| require f }
+Dir.glob(File.join(File.dirname(__FILE__), 'parameters', '*.rb')).each { |f| require f }
 
 module Chewy
   module Search
@@ -53,7 +53,7 @@ module Chewy
       # @param other [Object] any object
       # @return [true, false]
       def ==(other)
-        super || other.is_a?(self.class) && compare_storages(other)
+        super || (other.is_a?(self.class) && compare_storages(other))
       end
 
       # Clones the specified storage, performs the operation

data/lib/chewy/search/request.rb

@@ -20,11 +20,11 @@ module Chewy
       UNDEFINED = Class.new.freeze
       EVERFIELDS = %w[_index _type _id _parent _routing].freeze
       DELEGATED_METHODS = %i[
-        query filter post_filter order reorder docvalue_fields
+        query filter post_filter knn order reorder docvalue_fields
         track_scores track_total_hits request_cache explain version profile
         search_type preference limit offset terminate_after
         timeout min_score source stored_fields search_after
-        load script_fields suggest aggs aggregations none
+        load script_fields suggest aggs aggregations collapse none
         indices_boost rescore highlight total total_count
         total_entries indices types delete_all count exists?
         exist? find pluck scroll_batches scroll_hits
@@ -41,7 +41,7 @@ module Chewy
       EXTRA_STORAGES = %i[aggs suggest].freeze
       # An array of storage names that are changing the returned hist collection in any way.
       WHERE_STORAGES = %i[
-        query filter post_filter none min_score rescore indices_boost
+        query filter post_filter knn none min_score rescore indices_boost collapse
       ].freeze
 
       delegate :hits, :wrappers, :objects, :records, :documents,
@@ -509,7 +509,29 @@ module Chewy
       #   @see https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html#multi-index
       #   @param value [true, false, nil]
       #   @return [Chewy::Search::Request]
-      %i[request_cache search_type preference timeout limit offset terminate_after min_score ignore_unavailable].each do |name|
+      #
+      # @!method collapse(value)
+      #   Replaces the value of the `collapse` request part.
+      #
+      #   @example
+      #     PlacesIndex.collapse(field: :name)
+      #     # => <PlacesIndex::Query {..., :body=>{:collapse=>{"field"=>:name}}}>
+      #   @see Chewy::Search::Parameters::Collapse
+      #   @see https://www.elastic.co/guide/en/elasticsearch/reference/current/collapse-search-results.html
+      #   @param value [Hash]
+      #   @return [Chewy::Search::Request]
+      #
+      # @!method knn(value)
+      #   Replaces the value of the `knn` request part.
+      #
+      #   @example
+      #     PlacesIndex.knn(field: :vector, query_vector: [4, 2], k: 5, num_candidates: 50)
+      #     # => <PlacesIndex::Query {..., :body=>{:knn=>{"field"=>:vector, "query_vector"=>[4, 2], "k"=>5, "num_candidates"=>50}}}>
+      #   @see Chewy::Search::Parameters::Knn
+      #   @see https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html
+      #   @param value [Hash]
+      #   @return [Chewy::Search::Request]
+      %i[request_cache search_type preference timeout limit offset terminate_after min_score ignore_unavailable collapse knn].each do |name|
         define_method name do |value|
           modify(name) { replace!(value) }
         end
@@ -800,8 +822,8 @@ module Chewy
       # Returns a new scope containing only specified storages.
       #
       # @example
-      #   PlacesIndex.limit(10).offset(10).order(:name).except(:offset, :order)
-      #   # => <PlacesIndex::Query {..., :body=>{:size=>10}}>
+      #   PlacesIndex.limit(10).offset(10).order(:name).only(:offset, :order)
+      #   # => <PlacesIndex::Query {..., :body=>{:from=>10, :sort=>["name"]}}>
       # @param values [Array<String, Symbol>]
       # @return [Chewy::Search::Request] new scope
       def only(*values)
@@ -811,8 +833,8 @@ module Chewy
       # Returns a new scope containing all the storages except specified.
       #
       # @example
-      #   PlacesIndex.limit(10).offset(10).order(:name).only(:offset, :order)
-      #   # => <PlacesIndex::Query {..., :body=>{:from=>10, :sort=>["name"]}}>
+      #   PlacesIndex.limit(10).offset(10).order(:name).except(:offset, :order)
+      #   # => <PlacesIndex::Query {..., :body=>{:size=>10}}>
       # @param values [Array<String, Symbol>]
       # @return [Chewy::Search::Request] new scope
       def except(*values)
@@ -951,10 +973,22 @@ module Chewy
       #
       # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
       # @note The result hash is different for different API used.
-      # @param refresh [true, false] field names
+      # @param refresh [true, false] Refreshes all shards involved in the delete by query
+      # @param wait_for_completion [true, false] wait for request completion or run it asynchronously
+      #    and return task reference at `.tasks/task/${taskId}`.
+      # @param requests_per_second [Float] The throttle for this request in sub-requests per second
+      # @param scroll_size [Integer] Size of the scroll request that powers the operation
+
       # @return [Hash] the result of query execution
-      def delete_all(refresh: true)
-        request_body = only(WHERE_STORAGES).render.merge(refresh: refresh)
+      def delete_all(refresh: true, wait_for_completion: nil, requests_per_second: nil, scroll_size: nil)
+        request_body = only(WHERE_STORAGES).render.merge(
+          {
+            refresh: refresh,
+            wait_for_completion: wait_for_completion,
+            requests_per_second: requests_per_second,
+            scroll_size: scroll_size
+          }.compact
+        )
         ActiveSupport::Notifications.instrument 'delete_query.chewy', notification_payload(request: request_body) do
           request_body[:body] = {query: {match_all: {}}} if request_body[:body].empty?
           Chewy.client.delete_by_query(request_body)

data/lib/chewy/search.rb

@@ -56,7 +56,7 @@ module Chewy
       #
       # @example
       #   PlacesIndex.query(match: {name: 'Moscow'})
-      ruby2_keywords def method_missing(name, *args, &block)
+      def method_missing(name, *args, &block)
         if search_class::DELEGATED_METHODS.include?(name)
           all.send(name, *args, &block)
         else
@@ -84,9 +84,12 @@ module Chewy
       def delegate_scoped(source, destination, methods)
         methods.each do |method|
           destination.class_eval do
-            define_method method do |*args, &block|
-              scoping { source.public_send(method, *args, &block) }
+            define_method method do |*args, **kwargs, &block|
+              scoping do
+                source.public_send(method, *args, **kwargs, &block)
+              end
             end
+            method
           end
         end
       end

data/lib/chewy/stash.rb

@@ -28,12 +28,12 @@ module Chewy
       # Cleans up all the journal entries until the specified time. If nothing is
       # specified - cleans up everything.
       #
-      # @param since_time [Time, DateTime] the time top boundary
+      # @param until_time [Time, DateTime] Clean everything before that date
       # @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
-      def self.clean(until_time = nil, only: [])
+      def self.clean(until_time = nil, only: [], delete_by_query_options: {})
         scope = self.for(only)
         scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
-        scope.delete_all
+        scope.delete_all(**delete_by_query_options)
       end
 
       # Selects all the journal entries for the specified indices.

data/lib/chewy/strategy/atomic_no_refresh.rb

@@ -0,0 +1,18 @@
+module Chewy
+  class Strategy
+    # This strategy works like atomic but import objects with `refresh=false` parameter.
+    #
+    #   Chewy.strategy(:atomic_no_refresh) do
+    #     User.all.map(&:save) # Does nothing here
+    #     Post.all.map(&:save) # And here
+    #     # It imports all the changed users and posts right here
+    #     # before block leaving with bulk ES API, kinda optimization
+    #   end
+    #
+    class AtomicNoRefresh < Atomic
+      def leave
+        @stash.all? { |type, ids| type.import!(ids, refresh: false) }
+      end
+    end
+  end
+end

data/lib/chewy/strategy/base.rb

@@ -22,6 +22,16 @@ module Chewy
       # strategies stack
       #
       def leave; end
+
+      # This method called when some model record is created or updated.
+      # Normally it will just evaluate all the Chewy callbacks and pass results
+      # to current strategy's update method.
+      # However it's possible to override it to achieve delayed evaluation of
+      # callbacks, e.g. using sidekiq.
+      #
+      def update_chewy_indices(object)
+        object.run_chewy_callbacks
+      end
     end
   end
 end