chewy 7.2.1 → 7.6.0

data/lib/chewy/search.rb

@@ -84,9 +84,12 @@ module Chewy
       def delegate_scoped(source, destination, methods)
         methods.each do |method|
           destination.class_eval do
-            define_method method do |*args, &block|
-              scoping { source.public_send(method, *args, &block) }
+            define_method method do |*args, **kwargs, &block|
+              scoping do
+                source.public_send(method, *args, **kwargs, &block)
+              end
             end
+            method
           end
         end
       end

data/lib/chewy/stash.rb

@@ -28,12 +28,12 @@ module Chewy
       # Cleans up all the journal entries until the specified time. If nothing is
       # specified - cleans up everything.
       #
-      # @param since_time [Time, DateTime] the time top boundary
+      # @param until_time [Time, DateTime] Clean everything before that date
       # @param only [Chewy::Index, Array<Chewy::Index>] indexes to clean up journal entries for
-      def self.clean(until_time = nil, only: [])
+      def self.clean(until_time = nil, only: [], delete_by_query_options: {})
         scope = self.for(only)
         scope = scope.filter(range: {created_at: {lte: until_time}}) if until_time
-        scope.delete_all
+        scope.delete_all(**delete_by_query_options)
       end
 
       # Selects all the journal entries for the specified indices.

data/lib/chewy/strategy/active_job.rb

@@ -15,7 +15,7 @@ module Chewy
 
         def perform(type, ids, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
-          type.constantize.import!(ids, options)
+          type.constantize.import!(ids, **options)
         end
       end
 

data/lib/chewy/strategy/atomic_no_refresh.rb

@@ -0,0 +1,18 @@
+module Chewy
+  class Strategy
+    # This strategy works like atomic but import objects with `refresh=false` parameter.
+    #
+    #   Chewy.strategy(:atomic_no_refresh) do
+    #     User.all.map(&:save) # Does nothing here
+    #     Post.all.map(&:save) # And here
+    #     # It imports all the changed users and posts right here
+    #     # before block leaving with bulk ES API, kinda optimization
+    #   end
+    #
+    class AtomicNoRefresh < Atomic
+      def leave
+        @stash.all? { |type, ids| type.import!(ids, refresh: false) }
+      end
+    end
+  end
+end

data/lib/chewy/strategy/base.rb

@@ -22,6 +22,16 @@ module Chewy
       # strategies stack
       #
       def leave; end
+
+      # This method called when some model record is created or updated.
+      # Normally it will just evaluate all the Chewy callbacks and pass results
+      # to current strategy's update method.
+      # However it's possible to override it to achieve delayed evaluation of
+      # callbacks, e.g. using sidekiq.
+      #
+      def update_chewy_indices(object)
+        object.run_chewy_callbacks
+      end
     end
   end
 end

data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require_relative '../../index'
+
+# The class is responsible for accumulating in redis [type, ids]
+# that were requested to be reindexed during `latency` seconds.
+# The reindex job is going to be scheduled after a `latency` seconds.
+# that job is going to read accumulated [type, ids] from the redis
+# and reindex all them at once.
+module Chewy
+  class Strategy
+    class DelayedSidekiq
+      require_relative 'worker'
+
+      LUA_SCRIPT = <<~LUA
+        local timechunk_key = KEYS[1]
+        local timechunks_key = KEYS[2]
+        local serialize_data = ARGV[1]
+        local at = ARGV[2]
+        local ttl = tonumber(ARGV[3])
+
+        local schedule_job = false
+
+        -- Check if the 'sadd?' method is available
+        if redis.call('exists', 'sadd?') == 1 then
+          redis.call('sadd?', timechunk_key, serialize_data)
+        else
+          redis.call('sadd', timechunk_key, serialize_data)
+        end
+
+        -- Set expiration for timechunk_key
+        redis.call('expire', timechunk_key, ttl)
+
+        -- Check if timechunk_key exists in the sorted set
+        if not redis.call('zrank', timechunks_key, timechunk_key) then
+            -- Add timechunk_key to the sorted set
+            redis.call('zadd', timechunks_key, at, timechunk_key)
+            -- Set expiration for timechunks_key
+            redis.call('expire', timechunks_key, ttl)
+            schedule_job = true
+        end
+
+        return schedule_job
+      LUA
+
+      class Scheduler
+        DEFAULT_TTL = 60 * 60 * 24 # in seconds
+        DEFAULT_LATENCY = 10
+        DEFAULT_MARGIN = 2
+        DEFAULT_QUEUE = 'chewy'
+        KEY_PREFIX = 'chewy:delayed_sidekiq'
+        FALLBACK_FIELDS = 'all'
+        FIELDS_IDS_SEPARATOR = ';'
+        IDS_SEPARATOR = ','
+
+        def initialize(type, ids, options = {})
+          @type = type
+          @ids = ids
+          @options = options
+        end
+
+        # the diagram:
+        #
+        #  inputs:
+        #  latency == 2
+        #  reindex_time = Time.current
+        #
+        #  Parallel OR Sequential triggers of reindex:          |  What is going on in reindex store (Redis):
+        #  --------------------------------------------------------------------------------------------------
+        #                                                       |
+        #  process 1 (reindex_time):                            |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1]
+        #    Schedule.new(CitiesIndex, [1]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & schedule a DelayedSidekiq::Worker at 1679347869 (at + 3)
+        #                                                       |    it will zpop chewy:delayed_sidekiq:timechunks up to 1679347866 score and reindex all ids with zpoped keys
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347866
+        #                                                       |
+        #                                                       |
+        #  process 2 (reindex_time):                            |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2]
+        #    Schedule.new(CitiesIndex, [2]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & do not schedule a new worker
+        #                                                       |
+        #                                                       |
+        #  process 1 (reindex_time + (latency - 1).seconds):    |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
+        #    Schedule.new(CitiesIndex, [3]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & do not schedule a new worker
+        #                                                       |
+        #                                                       |
+        #  process 2 (reindex_time + (latency + 1).seconds):    |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
+        #    Schedule.new(CitiesIndex, [4]).postpone            |  chewy:delayed_sidekiq:CitiesIndex:1679347868 = [4]
+        #                                                       |  chewy:delayed_sidekiq:timechunks = [
+        #                                                       |    { score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}
+        #                                                       |    { score: 1679347868, "chewy:delayed_sidekiq:CitiesIndex:1679347868"}
+        #                                                       |  ]
+        #                                                       |  & schedule a DelayedSidekiq::Worker at 1679347871 (at + 3)
+        #                                                       |    it will zpop chewy:delayed_sidekiq:timechunks up to 1679347868 score and reindex all ids with zpoped keys
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347866 (in case of failed previous reindex),
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347868
+        def postpone
+          ::Sidekiq.redis do |redis|
+            # do the redis stuff in a single command to avoid concurrency issues
+            if redis.eval(LUA_SCRIPT, keys: [timechunk_key, timechunks_key], argv: [serialize_data, at, ttl])
+              ::Sidekiq::Client.push(
+                'queue' => sidekiq_queue,
+                'at' => at + margin,
+                'class' => Chewy::Strategy::DelayedSidekiq::Worker,
+                'args' => [type_name, at]
+              )
+            end
+          end
+        end
+
+      private
+
+        attr_reader :type, :ids, :options
+
+        # this method returns predictable value that jumps by latency value
+        # another words each latency seconds it return the same value
+        def at
+          @at ||= begin
+            schedule_at = latency.seconds.from_now.to_f
+
+            (schedule_at - (schedule_at % latency)).to_i
+          end
+        end
+
+        def fields
+          options[:update_fields].presence || [FALLBACK_FIELDS]
+        end
+
+        def timechunks_key
+          "#{KEY_PREFIX}:#{type_name}:timechunks"
+        end
+
+        def timechunk_key
+          "#{KEY_PREFIX}:#{type_name}:#{at}"
+        end
+
+        def serialize_data
+          [ids.join(IDS_SEPARATOR), fields.join(IDS_SEPARATOR)].join(FIELDS_IDS_SEPARATOR)
+        end
+
+        def type_name
+          type.name
+        end
+
+        def latency
+          strategy_config.latency || DEFAULT_LATENCY
+        end
+
+        def margin
+          strategy_config.margin || DEFAULT_MARGIN
+        end
+
+        def ttl
+          strategy_config.ttl || DEFAULT_TTL
+        end
+
+        def sidekiq_queue
+          Chewy.settings.dig(:sidekiq, :queue) || DEFAULT_QUEUE
+        end
+
+        def strategy_config
+          type.strategy_config.delayed_sidekiq
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/delayed_sidekiq/worker.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class DelayedSidekiq
+      class Worker
+        include ::Sidekiq::Worker
+
+        LUA_SCRIPT = <<~LUA
+          local type = ARGV[1]
+          local score = tonumber(ARGV[2])
+          local prefix = ARGV[3]
+          local timechunks_key = prefix .. ":" .. type .. ":timechunks"
+
+          -- Get timechunk_keys with scores less than or equal to the specified score
+          local timechunk_keys = redis.call('zrangebyscore', timechunks_key, '-inf', score)
+
+          -- Get all members from the sets associated with the timechunk_keys
+          local members = {}
+          for _, timechunk_key in ipairs(timechunk_keys) do
+              local set_members = redis.call('smembers', timechunk_key)
+              for _, member in ipairs(set_members) do
+                  table.insert(members, member)
+              end
+          end
+
+          -- Remove timechunk_keys and their associated sets
+          for _, timechunk_key in ipairs(timechunk_keys) do
+              redis.call('del', timechunk_key)
+          end
+
+          -- Remove timechunks with scores less than or equal to the specified score
+          redis.call('zremrangebyscore', timechunks_key, '-inf', score)
+
+          return members
+        LUA
+
+        def perform(type, score, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+
+          ::Sidekiq.redis do |redis|
+            members = redis.eval(LUA_SCRIPT, keys: [], argv: [type, score, Scheduler::KEY_PREFIX])
+
+            # extract ids and fields & do the reset of records
+            ids, fields = extract_ids_and_fields(members)
+            options[:update_fields] = fields if fields
+
+            index = type.constantize
+            index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
+              options.any? ? index.import!(ids, **options) : index.import!(ids)
+            end
+          end
+        end
+
+      private
+
+        def extract_ids_and_fields(members)
+          ids = []
+          fields = []
+
+          members.each do |member|
+            member_ids, member_fields = member.split(Scheduler::FIELDS_IDS_SEPARATOR).map do |v|
+              v.split(Scheduler::IDS_SEPARATOR)
+            end
+            ids |= member_ids
+            fields |= member_fields
+          end
+
+          fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS)
+
+          [ids.map(&:to_i), fields]
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/delayed_sidekiq.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class DelayedSidekiq < Sidekiq
+      require_relative 'delayed_sidekiq/scheduler'
+
+      # cleanup the redis sets used internally. Useful mainly in tests to avoid
+      # leak and potential flaky tests.
+      def self.clear_timechunks!
+        ::Sidekiq.redis do |redis|
+          keys_to_delete = redis.keys("#{Scheduler::KEY_PREFIX}*")
+
+          # Delete keys one by one
+          keys_to_delete.each do |key|
+            redis.del(key)
+          end
+        end
+      end
+
+      def leave
+        @stash.each do |type, ids|
+          next if ids.empty?
+
+          DelayedSidekiq::Scheduler.new(type, ids).postpone
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/lazy_sidekiq.rb

@@ -0,0 +1,64 @@
+module Chewy
+  class Strategy
+    # The strategy works the same way as sidekiq, but performs
+    # async evaluation of all index callbacks on model create and update
+    # driven by sidekiq
+    #
+    #   Chewy.strategy(:lazy_sidekiq) do
+    #     User.all.map(&:save) # Does nothing here
+    #     Post.all.map(&:save) # And here
+    #     # It schedules import of all the changed users and posts right here
+    #   end
+    #
+    class LazySidekiq < Sidekiq
+      class IndicesUpdateWorker
+        include ::Sidekiq::Worker
+
+        def perform(models)
+          Chewy.strategy(strategy) do
+            models.each do |model_type, model_ids|
+              model_type.constantize.where(id: model_ids).each(&:run_chewy_callbacks)
+            end
+          end
+        end
+
+      private
+
+        def strategy
+          Chewy.disable_refresh_async ? :atomic_no_refresh : :atomic
+        end
+      end
+
+      def initialize
+        # Use parent's @stash to store destroyed records, since callbacks for them have to
+        # be run immediately on the strategy block end because we won't be able to fetch
+        # records further in IndicesUpdateWorker. This will be done by avoiding of
+        # LazySidekiq#update_chewy_indices call and calling LazySidekiq#update instead.
+        super
+
+        # @lazy_stash is used to store all the lazy evaluated callbacks with call of
+        # strategy's #update_chewy_indices.
+        @lazy_stash = {}
+      end
+
+      def leave
+        # Fallback to Sidekiq#leave implementation for destroyed records stored in @stash.
+        super
+
+        # Proceed with other records stored in @lazy_stash
+        return if @lazy_stash.empty?
+
+        ::Sidekiq::Client.push(
+          'queue' => sidekiq_queue,
+          'class' => Chewy::Strategy::LazySidekiq::IndicesUpdateWorker,
+          'args'  => [@lazy_stash]
+        )
+      end
+
+      def update_chewy_indices(object)
+        @lazy_stash[object.class.name] ||= []
+        @lazy_stash[object.class.name] |= Array.wrap(object.id)
+      end
+    end
+  end
+end

data/lib/chewy/strategy/sidekiq.rb

@@ -15,7 +15,7 @@ module Chewy
 
         def perform(type, ids, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
-          type.constantize.import!(ids, options)
+          type.constantize.import!(ids, **options)
         end
       end
 

data/lib/chewy/strategy.rb

@@ -2,10 +2,13 @@ require 'chewy/strategy/base'
 require 'chewy/strategy/bypass'
 require 'chewy/strategy/urgent'
 require 'chewy/strategy/atomic'
+require 'chewy/strategy/atomic_no_refresh'
 
 begin
   require 'sidekiq'
   require 'chewy/strategy/sidekiq'
+  require 'chewy/strategy/lazy_sidekiq'
+  require 'chewy/strategy/delayed_sidekiq'
 rescue LoadError
   nil
 end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.2.1'.freeze
+  VERSION = '7.6.0'.freeze
 end

data/lib/chewy.rb

@@ -1,21 +1,22 @@
+require 'active_support'
 require 'active_support/version'
 require 'active_support/concern'
 require 'active_support/deprecation'
 require 'active_support/json'
 require 'active_support/log_subscriber'
 
+require 'active_support/isolated_execution_state' if ActiveSupport::VERSION::MAJOR >= 7
 require 'active_support/core_ext/array/access'
 require 'active_support/core_ext/array/wrap'
 require 'active_support/core_ext/enumerable'
 require 'active_support/core_ext/hash/reverse_merge'
+require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/numeric/bytes'
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/object/inclusion'
 require 'active_support/core_ext/string/inflections'
 
-require 'i18n/core_ext/hash'
-require 'chewy/backports/deep_dup' unless Object.respond_to?(:deep_dup)
 require 'singleton'
 require 'base64'
 
@@ -47,10 +48,11 @@ require 'chewy/index'
 require 'chewy/fields/base'
 require 'chewy/fields/root'
 require 'chewy/journal'
-require 'chewy/railtie' if defined?(::Rails::Railtie)
+require 'chewy/railtie' if defined?(Rails::Railtie)
+require 'chewy/elastic_client'
 
 ActiveSupport.on_load(:active_record) do
-  extend Chewy::Index::Observe::ActiveRecordMethods
+  include Chewy::Index::Observe::ActiveRecordMethods
 end
 
 module Chewy
@@ -62,6 +64,16 @@ module Chewy
   class << self
     attr_accessor :adapters
 
+    # A thread-local variables accessor
+    # @return [Hash]
+    def current
+      unless Thread.current.thread_variable?(:chewy)
+        Thread.current.thread_variable_set(:chewy, {})
+      end
+
+      Thread.current.thread_variable_get(:chewy)
+    end
+
     # Derives an index for the passed string identifier if possible.
     #
     # @example
@@ -86,12 +98,7 @@ module Chewy
     # Main elasticsearch-ruby client instance
     #
     def client
-      Thread.current[:chewy_client] ||= begin
-        client_configuration = configuration.deep_dup
-        client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client
-        block = client_configuration[:transport_options].try(:delete, :proc)
-        ::Elasticsearch::Client.new(client_configuration, &block)
-      end
+      Chewy.current[:chewy_client] ||= Chewy::ElasticClient.new
     end
 
     # Sends wait_for_status request to ElasticSearch with status
@@ -138,15 +145,15 @@ module Chewy
     #   city3.do_update! # index updated again
     #
     def strategy(name = nil, &block)
-      Thread.current[:chewy_strategy] ||= Chewy::Strategy.new
+      Chewy.current[:chewy_strategy] ||= Chewy::Strategy.new
       if name
         if block
-          Thread.current[:chewy_strategy].wrap name, &block
+          Chewy.current[:chewy_strategy].wrap name, &block
         else
-          Thread.current[:chewy_strategy].push name
+          Chewy.current[:chewy_strategy].push name
         end
       else
-        Thread.current[:chewy_strategy]
+        Chewy.current[:chewy_strategy]
       end
     end
 

data/lib/tasks/chewy.rake

@@ -54,7 +54,12 @@ namespace :chewy do
 
   desc 'Update mapping of exising index with body hash'
   task :update_mapping, %i[index_name] => :environment do |_task, args|
-    Chewy::RakeHelper.update_mapping(name: args[:name])
+    Chewy::RakeHelper.update_mapping(name: args[:index_name])
+  end
+
+  desc 'Creates missing indexes'
+  task create_missing_indexes: :environment do
+    Chewy::RakeHelper.create_missing_indexes!
   end
 
   namespace :parallel do
@@ -87,6 +92,11 @@ namespace :chewy do
   end
 
   namespace :journal do
+    desc 'Create manually journal, useful when `skip_journal_creation_on_import` is used'
+    task create: :environment do |_task, _args|
+      Chewy::RakeHelper.journal_create
+    end
+
     desc 'Applies changes that were done after the specified time for the specified indexes/types or all of them'
     task apply: :environment do |_task, args|
       Chewy::RakeHelper.journal_apply(**parse_journal_args(args.extras))
@@ -94,7 +104,13 @@ namespace :chewy do
 
     desc 'Removes journal records created before the specified timestamp for the specified indexes/types or all of them'
     task clean: :environment do |_task, args|
-      Chewy::RakeHelper.journal_clean(**parse_journal_args(args.extras))
+      delete_options = Chewy::RakeHelper.delete_by_query_options_from_env(ENV)
+      Chewy::RakeHelper.journal_clean(
+        **[
+          parse_journal_args(args.extras),
+          {delete_by_query_options: delete_options}
+        ].reduce({}, :merge)
+      )
     end
   end
 end

data/migration_guide.md

@@ -9,7 +9,7 @@ Chewy alongside a matching Elasticsearch version.
 In order to upgrade Chewy 6/Elasticsearch 6 to Chewy 7/Elasticsearch 7 in the most seamless manner you have to:
 
 * Upgrade to the latest 6.x stable releases, namely Chewy 6.0, Elasticsearch 6.8
-* Study carefully [Breaking changes in 7.0](https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.htmll), make sure your application conforms.
+* Study carefully [Breaking changes in 7.0](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/breaking-changes-7.0.html), make sure your application conforms.
 * Run your test suite on Chewy 7.0 / Elasticsearch 7
 * Run manual tests on Chewy 7.0 / Elasticsearch 7
 * Upgrade to Chewy 7.0

data/spec/chewy/config_spec.rb

@@ -22,7 +22,7 @@ describe Chewy::Config do
 
     specify do
       expect { subject.transport_logger = logger }
-        .to change { Chewy.client.transport.logger }.to(logger)
+        .to change { Chewy.client.transport.transport.logger }.to(logger)
     end
     specify do
       expect { subject.transport_logger = logger }
@@ -40,7 +40,7 @@ describe Chewy::Config do
 
     specify do
       expect { subject.transport_tracer = tracer }
-        .to change { Chewy.client.transport.tracer }.to(tracer)
+        .to change { Chewy.client.transport.transport.tracer }.to(tracer)
     end
     specify do
       expect { subject.transport_tracer = tracer }

data/spec/chewy/elastic_client_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Chewy::ElasticClient do
+  describe 'payload inspection' do
+    let(:filter) { instance_double('Proc') }
+    let!(:filter_previous_value) { Chewy.before_es_request_filter }
+
+    before do
+      Chewy.massacre
+      stub_index(:products) do
+        field :id, type: :integer
+      end
+      ProductsIndex.create
+      Chewy.before_es_request_filter = filter
+    end
+
+    after do
+      Chewy.before_es_request_filter = filter_previous_value
+    end
+
+    it 'call filter with the request body' do
+      expect(filter).to receive(:call).with(:search, [{body: {size: 0}, index: ['products']}], {})
+      Chewy.client.search({index: ['products'], body: {size: 0}}).to_a
+    end
+  end
+end