chewy 7.4.0 → 7.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5afbccc50c9249e0a78e4f2383cef346360cef2fe9f1015d05972f4df31c26f
-  data.tar.gz: 3f0238841652782eff513dfdaa4916e8151469db5749e95e661f9c0236c526f1
+  metadata.gz: 64f061f379d740c56b7af2678ec03e3c80c1bd4c072b2aa3f68bf7803d4bc0fe
+  data.tar.gz: 6226557f9cec700d91473f36bb8623ff5bbf838211463188ac0f83eaf0ab227d
 SHA512:
-  metadata.gz: e46394c42fc5cadabd70e5917a795bc071667c4bdf69a2b0d1d1a53c1db547a9b3221839ca8463c738030192bf135998a2de5c3652c87b6523cd08f7e69d80ad
-  data.tar.gz: 0bbf17766315cd0bf21bd9ec5e402584212515094cd086a003c08e23594083e2ac5b44758f27b1750b82f284260c0eb8e4cb8d12e32357cf770efc917da6595e
+  metadata.gz: 8cb278cc5fa838a97e0c9fb94c90ea08d4a84effb041a809f850a267514c61ce78f1a919a8a0fdc5119ca968e217fe9b0aa38c307f9589f8413501f690efb314
+  data.tar.gz: 50ad4271e4948b22b626202843be7107fcc720fd3da490c14a372ca018ab0e35c0e151d5dc5b55f23b7c382a0d19cf72f7cc63e40a2dc1fc0ca286c1de717941

data/.rubocop.yml

@@ -59,3 +59,6 @@ Metrics/ModuleLength:
   Exclude:
     - 'lib/chewy/rake_helper.rb'
     - '**/*_spec.rb'
+
+Style/ArgumentsForwarding:
+  Enabled: false

data/CHANGELOG.md

@@ -8,6 +8,23 @@
 
 ### Bugs Fixed
 
+## 7.5.1 (2024-01-30)
+
+### New Features
+
+* [#925](https://github.com/toptal/chewy/pull/925): Add configuration option for default scope cleanup behavior. ([@barthez][])
+
+### Changes
+
+### Bugs Fixed
+
+## 7.5.0 (2024-01-15)
+
+### New Features
+
+* [#894](https://github.com/toptal/chewy/pull/894): Way of cleaning redis from artifacts left by `delayed_sidekiq` strategy which could potentially cause flaky tests. ([@Drowze](https://github.com/Drowze))
+* [#919](https://github.com/toptal/chewy/pull/919): Add pre-request filter ([@konalegi][https://github.com/konalegi])
+
 ## 7.4.0 (2023-12-13)
 
 ### New Features
@@ -18,7 +35,7 @@
 
 ### Bugs Fixed
 
-## 7.3.5 (2023-12-13)
+## 7.3.6 (2023-12-13)
 
 ### New Features
 

data/README.md

@@ -848,6 +848,12 @@ Explicit call of the reindex using `:delayed_sidekiq` strategy with `:update_fie
 CitiesIndex.import([1, 2, 3], update_fields: [:name], strategy: :delayed_sidekiq)
 ```
 
+While running tests with delayed_sidekiq strategy and Sidekiq is using a real redis instance that is NOT cleaned up in between tests (via e.g. `Sidekiq.redis(&:flushdb)`), you'll want to cleanup some redis keys in between tests to avoid state leaking and flaky tests. Chewy provides a convenience method for that:
+```ruby
+# it might be a good idea to also add to your testing setup, e.g.: a rspec `before` hook
+Chewy::Strategy::DelayedSidekiq.clear_timechunks!
+```
+
 #### `:active_job`
 
 This does the same thing as `:atomic`, but using ActiveJob. This will inherit the ActiveJob configuration settings including the `active_job.queue_adapter` setting for the environment. Patch `Chewy::Strategy::ActiveJob::Worker` for index updates improving.
@@ -1275,6 +1281,41 @@ If you use `DatabaseCleaner` in your tests with [the `transaction` strategy](htt
 Chewy.use_after_commit_callbacks = !Rails.env.test?
 ```
 
+### Pre-request Filter
+
+Should you need to inspect the query prior to it being dispatched to ElasticSearch during any queries, you can use the `before_es_request_filter`. `before_es_request_filter` is a callable object, as demonstrated below:
+
+```ruby
+Chewy.before_es_request_filter = -> (method_name, args, kw_args) { ... }
+```
+
+While using the `before_es_request_filter`, please consider the following:
+
+* `before_es_request_filter` acts as a simple proxy before any request made via the    `ElasticSearch::Client`. The arguments passed to this filter include:
+  * `method_name` -  The name of the method being called. Examples are search, count, bulk and etc.
+  * `args` and `kw_args` - These are the positional arguments provided in the method call.
+* The operation is synchronous, so avoid executing any heavy or time-consuming operations within the filter to prevent performance degradation.
+* The return value of the proc is disregarded. This filter is intended for inspection or modification of the query rather than generating a response.
+* Any exception raised inside the callback will propagate upward and halt the execution of the query. It is essential to handle potential errors adequately to ensure the stability of your search functionality.
+
+### Import scope clean-up behavior
+
+Whenever you set the `import_scope` for the index, in the case of ActiveRecord,
+options for order, offset and limit will be removed. You can set the behavior of
+chewy, before the clean-up itself.
+
+The default behavior is a warning sent to the Chewy logger (`:warn`). Another more
+restrictive option is raising an exception (`:raise`). Both options have a
+negative impact on performance since verifying whether the code uses any of
+these options requires building AREL query.
+
+To avoid the loading time impact, you can ignore the check (`:ignore`) before
+the clean-up.
+
+```
+Chewy.import_scope_cleanup_behavior = :ignore
+```
+
 ## Contributing
 
 1. Fork it (http://github.com/toptal/chewy/fork)

data/gemfiles/base.gemfile

@@ -6,7 +6,7 @@ gem 'rake'
 gem 'rspec', '>= 3.7.0'
 gem 'rspec-collection_matchers'
 gem 'rspec-its'
-gem 'rubocop', '1.48'
+gem 'rubocop', '1.60.1'
 gem 'sqlite3'
 gem 'timecop'
 gem 'unparser'

data/lib/chewy/config.rb

@@ -38,7 +38,12 @@ module Chewy
                   # for type mappings like `_all`.
                   :default_root_options,
                   # Default field type for any field in any Chewy type. Defaults to 'text'.
-                  :default_field_type
+                  :default_field_type,
+                  # Callback called on each search request to be done into ES
+                  :before_es_request_filter,
+                  # Behavior when import scope for index includes order, offset or limit.
+                  # Can be :ignore, :warn, :raise. Defaults to :warn
+                  :import_scope_cleanup_behavior
 
     attr_reader :transport_logger, :transport_tracer,
                 # Chewy search request DSL base class, used by every index.
@@ -60,6 +65,7 @@ module Chewy
       @indices_path = 'app/chewy'
       @default_root_options = {}
       @default_field_type = 'text'.freeze
+      @import_scope_cleanup_behavior = :warn
       @search_class = build_search_class(Chewy::Search::Request)
     end
 

data/lib/chewy/elastic_client.rb

@@ -0,0 +1,31 @@
+module Chewy
+  # Replacement for Chewy.client
+  class ElasticClient
+    def self.build_es_client(configuration = Chewy.configuration)
+      client_configuration = configuration.deep_dup
+      client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client
+      block = client_configuration[:transport_options].try(:delete, :proc)
+      ::Elasticsearch::Client.new(client_configuration, &block)
+    end
+
+    def initialize(elastic_client = self.class.build_es_client)
+      @elastic_client = elastic_client
+    end
+
+  private
+
+    def method_missing(name, *args, **kwargs, &block)
+      inspect_payload(name, args, kwargs)
+
+      @elastic_client.__send__(name, *args, **kwargs, &block)
+    end
+
+    def respond_to_missing?(name, _include_private = false)
+      @elastic_client.respond_to?(name) || super
+    end
+
+    def inspect_payload(name, args, kwargs)
+      Chewy.config.before_es_request_filter&.call(name, args, kwargs)
+    end
+  end
+end

data/lib/chewy/errors.rb

@@ -7,7 +7,7 @@ module Chewy
 
   class UndefinedUpdateStrategy < Error
     def initialize(_type)
-      super <<-MESSAGE
+      super(<<-MESSAGE)
   Index update strategy is undefined for current context.
   Please wrap your code with `Chewy.strategy(:strategy_name) block.`
       MESSAGE
@@ -27,7 +27,7 @@ module Chewy
           message << "        on #{documents.count} documents: #{documents}\n"
         end
       end
-      super message
+      super(message)
     end
   end
 
@@ -36,4 +36,7 @@ module Chewy
       super("`#{join_field_type}` set for the join field `#{join_field_name}` is not on the :relations list (#{relations})")
     end
   end
+
+  class ImportScopeCleanupError < Error
+  end
 end

data/lib/chewy/index/adapter/active_record.rb

@@ -13,9 +13,19 @@ module Chewy
       private
 
         def cleanup_default_scope!
-          if Chewy.logger && (@default_scope.arel.orders.present? ||
+          behavior = Chewy.config.import_scope_cleanup_behavior
+
+          if behavior != :ignore && (@default_scope.arel.orders.present? ||
              @default_scope.arel.limit.present? || @default_scope.arel.offset.present?)
-            Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified')
+            if behavior == :warn && Chewy.logger
+              gem_dir = File.realpath('../..', __dir__)
+              source = caller.grep_v(Regexp.new(gem_dir)).first
+              Chewy.logger.warn(
+                "Default type scope order, limit and offset are ignored and will be nullified (called from: #{source})"
+              )
+            elsif behavior == :raise
+              raise ImportScopeCleanupError, 'Default type scope order, limit and offset are ignored and will be nullified'
+            end
           end
 
           @default_scope = @default_scope.reorder(nil).limit(nil).offset(nil)

data/lib/chewy/rake_helper.rb

@@ -320,11 +320,7 @@ module Chewy
           all_indexes
         end
 
-        indexes = if except.present?
-          indexes - normalize_indexes(Array.wrap(except))
-        else
-          indexes
-        end
+        indexes -= normalize_indexes(Array.wrap(except)) if except.present?
 
         indexes.sort_by(&:derivable_name)
       end

data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb

@@ -18,6 +18,7 @@ module Chewy
         DEFAULT_MARGIN = 2
         DEFAULT_QUEUE = 'chewy'
         KEY_PREFIX = 'chewy:delayed_sidekiq'
+        ALL_SETS_KEY = "#{KEY_PREFIX}:all_sets".freeze
         FALLBACK_FIELDS = 'all'
         FIELDS_IDS_SEPARATOR = ';'
         IDS_SEPARATOR = ','
@@ -68,8 +69,10 @@ module Chewy
           ::Sidekiq.redis do |redis|
             # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
             if redis.respond_to?(:sadd?)
+              redis.sadd?(ALL_SETS_KEY, timechunks_key)
               redis.sadd?(timechunk_key, serialize_data)
             else
+              redis.sadd(ALL_SETS_KEY, timechunks_key)
               redis.sadd(timechunk_key, serialize_data)
             end
 

data/lib/chewy/strategy/delayed_sidekiq.rb

@@ -5,6 +5,19 @@ module Chewy
     class DelayedSidekiq < Sidekiq
       require_relative 'delayed_sidekiq/scheduler'
 
+      # cleanup the redis sets used internally. Useful mainly in tests to avoid
+      # leak and potential flaky tests.
+      def self.clear_timechunks!
+        ::Sidekiq.redis do |redis|
+          timechunk_sets = redis.smembers(Chewy::Strategy::DelayedSidekiq::Scheduler::ALL_SETS_KEY)
+          break if timechunk_sets.empty?
+
+          redis.pipelined do |pipeline|
+            timechunk_sets.each { |set| pipeline.del(set) }
+          end
+        end
+      end
+
       def leave
         @stash.each do |type, ids|
           next if ids.empty?

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.4.0'.freeze
+  VERSION = '7.5.1'.freeze
 end

data/lib/chewy.rb

@@ -49,6 +49,7 @@ require 'chewy/fields/base'
 require 'chewy/fields/root'
 require 'chewy/journal'
 require 'chewy/railtie' if defined?(Rails::Railtie)
+require 'chewy/elastic_client'
 
 ActiveSupport.on_load(:active_record) do
   include Chewy::Index::Observe::ActiveRecordMethods
@@ -97,12 +98,7 @@ module Chewy
     # Main elasticsearch-ruby client instance
     #
     def client
-      Chewy.current[:chewy_client] ||= begin
-        client_configuration = configuration.deep_dup
-        client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client
-        block = client_configuration[:transport_options].try(:delete, :proc)
-        ::Elasticsearch::Client.new(client_configuration, &block)
-      end
+      Chewy.current[:chewy_client] ||= Chewy::ElasticClient.new
     end
 
     # Sends wait_for_status request to ElasticSearch with status

data/spec/chewy/elastic_client_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Chewy::ElasticClient do
+  describe 'payload inspection' do
+    let(:filter) { instance_double('Proc') }
+    let!(:filter_previous_value) { Chewy.before_es_request_filter }
+
+    before do
+      Chewy.massacre
+      stub_index(:products) do
+        field :id, type: :integer
+      end
+      ProductsIndex.create
+      Chewy.before_es_request_filter = filter
+    end
+
+    after do
+      Chewy.before_es_request_filter = filter_previous_value
+    end
+
+    it 'call filter with the request body' do
+      expect(filter).to receive(:call).with(:search, [{body: {size: 0}, index: ['products']}], {})
+      Chewy.client.search({index: ['products'], body: {size: 0}}).to_a
+    end
+  end
+end

data/spec/chewy/index/adapter/active_record_spec.rb

@@ -35,6 +35,68 @@ describe Chewy::Index::Adapter::ActiveRecord, :active_record do
     specify { expect(described_class.new(City.where(rating: 10)).default_scope).to eq(City.where(rating: 10)) }
   end
 
+  describe '.new' do
+    context 'with logger' do
+      let(:test_logger) { Logger.new('/dev/null') }
+      let(:default_scope_behavior) { :warn }
+
+      around do |example|
+        previous_logger = Chewy.logger
+        Chewy.logger = test_logger
+
+        previous_default_scope_behavior = Chewy.config.import_scope_cleanup_behavior
+        Chewy.config.import_scope_cleanup_behavior = default_scope_behavior
+
+        example.run
+      ensure
+        Chewy.logger = previous_logger
+        Chewy.config.import_scope_cleanup_behavior = previous_default_scope_behavior
+      end
+
+      specify do
+        expect(test_logger).to receive(:warn)
+        described_class.new(City.order(:id))
+      end
+
+      specify do
+        expect(test_logger).to receive(:warn)
+        described_class.new(City.offset(10))
+      end
+
+      specify do
+        expect(test_logger).to receive(:warn)
+        described_class.new(City.limit(10))
+      end
+
+      context 'ignore import scope warning' do
+        let(:default_scope_behavior) { :ignore }
+
+        specify do
+          expect(test_logger).not_to receive(:warn)
+          described_class.new(City.order(:id))
+        end
+
+        specify do
+          expect(test_logger).not_to receive(:warn)
+          described_class.new(City.offset(10))
+        end
+
+        specify do
+          expect(test_logger).not_to receive(:warn)
+          described_class.new(City.limit(10))
+        end
+      end
+
+      context 'raise exception on import scope with order/limit/offset' do
+        let(:default_scope_behavior) { :raise }
+
+        specify { expect { described_class.new(City.order(:id)) }.to raise_error(Chewy::ImportScopeCleanupError) }
+        specify { expect { described_class.new(City.limit(10)) }.to raise_error(Chewy::ImportScopeCleanupError) }
+        specify { expect { described_class.new(City.offset(10)) }.to raise_error(Chewy::ImportScopeCleanupError) }
+      end
+    end
+  end
+
   describe '#type_name' do
     specify { expect(described_class.new(City).type_name).to eq('city') }
     specify { expect(described_class.new(City.order(:id)).type_name).to eq('city') }

data/spec/chewy/strategy/delayed_sidekiq_spec.rb

@@ -186,5 +186,17 @@ if defined?(Sidekiq)
         end
       end
     end
+
+    describe '#clear_delayed_sidekiq_timechunks test helper' do
+      it 'clears redis from the timechunk sorted sets to avoid leak between tests' do
+        timechunks_set = -> { Sidekiq.redis { |redis| redis.zrange('chewy:delayed_sidekiq:CitiesIndex:timechunks', 0, -1) } }
+
+        expect { CitiesIndex.import!([1], strategy: :delayed_sidekiq) }
+          .to change { timechunks_set.call.size }.by(1)
+
+        expect { Chewy::Strategy::DelayedSidekiq.clear_timechunks! }
+          .to change { timechunks_set.call.size }.to(0)
+      end
+    end
   end
 end

data/spec/chewy_spec.rb

@@ -57,18 +57,21 @@ describe Chewy do
 
     before do
       Chewy.current[:chewy_client] = nil
-      allow(Chewy).to receive_messages(configuration: {transport_options: {proc: faraday_block}})
+    end
+
+    specify do
+      expect(Chewy).to receive_messages(configuration: {transport_options: {proc: faraday_block}})
 
-      allow(Elasticsearch::Client).to receive(:new).with(expected_client_config) do |*_args, &passed_block|
+      expect(Elasticsearch::Client).to receive(:new).with(expected_client_config) do |*_args, &passed_block|
         # RSpec's `with(..., &block)` was used previously, but doesn't actually do
         # any verification of the passed block (even of its presence).
         expect(passed_block.source_location).to eq(faraday_block.source_location)
 
         mock_client
       end
-    end
 
-    its(:client) { is_expected.to eq(mock_client) }
+      expect(Chewy.client).to be_a(Chewy::ElasticClient)
+    end
 
     after { Chewy.current[:chewy_client] = initial_client }
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chewy
 version: !ruby/object:Gem::Version
-  version: 7.4.0
+  version: 7.5.1
 platform: ruby
 authors:
 - Toptal, LLC
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-13 00:00:00.000000000 Z
+date: 2024-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -95,6 +95,7 @@ files:
 - gemfiles/rails.7.1.activerecord.gemfile
 - lib/chewy.rb
 - lib/chewy/config.rb
+- lib/chewy/elastic_client.rb
 - lib/chewy/errors.rb
 - lib/chewy/fields/base.rb
 - lib/chewy/fields/root.rb
@@ -204,6 +205,7 @@ files:
 - lib/tasks/chewy.rake
 - migration_guide.md
 - spec/chewy/config_spec.rb
+- spec/chewy/elastic_client_spec.rb
 - spec/chewy/fields/base_spec.rb
 - spec/chewy/fields/root_spec.rb
 - spec/chewy/fields/time_fields_spec.rb
@@ -318,7 +320,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Elasticsearch ODM client wrapper