chewy 7.4.0 → 7.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5afbccc50c9249e0a78e4f2383cef346360cef2fe9f1015d05972f4df31c26f
-  data.tar.gz: 3f0238841652782eff513dfdaa4916e8151469db5749e95e661f9c0236c526f1
+  metadata.gz: 1b939ad8c84f1499caf873e2db0ba62794b785da0297604a38404c0f4f68a930
+  data.tar.gz: bba3a09f52a65c3433b385c227c2f900645b145fba18e11737d7a9b2958d51ef
 SHA512:
-  metadata.gz: e46394c42fc5cadabd70e5917a795bc071667c4bdf69a2b0d1d1a53c1db547a9b3221839ca8463c738030192bf135998a2de5c3652c87b6523cd08f7e69d80ad
-  data.tar.gz: 0bbf17766315cd0bf21bd9ec5e402584212515094cd086a003c08e23594083e2ac5b44758f27b1750b82f284260c0eb8e4cb8d12e32357cf770efc917da6595e
+  metadata.gz: cb4f984274557d91d9d4d08306516ca4b0592bf97de3b3432585bfa9da14906d64d809da7803f19f36ba908cd8623843398e30bc78db22951cfb73015bf60a4f
+  data.tar.gz: 02ecef485783e555bd112240d74c75832779b8415aded37b8eb527357729e78053caead1615928fd1d54fa053a146779bb281d2bdf9f26f448c1ee680fea9bee

data/CHANGELOG.md

@@ -8,6 +8,13 @@
 
 ### Bugs Fixed
 
+## 7.5.0 (2023-01-15)
+
+### New Features
+
+* [#894](https://github.com/toptal/chewy/pull/894): Way of cleaning redis from artifacts left by `delayed_sidekiq` strategy which could potentially cause flaky tests. ([@Drowze](https://github.com/Drowze))
+* [#919](https://github.com/toptal/chewy/pull/919): Add pre-request filter ([@konalegi][https://github.com/konalegi])
+
 ## 7.4.0 (2023-12-13)
 
 ### New Features
@@ -18,7 +25,7 @@
 
 ### Bugs Fixed
 
-## 7.3.5 (2023-12-13)
+## 7.3.6 (2023-12-13)
 
 ### New Features
 

data/README.md

@@ -848,6 +848,12 @@ Explicit call of the reindex using `:delayed_sidekiq` strategy with `:update_fie
 CitiesIndex.import([1, 2, 3], update_fields: [:name], strategy: :delayed_sidekiq)
 ```
 
+While running tests with delayed_sidekiq strategy and Sidekiq is using a real redis instance that is NOT cleaned up in between tests (via e.g. `Sidekiq.redis(&:flushdb)`), you'll want to cleanup some redis keys in between tests to avoid state leaking and flaky tests. Chewy provides a convenience method for that:
+```ruby
+# it might be a good idea to also add to your testing setup, e.g.: a rspec `before` hook
+Chewy::Strategy::DelayedSidekiq.clear_timechunks!
+```
+
 #### `:active_job`
 
 This does the same thing as `:atomic`, but using ActiveJob. This will inherit the ActiveJob configuration settings including the `active_job.queue_adapter` setting for the environment. Patch `Chewy::Strategy::ActiveJob::Worker` for index updates improving.
@@ -1275,6 +1281,23 @@ If you use `DatabaseCleaner` in your tests with [the `transaction` strategy](htt
 Chewy.use_after_commit_callbacks = !Rails.env.test?
 ```
 
+### Pre-request Filter
+
+Should you need to inspect the query prior to it being dispatched to ElasticSearch during any queries, you can use the `before_es_request_filter`. `before_es_request_filter` is a callable object, as demonstrated below:
+
+```ruby
+Chewy.before_es_request_filter = -> (method_name, args, kw_args) { ... }
+```
+
+While using the `before_es_request_filter`, please consider the following:
+
+* `before_es_request_filter` acts as a simple proxy before any request made via the    `ElasticSearch::Client`. The arguments passed to this filter include:
+  * `method_name` -  The name of the method being called. Examples are search, count, bulk and etc.
+  * `args` and `kw_args` - These are the positional arguments provided in the method call.
+* The operation is synchronous, so avoid executing any heavy or time-consuming operations within the filter to prevent performance degradation.
+* The return value of the proc is disregarded. This filter is intended for inspection or modification of the query rather than generating a response.
+* Any exception raised inside the callback will propagate upward and halt the execution of the query. It is essential to handle potential errors adequately to ensure the stability of your search functionality.
+
 ## Contributing
 
 1. Fork it (http://github.com/toptal/chewy/fork)

data/lib/chewy/config.rb

@@ -38,7 +38,9 @@ module Chewy
                   # for type mappings like `_all`.
                   :default_root_options,
                   # Default field type for any field in any Chewy type. Defaults to 'text'.
-                  :default_field_type
+                  :default_field_type,
+                  # Callback called on each search request to be done into ES
+                  :before_es_request_filter
 
     attr_reader :transport_logger, :transport_tracer,
                 # Chewy search request DSL base class, used by every index.

data/lib/chewy/elastic_client.rb

@@ -0,0 +1,31 @@
+module Chewy
+  # Replacement for Chewy.client
+  class ElasticClient
+    def self.build_es_client(configuration = Chewy.configuration)
+      client_configuration = configuration.deep_dup
+      client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client
+      block = client_configuration[:transport_options].try(:delete, :proc)
+      ::Elasticsearch::Client.new(client_configuration, &block)
+    end
+
+    def initialize(elastic_client = self.class.build_es_client)
+      @elastic_client = elastic_client
+    end
+
+  private
+
+    def method_missing(name, *args, **kwargs, &block)
+      inspect_payload(name, args, kwargs)
+
+      @elastic_client.__send__(name, *args, **kwargs, &block)
+    end
+
+    def respond_to_missing?(name, _include_private = false)
+      @elastic_client.respond_to?(name) || super
+    end
+
+    def inspect_payload(name, args, kwargs)
+      Chewy.config.before_es_request_filter&.call(name, args, kwargs)
+    end
+  end
+end

data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb

@@ -18,6 +18,7 @@ module Chewy
         DEFAULT_MARGIN = 2
         DEFAULT_QUEUE = 'chewy'
         KEY_PREFIX = 'chewy:delayed_sidekiq'
+        ALL_SETS_KEY = "#{KEY_PREFIX}:all_sets".freeze
         FALLBACK_FIELDS = 'all'
         FIELDS_IDS_SEPARATOR = ';'
         IDS_SEPARATOR = ','
@@ -68,8 +69,10 @@ module Chewy
           ::Sidekiq.redis do |redis|
             # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
             if redis.respond_to?(:sadd?)
+              redis.sadd?(ALL_SETS_KEY, timechunks_key)
               redis.sadd?(timechunk_key, serialize_data)
             else
+              redis.sadd(ALL_SETS_KEY, timechunks_key)
               redis.sadd(timechunk_key, serialize_data)
             end
 

data/lib/chewy/strategy/delayed_sidekiq.rb

@@ -5,6 +5,19 @@ module Chewy
     class DelayedSidekiq < Sidekiq
       require_relative 'delayed_sidekiq/scheduler'
 
+      # cleanup the redis sets used internally. Useful mainly in tests to avoid
+      # leak and potential flaky tests.
+      def self.clear_timechunks!
+        ::Sidekiq.redis do |redis|
+          timechunk_sets = redis.smembers(Chewy::Strategy::DelayedSidekiq::Scheduler::ALL_SETS_KEY)
+          break if timechunk_sets.empty?
+
+          redis.pipelined do |pipeline|
+            timechunk_sets.each { |set| pipeline.del(set) }
+          end
+        end
+      end
+
       def leave
         @stash.each do |type, ids|
           next if ids.empty?

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.4.0'.freeze
+  VERSION = '7.5.0'.freeze
 end

data/lib/chewy.rb

@@ -49,6 +49,7 @@ require 'chewy/fields/base'
 require 'chewy/fields/root'
 require 'chewy/journal'
 require 'chewy/railtie' if defined?(Rails::Railtie)
+require 'chewy/elastic_client'
 
 ActiveSupport.on_load(:active_record) do
   include Chewy::Index::Observe::ActiveRecordMethods
@@ -97,12 +98,7 @@ module Chewy
     # Main elasticsearch-ruby client instance
     #
     def client
-      Chewy.current[:chewy_client] ||= begin
-        client_configuration = configuration.deep_dup
-        client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client
-        block = client_configuration[:transport_options].try(:delete, :proc)
-        ::Elasticsearch::Client.new(client_configuration, &block)
-      end
+      Chewy.current[:chewy_client] ||= Chewy::ElasticClient.new
     end
 
     # Sends wait_for_status request to ElasticSearch with status

data/spec/chewy/elastic_client_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Chewy::ElasticClient do
+  describe 'payload inspection' do
+    let(:filter) { instance_double('Proc') }
+    let!(:filter_previous_value) { Chewy.before_es_request_filter }
+
+    before do
+      Chewy.massacre
+      stub_index(:products) do
+        field :id, type: :integer
+      end
+      ProductsIndex.create
+      Chewy.before_es_request_filter = filter
+    end
+
+    after do
+      Chewy.before_es_request_filter = filter_previous_value
+    end
+
+    it 'call filter with the request body' do
+      expect(filter).to receive(:call).with(:search, [{body: {size: 0}, index: ['products']}], {})
+      Chewy.client.search({index: ['products'], body: {size: 0}}).to_a
+    end
+  end
+end

data/spec/chewy/strategy/delayed_sidekiq_spec.rb

@@ -186,5 +186,17 @@ if defined?(Sidekiq)
         end
       end
     end
+
+    describe '#clear_delayed_sidekiq_timechunks test helper' do
+      it 'clears redis from the timechunk sorted sets to avoid leak between tests' do
+        timechunks_set = -> { Sidekiq.redis { |redis| redis.zrange('chewy:delayed_sidekiq:CitiesIndex:timechunks', 0, -1) } }
+
+        expect { CitiesIndex.import!([1], strategy: :delayed_sidekiq) }
+          .to change { timechunks_set.call.size }.by(1)
+
+        expect { Chewy::Strategy::DelayedSidekiq.clear_timechunks! }
+          .to change { timechunks_set.call.size }.to(0)
+      end
+    end
   end
 end

data/spec/chewy_spec.rb

@@ -57,18 +57,21 @@ describe Chewy do
 
     before do
       Chewy.current[:chewy_client] = nil
-      allow(Chewy).to receive_messages(configuration: {transport_options: {proc: faraday_block}})
+    end
+
+    specify do
+      expect(Chewy).to receive_messages(configuration: {transport_options: {proc: faraday_block}})
 
-      allow(Elasticsearch::Client).to receive(:new).with(expected_client_config) do |*_args, &passed_block|
+      expect(Elasticsearch::Client).to receive(:new).with(expected_client_config) do |*_args, &passed_block|
         # RSpec's `with(..., &block)` was used previously, but doesn't actually do
         # any verification of the passed block (even of its presence).
         expect(passed_block.source_location).to eq(faraday_block.source_location)
 
         mock_client
       end
-    end
 
-    its(:client) { is_expected.to eq(mock_client) }
+      expect(Chewy.client).to be_a(Chewy::ElasticClient)
+    end
 
     after { Chewy.current[:chewy_client] = initial_client }
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chewy
 version: !ruby/object:Gem::Version
-  version: 7.4.0
+  version: 7.5.0
 platform: ruby
 authors:
 - Toptal, LLC
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-13 00:00:00.000000000 Z
+date: 2024-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -95,6 +95,7 @@ files:
 - gemfiles/rails.7.1.activerecord.gemfile
 - lib/chewy.rb
 - lib/chewy/config.rb
+- lib/chewy/elastic_client.rb
 - lib/chewy/errors.rb
 - lib/chewy/fields/base.rb
 - lib/chewy/fields/root.rb
@@ -204,6 +205,7 @@ files:
 - lib/tasks/chewy.rake
 - migration_guide.md
 - spec/chewy/config_spec.rb
+- spec/chewy/elastic_client_spec.rb
 - spec/chewy/fields/base_spec.rb
 - spec/chewy/fields/root_spec.rb
 - spec/chewy/fields/time_fields_spec.rb