chewy 7.5.0 → 7.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b939ad8c84f1499caf873e2db0ba62794b785da0297604a38404c0f4f68a930
-  data.tar.gz: bba3a09f52a65c3433b385c227c2f900645b145fba18e11737d7a9b2958d51ef
+  metadata.gz: a15165f889275fecc6a0d590c3339ce0ac9b7944f916306fd6883e7c2be67747
+  data.tar.gz: 8efc6201add68bf1c378f598934b6c52c74bd9a62056e7aafdd357358d5cd2b0
 SHA512:
-  metadata.gz: cb4f984274557d91d9d4d08306516ca4b0592bf97de3b3432585bfa9da14906d64d809da7803f19f36ba908cd8623843398e30bc78db22951cfb73015bf60a4f
-  data.tar.gz: 02ecef485783e555bd112240d74c75832779b8415aded37b8eb527357729e78053caead1615928fd1d54fa053a146779bb281d2bdf9f26f448c1ee680fea9bee
+  metadata.gz: cfc7f1297fc72fcbdfdbedbcd82c6afe12dd9e15583b4e3467a8a77da89cd1aa8f20b9d0c0a8dd1841275e2916715030e28e9ed10cf510d6985904be9574890e
+  data.tar.gz: 6917027945ce94dab50d2f6d679570be58e4bb86472f1d591805d915ca425ecbd24e4cc13e30a459fe2dae4d572002735b69cef138a5c6038c52803a251b8a54

data/.github/workflows/ruby.yml

@@ -23,6 +23,18 @@ jobs:
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
 
+    services:
+      redis:
+        # Docker Hub image
+        image: redis
+        ports:
+          - '6379:6379'
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1

data/.rubocop.yml

@@ -59,3 +59,6 @@ Metrics/ModuleLength:
   Exclude:
     - 'lib/chewy/rake_helper.rb'
     - '**/*_spec.rb'
+
+Style/ArgumentsForwarding:
+  Enabled: false

data/CHANGELOG.md

@@ -8,7 +8,28 @@
 
 ### Bugs Fixed
 
-## 7.5.0 (2023-01-15)
+## 7.6.0 (2024-05-03)
+
+### Changes
+
+* [#933](https://github.com/toptal/chewy/pull/933): Relax allowed `elasticsearch` dependency versions. ([@mjankowski][])
+
+### Bugs Fixed
+* [#937](https://github.com/toptal/chewy/pull/937): Fix for race condition while using the `delayed_sidekiq` strategy. Also, fix for Redis bloating in case of reindexing error ([@skcc321](https://github.com/skcc321))
+
+* [#947](https://github.com/toptal/chewy/pull/947): Fix intermittent time-based failure in delayed sidekiq spec. ([@mjankowski][])
+
+## 7.5.1 (2024-01-30)
+
+### New Features
+
+* [#925](https://github.com/toptal/chewy/pull/925): Add configuration option for default scope cleanup behavior. ([@barthez][])
+
+### Changes
+
+### Bugs Fixed
+
+## 7.5.0 (2024-01-15)
 
 ### New Features
 

data/README.md

@@ -776,9 +776,12 @@ Chewy.settings[:sidekiq] = {queue: :low}
 
 #### `:delayed_sidekiq`
 
-It accumulates ids of records to be reindexed during the latency window in redis and then does the reindexing of all accumulated records at once.
-The strategy is very useful in case of frequently mutated records.
-It supports `update_fields` option, so it will try to select just enough data from the DB
+It accumulates IDs of records to be reindexed during the latency window in Redis and then performs the reindexing of all accumulated records at once. 
+This strategy is very useful in the case of frequently mutated records. 
+It supports the `update_fields` option, so it will attempt to select just enough data from the database.
+
+Keep in mind, this strategy does not guarantee reindexing in the event of Sidekiq worker termination or an error during the reindexing phase. 
+This behavior is intentional to prevent continuous growth of Redis db.
 
 There are three options that can be defined in the index:
 ```ruby
@@ -1298,6 +1301,24 @@ While using the `before_es_request_filter`, please consider the following:
 * The return value of the proc is disregarded. This filter is intended for inspection or modification of the query rather than generating a response.
 * Any exception raised inside the callback will propagate upward and halt the execution of the query. It is essential to handle potential errors adequately to ensure the stability of your search functionality.
 
+### Import scope clean-up behavior
+
+Whenever you set the `import_scope` for the index, in the case of ActiveRecord,
+options for order, offset and limit will be removed. You can set the behavior of
+chewy, before the clean-up itself.
+
+The default behavior is a warning sent to the Chewy logger (`:warn`). Another more
+restrictive option is raising an exception (`:raise`). Both options have a
+negative impact on performance since verifying whether the code uses any of
+these options requires building AREL query.
+
+To avoid the loading time impact, you can ignore the check (`:ignore`) before
+the clean-up.
+
+```
+Chewy.import_scope_cleanup_behavior = :ignore
+```
+
 ## Contributing
 
 1. Fork it (http://github.com/toptal/chewy/fork)

data/chewy.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport', '>= 5.2' # Remove with major version bump, 8.x
-  spec.add_dependency 'elasticsearch', '>= 7.12.0', '< 7.14.0'
+  spec.add_dependency 'elasticsearch', '>= 7.14.0', '< 8'
   spec.add_dependency 'elasticsearch-dsl'
   spec.metadata['rubygems_mfa_required'] = 'true'
 end

data/gemfiles/base.gemfile

@@ -1,12 +1,12 @@
 gem 'database_cleaner'
 gem 'elasticsearch-extensions'
 gem 'method_source'
-gem 'mock_redis'
 gem 'rake'
+gem 'redis', require: false
 gem 'rspec', '>= 3.7.0'
 gem 'rspec-collection_matchers'
 gem 'rspec-its'
-gem 'rubocop', '1.48'
-gem 'sqlite3'
+gem 'rubocop', '1.63.4'
+gem 'sqlite3', '~> 1.4'
 gem 'timecop'
 gem 'unparser'

data/lib/chewy/config.rb

@@ -40,7 +40,10 @@ module Chewy
                   # Default field type for any field in any Chewy type. Defaults to 'text'.
                   :default_field_type,
                   # Callback called on each search request to be done into ES
-                  :before_es_request_filter
+                  :before_es_request_filter,
+                  # Behavior when import scope for index includes order, offset or limit.
+                  # Can be :ignore, :warn, :raise. Defaults to :warn
+                  :import_scope_cleanup_behavior
 
     attr_reader :transport_logger, :transport_tracer,
                 # Chewy search request DSL base class, used by every index.
@@ -62,16 +65,17 @@ module Chewy
       @indices_path = 'app/chewy'
       @default_root_options = {}
       @default_field_type = 'text'.freeze
+      @import_scope_cleanup_behavior = :warn
       @search_class = build_search_class(Chewy::Search::Request)
     end
 
     def transport_logger=(logger)
-      Chewy.client.transport.logger = logger
+      Chewy.client.transport.transport.logger = logger
       @transport_logger = logger
     end
 
     def transport_tracer=(tracer)
-      Chewy.client.transport.tracer = tracer
+      Chewy.client.transport.transport.tracer = tracer
       @transport_tracer = tracer
     end
 

data/lib/chewy/errors.rb

@@ -7,7 +7,7 @@ module Chewy
 
   class UndefinedUpdateStrategy < Error
     def initialize(_type)
-      super <<-MESSAGE
+      super(<<-MESSAGE)
   Index update strategy is undefined for current context.
   Please wrap your code with `Chewy.strategy(:strategy_name) block.`
       MESSAGE
@@ -27,7 +27,7 @@ module Chewy
           message << "        on #{documents.count} documents: #{documents}\n"
         end
       end
-      super message
+      super(message)
     end
   end
 
@@ -36,4 +36,7 @@ module Chewy
       super("`#{join_field_type}` set for the join field `#{join_field_name}` is not on the :relations list (#{relations})")
     end
   end
+
+  class ImportScopeCleanupError < Error
+  end
 end

data/lib/chewy/index/adapter/active_record.rb

@@ -13,9 +13,19 @@ module Chewy
       private
 
         def cleanup_default_scope!
-          if Chewy.logger && (@default_scope.arel.orders.present? ||
+          behavior = Chewy.config.import_scope_cleanup_behavior
+
+          if behavior != :ignore && (@default_scope.arel.orders.present? ||
              @default_scope.arel.limit.present? || @default_scope.arel.offset.present?)
-            Chewy.logger.warn('Default type scope order, limit and offset are ignored and will be nullified')
+            if behavior == :warn && Chewy.logger
+              gem_dir = File.realpath('../..', __dir__)
+              source = caller.grep_v(Regexp.new(gem_dir)).first
+              Chewy.logger.warn(
+                "Default type scope order, limit and offset are ignored and will be nullified (called from: #{source})"
+              )
+            elsif behavior == :raise
+              raise ImportScopeCleanupError, 'Default type scope order, limit and offset are ignored and will be nullified'
+            end
           end
 
           @default_scope = @default_scope.reorder(nil).limit(nil).offset(nil)

data/lib/chewy/rake_helper.rb

@@ -320,11 +320,7 @@ module Chewy
           all_indexes
         end
 
-        indexes = if except.present?
-          indexes - normalize_indexes(Array.wrap(except))
-        else
-          indexes
-        end
+        indexes -= normalize_indexes(Array.wrap(except)) if except.present?
 
         indexes.sort_by(&:derivable_name)
       end

data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb

@@ -12,13 +12,43 @@ module Chewy
     class DelayedSidekiq
       require_relative 'worker'
 
+      LUA_SCRIPT = <<~LUA
+        local timechunk_key = KEYS[1]
+        local timechunks_key = KEYS[2]
+        local serialize_data = ARGV[1]
+        local at = ARGV[2]
+        local ttl = tonumber(ARGV[3])
+
+        local schedule_job = false
+
+        -- Check if the 'sadd?' method is available
+        if redis.call('exists', 'sadd?') == 1 then
+          redis.call('sadd?', timechunk_key, serialize_data)
+        else
+          redis.call('sadd', timechunk_key, serialize_data)
+        end
+
+        -- Set expiration for timechunk_key
+        redis.call('expire', timechunk_key, ttl)
+
+        -- Check if timechunk_key exists in the sorted set
+        if not redis.call('zrank', timechunks_key, timechunk_key) then
+            -- Add timechunk_key to the sorted set
+            redis.call('zadd', timechunks_key, at, timechunk_key)
+            -- Set expiration for timechunks_key
+            redis.call('expire', timechunks_key, ttl)
+            schedule_job = true
+        end
+
+        return schedule_job
+      LUA
+
       class Scheduler
         DEFAULT_TTL = 60 * 60 * 24 # in seconds
         DEFAULT_LATENCY = 10
         DEFAULT_MARGIN = 2
         DEFAULT_QUEUE = 'chewy'
         KEY_PREFIX = 'chewy:delayed_sidekiq'
-        ALL_SETS_KEY = "#{KEY_PREFIX}:all_sets".freeze
         FALLBACK_FIELDS = 'all'
         FIELDS_IDS_SEPARATOR = ';'
         IDS_SEPARATOR = ','
@@ -67,21 +97,8 @@ module Chewy
         #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347868
         def postpone
           ::Sidekiq.redis do |redis|
-            # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
-            if redis.respond_to?(:sadd?)
-              redis.sadd?(ALL_SETS_KEY, timechunks_key)
-              redis.sadd?(timechunk_key, serialize_data)
-            else
-              redis.sadd(ALL_SETS_KEY, timechunks_key)
-              redis.sadd(timechunk_key, serialize_data)
-            end
-
-            redis.expire(timechunk_key, ttl)
-
-            unless redis.zrank(timechunks_key, timechunk_key)
-              redis.zadd(timechunks_key, at, timechunk_key)
-              redis.expire(timechunks_key, ttl)
-
+            # do the redis stuff in a single command to avoid concurrency issues
+            if redis.eval(LUA_SCRIPT, keys: [timechunk_key, timechunks_key], argv: [serialize_data, at, ttl])
               ::Sidekiq::Client.push(
                 'queue' => sidekiq_queue,
                 'at' => at + margin,

data/lib/chewy/strategy/delayed_sidekiq/worker.rb

@@ -6,13 +6,40 @@ module Chewy
       class Worker
         include ::Sidekiq::Worker
 
+        LUA_SCRIPT = <<~LUA
+          local type = ARGV[1]
+          local score = tonumber(ARGV[2])
+          local prefix = ARGV[3]
+          local timechunks_key = prefix .. ":" .. type .. ":timechunks"
+
+          -- Get timechunk_keys with scores less than or equal to the specified score
+          local timechunk_keys = redis.call('zrangebyscore', timechunks_key, '-inf', score)
+
+          -- Get all members from the sets associated with the timechunk_keys
+          local members = {}
+          for _, timechunk_key in ipairs(timechunk_keys) do
+              local set_members = redis.call('smembers', timechunk_key)
+              for _, member in ipairs(set_members) do
+                  table.insert(members, member)
+              end
+          end
+
+          -- Remove timechunk_keys and their associated sets
+          for _, timechunk_key in ipairs(timechunk_keys) do
+              redis.call('del', timechunk_key)
+          end
+
+          -- Remove timechunks with scores less than or equal to the specified score
+          redis.call('zremrangebyscore', timechunks_key, '-inf', score)
+
+          return members
+        LUA
+
         def perform(type, score, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
 
           ::Sidekiq.redis do |redis|
-            timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks"
-            timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score)
-            members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact
+            members = redis.eval(LUA_SCRIPT, keys: [], argv: [type, score, Scheduler::KEY_PREFIX])
 
             # extract ids and fields & do the reset of records
             ids, fields = extract_ids_and_fields(members)
@@ -22,9 +49,6 @@ module Chewy
             index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
               options.any? ? index.import!(ids, **options) : index.import!(ids)
             end
-
-            redis.del(timechunk_keys)
-            redis.zremrangebyscore(timechunks_key, -1, score)
           end
         end
 

data/lib/chewy/strategy/delayed_sidekiq.rb

@@ -9,11 +9,11 @@ module Chewy
       # leak and potential flaky tests.
       def self.clear_timechunks!
         ::Sidekiq.redis do |redis|
-          timechunk_sets = redis.smembers(Chewy::Strategy::DelayedSidekiq::Scheduler::ALL_SETS_KEY)
-          break if timechunk_sets.empty?
+          keys_to_delete = redis.keys("#{Scheduler::KEY_PREFIX}*")
 
-          redis.pipelined do |pipeline|
-            timechunk_sets.each { |set| pipeline.del(set) }
+          # Delete keys one by one
+          keys_to_delete.each do |key|
+            redis.del(key)
           end
         end
       end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.5.0'.freeze
+  VERSION = '7.6.0'.freeze
 end

data/spec/chewy/config_spec.rb

@@ -22,7 +22,7 @@ describe Chewy::Config do
 
     specify do
       expect { subject.transport_logger = logger }
-        .to change { Chewy.client.transport.logger }.to(logger)
+        .to change { Chewy.client.transport.transport.logger }.to(logger)
     end
     specify do
       expect { subject.transport_logger = logger }
@@ -40,7 +40,7 @@ describe Chewy::Config do
 
     specify do
       expect { subject.transport_tracer = tracer }
-        .to change { Chewy.client.transport.tracer }.to(tracer)
+        .to change { Chewy.client.transport.transport.tracer }.to(tracer)
     end
     specify do
       expect { subject.transport_tracer = tracer }

data/spec/chewy/index/adapter/active_record_spec.rb

@@ -35,6 +35,68 @@ describe Chewy::Index::Adapter::ActiveRecord, :active_record do
     specify { expect(described_class.new(City.where(rating: 10)).default_scope).to eq(City.where(rating: 10)) }
   end
 
+  describe '.new' do
+    context 'with logger' do
+      let(:test_logger) { Logger.new('/dev/null') }
+      let(:default_scope_behavior) { :warn }
+
+      around do |example|
+        previous_logger = Chewy.logger
+        Chewy.logger = test_logger
+
+        previous_default_scope_behavior = Chewy.config.import_scope_cleanup_behavior
+        Chewy.config.import_scope_cleanup_behavior = default_scope_behavior
+
+        example.run
+      ensure
+        Chewy.logger = previous_logger
+        Chewy.config.import_scope_cleanup_behavior = previous_default_scope_behavior
+      end
+
+      specify do
+        expect(test_logger).to receive(:warn)
+        described_class.new(City.order(:id))
+      end
+
+      specify do
+        expect(test_logger).to receive(:warn)
+        described_class.new(City.offset(10))
+      end
+
+      specify do
+        expect(test_logger).to receive(:warn)
+        described_class.new(City.limit(10))
+      end
+
+      context 'ignore import scope warning' do
+        let(:default_scope_behavior) { :ignore }
+
+        specify do
+          expect(test_logger).not_to receive(:warn)
+          described_class.new(City.order(:id))
+        end
+
+        specify do
+          expect(test_logger).not_to receive(:warn)
+          described_class.new(City.offset(10))
+        end
+
+        specify do
+          expect(test_logger).not_to receive(:warn)
+          described_class.new(City.limit(10))
+        end
+      end
+
+      context 'raise exception on import scope with order/limit/offset' do
+        let(:default_scope_behavior) { :raise }
+
+        specify { expect { described_class.new(City.order(:id)) }.to raise_error(Chewy::ImportScopeCleanupError) }
+        specify { expect { described_class.new(City.limit(10)) }.to raise_error(Chewy::ImportScopeCleanupError) }
+        specify { expect { described_class.new(City.offset(10)) }.to raise_error(Chewy::ImportScopeCleanupError) }
+      end
+    end
+  end
+
   describe '#type_name' do
     specify { expect(described_class.new(City).type_name).to eq('city') }
     specify { expect(described_class.new(City.order(:id)).type_name).to eq('city') }

data/spec/chewy/strategy/delayed_sidekiq_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 if defined?(Sidekiq)
   require 'sidekiq/testing'
-  require 'mock_redis'
+  require 'redis'
 
   describe Chewy::Strategy::DelayedSidekiq do
     around do |example|
@@ -10,9 +10,10 @@ if defined?(Sidekiq)
     end
 
     before do
-      redis = MockRedis.new
+      redis = Redis.new
       allow(Sidekiq).to receive(:redis).and_yield(redis)
       Sidekiq::Worker.clear_all
+      described_class.clear_timechunks!
     end
 
     before do
@@ -35,7 +36,7 @@ if defined?(Sidekiq)
 
     it "respects 'refresh: false' options" do
       allow(Chewy).to receive(:disable_refresh_async).and_return(true)
-      expect(CitiesIndex).to receive(:import!).with([city.id, other_city.id], refresh: false)
+      expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), refresh: false)
       scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id, other_city.id])
       scheduler.postpone
       Chewy::Strategy::DelayedSidekiq::Worker.drain
@@ -47,7 +48,7 @@ if defined?(Sidekiq)
           expect(Sidekiq::Client).to receive(:push).with(
             hash_including(
               'queue' => 'chewy',
-              'at' => (Time.current.to_i.ceil(-1) + 2.seconds).to_i,
+              'at' => expected_at_time.to_i,
               'class' => Chewy::Strategy::DelayedSidekiq::Worker,
               'args' => ['CitiesIndex', an_instance_of(Integer)]
             )
@@ -62,6 +63,11 @@ if defined?(Sidekiq)
           end
         end
       end
+
+      def expected_at_time
+        target = described_class::Scheduler::DEFAULT_LATENCY.seconds.from_now.to_i
+        target - (target % described_class::Scheduler::DEFAULT_LATENCY) + described_class::Scheduler::DEFAULT_MARGIN.seconds
+      end
     end
 
     context 'with custom config' do
@@ -103,7 +109,7 @@ if defined?(Sidekiq)
     context 'two reindex call within the timewindow' do
       it 'accumulates all ids does the reindex one time' do
         Timecop.freeze do
-          expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once
+          expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once
           scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id])
           scheduler.postpone
           scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
@@ -115,7 +121,7 @@ if defined?(Sidekiq)
       context 'one call with update_fields another one without update_fields' do
         it 'does reindex of all fields' do
           Timecop.freeze do
-            expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once
+            expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
             scheduler.postpone
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
@@ -128,7 +134,7 @@ if defined?(Sidekiq)
       context 'both calls with different update fields' do
         it 'deos reindex with union of fields' do
           Timecop.freeze do
-            expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id], update_fields: %w[description name]).once
+            expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), update_fields: %w[name description]).once
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
             scheduler.postpone
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description'])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chewy
 version: !ruby/object:Gem::Version
-  version: 7.5.0
+  version: 7.6.0
 platform: ruby
 authors:
 - Toptal, LLC
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-15 00:00:00.000000000 Z
+date: 2024-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -31,20 +31,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.12.0
+        version: 7.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: '8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.12.0
+        version: 7.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: elasticsearch-dsl
   requirement: !ruby/object:Gem::Requirement