chewy 7.5.1 → 7.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64f061f379d740c56b7af2678ec03e3c80c1bd4c072b2aa3f68bf7803d4bc0fe
-  data.tar.gz: 6226557f9cec700d91473f36bb8623ff5bbf838211463188ac0f83eaf0ab227d
+  metadata.gz: a15165f889275fecc6a0d590c3339ce0ac9b7944f916306fd6883e7c2be67747
+  data.tar.gz: 8efc6201add68bf1c378f598934b6c52c74bd9a62056e7aafdd357358d5cd2b0
 SHA512:
-  metadata.gz: 8cb278cc5fa838a97e0c9fb94c90ea08d4a84effb041a809f850a267514c61ce78f1a919a8a0fdc5119ca968e217fe9b0aa38c307f9589f8413501f690efb314
-  data.tar.gz: 50ad4271e4948b22b626202843be7107fcc720fd3da490c14a372ca018ab0e35c0e151d5dc5b55f23b7c382a0d19cf72f7cc63e40a2dc1fc0ca286c1de717941
+  metadata.gz: cfc7f1297fc72fcbdfdbedbcd82c6afe12dd9e15583b4e3467a8a77da89cd1aa8f20b9d0c0a8dd1841275e2916715030e28e9ed10cf510d6985904be9574890e
+  data.tar.gz: 6917027945ce94dab50d2f6d679570be58e4bb86472f1d591805d915ca425ecbd24e4cc13e30a459fe2dae4d572002735b69cef138a5c6038c52803a251b8a54

data/.github/workflows/ruby.yml

@@ -23,6 +23,18 @@ jobs:
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
 
+    services:
+      redis:
+        # Docker Hub image
+        image: redis
+        ports:
+          - '6379:6379'
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1

data/CHANGELOG.md

@@ -8,6 +8,17 @@
 
 ### Bugs Fixed
 
+## 7.6.0 (2024-05-03)
+
+### Changes
+
+* [#933](https://github.com/toptal/chewy/pull/933): Relax allowed `elasticsearch` dependency versions. ([@mjankowski][])
+
+### Bugs Fixed
+* [#937](https://github.com/toptal/chewy/pull/937): Fix for race condition while using the `delayed_sidekiq` strategy. Also, fix for Redis bloating in case of reindexing error ([@skcc321](https://github.com/skcc321))
+
+* [#947](https://github.com/toptal/chewy/pull/947): Fix intermittent time-based failure in delayed sidekiq spec. ([@mjankowski][])
+
 ## 7.5.1 (2024-01-30)
 
 ### New Features

data/README.md

@@ -776,9 +776,12 @@ Chewy.settings[:sidekiq] = {queue: :low}
 
 #### `:delayed_sidekiq`
 
-It accumulates ids of records to be reindexed during the latency window in redis and then does the reindexing of all accumulated records at once.
-The strategy is very useful in case of frequently mutated records.
-It supports `update_fields` option, so it will try to select just enough data from the DB
+It accumulates IDs of records to be reindexed during the latency window in Redis and then performs the reindexing of all accumulated records at once. 
+This strategy is very useful in the case of frequently mutated records. 
+It supports the `update_fields` option, so it will attempt to select just enough data from the database.
+
+Keep in mind, this strategy does not guarantee reindexing in the event of Sidekiq worker termination or an error during the reindexing phase. 
+This behavior is intentional to prevent continuous growth of Redis db.
 
 There are three options that can be defined in the index:
 ```ruby

data/chewy.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport', '>= 5.2' # Remove with major version bump, 8.x
-  spec.add_dependency 'elasticsearch', '>= 7.12.0', '< 7.14.0'
+  spec.add_dependency 'elasticsearch', '>= 7.14.0', '< 8'
   spec.add_dependency 'elasticsearch-dsl'
   spec.metadata['rubygems_mfa_required'] = 'true'
 end

data/gemfiles/base.gemfile

@@ -1,12 +1,12 @@
 gem 'database_cleaner'
 gem 'elasticsearch-extensions'
 gem 'method_source'
-gem 'mock_redis'
 gem 'rake'
+gem 'redis', require: false
 gem 'rspec', '>= 3.7.0'
 gem 'rspec-collection_matchers'
 gem 'rspec-its'
-gem 'rubocop', '1.60.1'
-gem 'sqlite3'
+gem 'rubocop', '1.63.4'
+gem 'sqlite3', '~> 1.4'
 gem 'timecop'
 gem 'unparser'

data/lib/chewy/config.rb

@@ -70,12 +70,12 @@ module Chewy
     end
 
     def transport_logger=(logger)
-      Chewy.client.transport.logger = logger
+      Chewy.client.transport.transport.logger = logger
       @transport_logger = logger
     end
 
     def transport_tracer=(tracer)
-      Chewy.client.transport.tracer = tracer
+      Chewy.client.transport.transport.tracer = tracer
       @transport_tracer = tracer
     end
 

data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb

@@ -12,13 +12,43 @@ module Chewy
     class DelayedSidekiq
       require_relative 'worker'
 
+      LUA_SCRIPT = <<~LUA
+        local timechunk_key = KEYS[1]
+        local timechunks_key = KEYS[2]
+        local serialize_data = ARGV[1]
+        local at = ARGV[2]
+        local ttl = tonumber(ARGV[3])
+
+        local schedule_job = false
+
+        -- Check if the 'sadd?' method is available
+        if redis.call('exists', 'sadd?') == 1 then
+          redis.call('sadd?', timechunk_key, serialize_data)
+        else
+          redis.call('sadd', timechunk_key, serialize_data)
+        end
+
+        -- Set expiration for timechunk_key
+        redis.call('expire', timechunk_key, ttl)
+
+        -- Check if timechunk_key exists in the sorted set
+        if not redis.call('zrank', timechunks_key, timechunk_key) then
+            -- Add timechunk_key to the sorted set
+            redis.call('zadd', timechunks_key, at, timechunk_key)
+            -- Set expiration for timechunks_key
+            redis.call('expire', timechunks_key, ttl)
+            schedule_job = true
+        end
+
+        return schedule_job
+      LUA
+
       class Scheduler
         DEFAULT_TTL = 60 * 60 * 24 # in seconds
         DEFAULT_LATENCY = 10
         DEFAULT_MARGIN = 2
         DEFAULT_QUEUE = 'chewy'
         KEY_PREFIX = 'chewy:delayed_sidekiq'
-        ALL_SETS_KEY = "#{KEY_PREFIX}:all_sets".freeze
         FALLBACK_FIELDS = 'all'
         FIELDS_IDS_SEPARATOR = ';'
         IDS_SEPARATOR = ','
@@ -67,21 +97,8 @@ module Chewy
         #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347868
         def postpone
           ::Sidekiq.redis do |redis|
-            # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
-            if redis.respond_to?(:sadd?)
-              redis.sadd?(ALL_SETS_KEY, timechunks_key)
-              redis.sadd?(timechunk_key, serialize_data)
-            else
-              redis.sadd(ALL_SETS_KEY, timechunks_key)
-              redis.sadd(timechunk_key, serialize_data)
-            end
-
-            redis.expire(timechunk_key, ttl)
-
-            unless redis.zrank(timechunks_key, timechunk_key)
-              redis.zadd(timechunks_key, at, timechunk_key)
-              redis.expire(timechunks_key, ttl)
-
+            # do the redis stuff in a single command to avoid concurrency issues
+            if redis.eval(LUA_SCRIPT, keys: [timechunk_key, timechunks_key], argv: [serialize_data, at, ttl])
               ::Sidekiq::Client.push(
                 'queue' => sidekiq_queue,
                 'at' => at + margin,

data/lib/chewy/strategy/delayed_sidekiq/worker.rb

@@ -6,13 +6,40 @@ module Chewy
       class Worker
         include ::Sidekiq::Worker
 
+        LUA_SCRIPT = <<~LUA
+          local type = ARGV[1]
+          local score = tonumber(ARGV[2])
+          local prefix = ARGV[3]
+          local timechunks_key = prefix .. ":" .. type .. ":timechunks"
+
+          -- Get timechunk_keys with scores less than or equal to the specified score
+          local timechunk_keys = redis.call('zrangebyscore', timechunks_key, '-inf', score)
+
+          -- Get all members from the sets associated with the timechunk_keys
+          local members = {}
+          for _, timechunk_key in ipairs(timechunk_keys) do
+              local set_members = redis.call('smembers', timechunk_key)
+              for _, member in ipairs(set_members) do
+                  table.insert(members, member)
+              end
+          end
+
+          -- Remove timechunk_keys and their associated sets
+          for _, timechunk_key in ipairs(timechunk_keys) do
+              redis.call('del', timechunk_key)
+          end
+
+          -- Remove timechunks with scores less than or equal to the specified score
+          redis.call('zremrangebyscore', timechunks_key, '-inf', score)
+
+          return members
+        LUA
+
         def perform(type, score, options = {})
           options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
 
           ::Sidekiq.redis do |redis|
-            timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks"
-            timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score)
-            members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact
+            members = redis.eval(LUA_SCRIPT, keys: [], argv: [type, score, Scheduler::KEY_PREFIX])
 
             # extract ids and fields & do the reset of records
             ids, fields = extract_ids_and_fields(members)
@@ -22,9 +49,6 @@ module Chewy
             index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
               options.any? ? index.import!(ids, **options) : index.import!(ids)
             end
-
-            redis.del(timechunk_keys)
-            redis.zremrangebyscore(timechunks_key, -1, score)
           end
         end
 

data/lib/chewy/strategy/delayed_sidekiq.rb

@@ -9,11 +9,11 @@ module Chewy
       # leak and potential flaky tests.
       def self.clear_timechunks!
         ::Sidekiq.redis do |redis|
-          timechunk_sets = redis.smembers(Chewy::Strategy::DelayedSidekiq::Scheduler::ALL_SETS_KEY)
-          break if timechunk_sets.empty?
+          keys_to_delete = redis.keys("#{Scheduler::KEY_PREFIX}*")
 
-          redis.pipelined do |pipeline|
-            timechunk_sets.each { |set| pipeline.del(set) }
+          # Delete keys one by one
+          keys_to_delete.each do |key|
+            redis.del(key)
           end
         end
       end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '7.5.1'.freeze
+  VERSION = '7.6.0'.freeze
 end

data/spec/chewy/config_spec.rb

@@ -22,7 +22,7 @@ describe Chewy::Config do
 
     specify do
       expect { subject.transport_logger = logger }
-        .to change { Chewy.client.transport.logger }.to(logger)
+        .to change { Chewy.client.transport.transport.logger }.to(logger)
     end
     specify do
       expect { subject.transport_logger = logger }
@@ -40,7 +40,7 @@ describe Chewy::Config do
 
     specify do
       expect { subject.transport_tracer = tracer }
-        .to change { Chewy.client.transport.tracer }.to(tracer)
+        .to change { Chewy.client.transport.transport.tracer }.to(tracer)
     end
     specify do
       expect { subject.transport_tracer = tracer }

data/spec/chewy/strategy/delayed_sidekiq_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 if defined?(Sidekiq)
   require 'sidekiq/testing'
-  require 'mock_redis'
+  require 'redis'
 
   describe Chewy::Strategy::DelayedSidekiq do
     around do |example|
@@ -10,9 +10,10 @@ if defined?(Sidekiq)
     end
 
     before do
-      redis = MockRedis.new
+      redis = Redis.new
       allow(Sidekiq).to receive(:redis).and_yield(redis)
       Sidekiq::Worker.clear_all
+      described_class.clear_timechunks!
     end
 
     before do
@@ -35,7 +36,7 @@ if defined?(Sidekiq)
 
     it "respects 'refresh: false' options" do
       allow(Chewy).to receive(:disable_refresh_async).and_return(true)
-      expect(CitiesIndex).to receive(:import!).with([city.id, other_city.id], refresh: false)
+      expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), refresh: false)
       scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id, other_city.id])
       scheduler.postpone
       Chewy::Strategy::DelayedSidekiq::Worker.drain
@@ -47,7 +48,7 @@ if defined?(Sidekiq)
           expect(Sidekiq::Client).to receive(:push).with(
             hash_including(
               'queue' => 'chewy',
-              'at' => (Time.current.to_i.ceil(-1) + 2.seconds).to_i,
+              'at' => expected_at_time.to_i,
               'class' => Chewy::Strategy::DelayedSidekiq::Worker,
               'args' => ['CitiesIndex', an_instance_of(Integer)]
             )
@@ -62,6 +63,11 @@ if defined?(Sidekiq)
           end
         end
       end
+
+      def expected_at_time
+        target = described_class::Scheduler::DEFAULT_LATENCY.seconds.from_now.to_i
+        target - (target % described_class::Scheduler::DEFAULT_LATENCY) + described_class::Scheduler::DEFAULT_MARGIN.seconds
+      end
     end
 
     context 'with custom config' do
@@ -103,7 +109,7 @@ if defined?(Sidekiq)
     context 'two reindex call within the timewindow' do
       it 'accumulates all ids does the reindex one time' do
         Timecop.freeze do
-          expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once
+          expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once
           scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id])
           scheduler.postpone
           scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
@@ -115,7 +121,7 @@ if defined?(Sidekiq)
       context 'one call with update_fields another one without update_fields' do
         it 'does reindex of all fields' do
           Timecop.freeze do
-            expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id]).once
+            expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id])).once
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
             scheduler.postpone
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id])
@@ -128,7 +134,7 @@ if defined?(Sidekiq)
       context 'both calls with different update fields' do
         it 'deos reindex with union of fields' do
           Timecop.freeze do
-            expect(CitiesIndex).to receive(:import!).with([other_city.id, city.id], update_fields: %w[description name]).once
+            expect(CitiesIndex).to receive(:import!).with(match_array([city.id, other_city.id]), update_fields: %w[name description]).once
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [city.id], update_fields: ['name'])
             scheduler.postpone
             scheduler = Chewy::Strategy::DelayedSidekiq::Scheduler.new(CitiesIndex, [other_city.id], update_fields: ['description'])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chewy
 version: !ruby/object:Gem::Version
-  version: 7.5.1
+  version: 7.6.0
 platform: ruby
 authors:
 - Toptal, LLC
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-30 00:00:00.000000000 Z
+date: 2024-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -31,20 +31,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.12.0
+        version: 7.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: '8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.12.0
+        version: 7.14.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: elasticsearch-dsl
   requirement: !ruby/object:Gem::Requirement
@@ -320,7 +320,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.33
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Elasticsearch ODM client wrapper