sidekiq-grouping 1.0.10 → 1.3.0

data/lib/sidekiq/grouping/redis.rb

@@ -1,82 +1,105 @@
+# frozen_string_literal: true
+
+require_relative "./redis_dispatcher"
+
 module Sidekiq
   module Grouping
     class Redis
+      include RedisDispatcher
 
       PLUCK_SCRIPT = <<-SCRIPT
-        local pluck_values = redis.call('lrange', KEYS[1], 0, ARGV[1] - 1)
-        redis.call('ltrim', KEYS[1], ARGV[1], -1)
-        for k, v in pairs(pluck_values) do
-          redis.call('srem', KEYS[2], v)
+        local pluck_values = redis.call('lpop', KEYS[1], ARGV[1]) or {}
+        if #pluck_values > 0 then
+          redis.call('srem', KEYS[2], unpack(pluck_values))
         end
         return pluck_values
       SCRIPT
 
-      def push_msg(name, msg, remember_unique = false)
+      def push_msg(name, msg, remember_unique: false)
         redis do |conn|
-          conn.multi do
-            conn.sadd(ns('batches'), name)
-            conn.rpush(ns(name), msg)
-            conn.sadd(unique_messages_key(name), msg) if remember_unique
+          conn.multi do |pipeline|
+            sadd = pipeline.respond_to?(:sadd?) ? :sadd? : :sadd
+            redis_connection_call(pipeline, sadd, ns("batches"), name)
+            redis_connection_call(pipeline, :rpush, ns(name), msg)
+
+            if remember_unique
+              redis_connection_call(
+                pipeline, sadd, unique_messages_key(name), msg
+              )
+            end
           end
         end
       end
 
       def enqueued?(name, msg)
-        redis do |conn|
-          conn.sismember(unique_messages_key(name), msg)
-        end
+        member = redis_call(:sismember, unique_messages_key(name), msg)
+        return member if member.is_a?(TrueClass) || member.is_a?(FalseClass)
+
+        member != 0
       end
 
       def batch_size(name)
-        redis { |conn| conn.llen(ns(name)) }
+        redis_call(:llen, ns(name))
       end
 
       def batches
-        redis { |conn| conn.smembers(ns('batches')) }
+        redis_call(:smembers, ns("batches"))
       end
 
       def pluck(name, limit)
-        keys = [ns(name), unique_messages_key(name)]
-        args = [limit]
-        redis { |conn| conn.eval PLUCK_SCRIPT, keys, args }
+        if new_redis_client?
+          redis_call(
+            :eval,
+            PLUCK_SCRIPT,
+            2,
+            ns(name),
+            unique_messages_key(name),
+            limit
+          )
+        else
+          keys = [ns(name), unique_messages_key(name)]
+          args = [limit]
+          redis_call(:eval, PLUCK_SCRIPT, keys, args)
+        end
       end
 
       def get_last_execution_time(name)
-        redis { |conn| conn.get(ns("last_execution_time:#{name}")) }
+        redis_call(:get, ns("last_execution_time:#{name}"))
       end
 
       def set_last_execution_time(name, time)
-        redis { |conn| conn.set(ns("last_execution_time:#{name}"), time.to_json) }
+        redis_call(
+          :set, ns("last_execution_time:#{name}"), time.to_json
+        )
       end
 
       def lock(name)
-        redis do |conn|
-          id = ns("lock:#{name}")
-          conn.set(id, true, nx: true, ex: Sidekiq::Grouping::Config.lock_ttl)
-        end
+        redis_call(
+          :set,
+          ns("lock:#{name}"),
+          "true",
+          nx: true,
+          ex: Sidekiq::Grouping::Config.lock_ttl
+        )
       end
 
       def delete(name)
         redis do |conn|
-          conn.del(ns("last_execution_time:#{name}"))
-          conn.del(ns(name))
-          conn.srem(ns('batches'), name)
+          redis_connection_call(conn, :del, ns("last_execution_time:#{name}"))
+          redis_connection_call(conn, :del, ns(name))
+          redis_connection_call(conn, :srem, ns("batches"), name)
         end
       end
 
       private
 
-      def unique_messages_key name
+      def unique_messages_key(name)
         ns("#{name}:unique_messages")
       end
 
       def ns(key = nil)
         "batching:#{key}"
       end
-
-      def redis(&block)
-        Sidekiq.redis(&block)
-      end
     end
   end
 end

data/lib/sidekiq/grouping/redis_dispatcher.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Grouping
+    module RedisDispatcher
+      def redis_call(command, *args, **kwargs)
+        redis do |connection|
+          redis_connection_call(connection, command, *args, **kwargs)
+        end
+      end
+
+      def redis_connection_call(connection, command, *args, **kwargs)
+        if new_redis_client? # redis-client
+          connection.call(command.to_s.upcase, *args, **kwargs)
+        else # redis
+          connection.public_send(command, *args, **kwargs)
+        end
+      end
+
+      def new_redis_client?
+        Sidekiq::VERSION[0].to_i >= 7
+      end
+
+      def redis(&block)
+        Sidekiq.redis(&block)
+      end
+    end
+  end
+end

data/lib/sidekiq/grouping/version.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Sidekiq
   module Grouping
-    VERSION = "1.0.10"
+    VERSION = "1.3.0"
   end
 end

data/lib/sidekiq/grouping/web.rb

@@ -1,24 +1,29 @@
-require 'sidekiq/web'
+# frozen_string_literal: true
+
+require "sidekiq/web"
 
 module Sidekiq
   module Grouping
     module Web
-      VIEWS = File.expand_path('views', File.dirname(__FILE__))
+      VIEWS = File.expand_path("views", File.dirname(__FILE__))
 
       def self.registered(app)
         app.get "/grouping" do
           @batches = Sidekiq::Grouping::Batch.all
-          erb File.read(File.join(VIEWS, 'index.erb')), locals: {view_path: VIEWS}
+          erb File.read(File.join(VIEWS, "index.erb")),
+              locals: { view_path: VIEWS }
         end
 
         app.post "/grouping/:name/delete" do
-          worker_class, queue = Sidekiq::Grouping::Batch.extract_worker_klass_and_queue(params['name'])
+          worker_class, queue =
+            Sidekiq::Grouping::Batch.extract_worker_klass_and_queue(
+              params["name"]
+            )
           batch = Sidekiq::Grouping::Batch.new(worker_class, queue)
           batch.delete
           redirect "#{root_path}grouping"
         end
       end
-
     end
   end
 end

data/lib/sidekiq/grouping.rb

@@ -1,39 +1,46 @@
+# frozen_string_literal: true
+
+require "active_support"
 require "active_support/core_ext/string"
 require "active_support/configurable"
 require "active_support/core_ext/numeric/time"
+require "sidekiq"
 require "sidekiq/grouping/version"
 require "concurrent"
 
-module Sidekiq::Grouping
-  autoload :Config, "sidekiq/grouping/config"
-  autoload :Redis, "sidekiq/grouping/redis"
-  autoload :Batch, "sidekiq/grouping/batch"
-  autoload :Middleware, "sidekiq/grouping/middleware"
-  autoload :Flusher, "sidekiq/grouping/flusher"
-  autoload :FlusherObserver, "sidekiq/grouping/flusher_observer"
-
-  class << self
-    attr_writer :logger
-
-    def logger
-      @logger ||= Sidekiq.logger
-    end
-
-    def force_flush_for_test!
-      Sidekiq::Grouping::Flusher.new.force_flush_for_test!
-    end
-
-    def start!
-      interval = Sidekiq::Grouping::Config.poll_interval
-      @observer = Sidekiq::Grouping::FlusherObserver.new
-      @task = Concurrent::TimerTask.new(
-        execution_interval: interval
-      ) { Sidekiq::Grouping::Flusher.new.flush }
-      @task.add_observer(@observer)
-      logger.info(
-        "[Sidekiq::Grouping] Started polling batches every #{interval} seconds"
-      )
-      @task.execute
+module Sidekiq
+  module Grouping
+    autoload :Config, "sidekiq/grouping/config"
+    autoload :Redis, "sidekiq/grouping/redis"
+    autoload :Batch, "sidekiq/grouping/batch"
+    autoload :Middleware, "sidekiq/grouping/middleware"
+    autoload :Flusher, "sidekiq/grouping/flusher"
+    autoload :FlusherObserver, "sidekiq/grouping/flusher_observer"
+
+    class << self
+      attr_writer :logger
+
+      def logger
+        @logger ||= Sidekiq.logger
+      end
+
+      def force_flush_for_test!
+        Sidekiq::Grouping::Flusher.new.force_flush_for_test!
+      end
+
+      def start!
+        interval = Sidekiq::Grouping::Config.poll_interval
+        @observer = Sidekiq::Grouping::FlusherObserver.new
+        @task = Concurrent::TimerTask.new(
+          execution_interval: interval
+        ) { Sidekiq::Grouping::Flusher.new.flush }
+        @task.add_observer(@observer)
+        logger.info(
+          "[Sidekiq::Grouping] Started polling batches every " \
+          "#{interval} seconds"
+        )
+        @task.execute
+      end
     end
   end
 end

data/sidekiq-grouping.gemspec

@@ -1,4 +1,6 @@
-lib = File.expand_path("../lib", __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "sidekiq/grouping/version"
 
@@ -7,26 +9,31 @@ Gem::Specification.new do |spec|
   spec.version       = Sidekiq::Grouping::VERSION
   spec.authors       = ["Victor Sokolov"]
   spec.email         = ["gzigzigzeo@gmail.com"]
-  spec.summary       = %q(
+  spec.summary       = <<~SUMMARY
     Allows identical sidekiq jobs to be processed with a single background call
-  )
+  SUMMARY
   spec.homepage      = "http://github.com/gzigzigzeo/sidekiq-grouping"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.required_ruby_version = ">= 2.7.0"
+
+  spec.add_development_dependency "appraisal"
+  spec.add_development_dependency "bundler", "> 1.5"
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
-  spec.add_development_dependency "simplecov"
   spec.add_development_dependency "rspec-sidekiq"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "rubocop-rspec"
+  spec.add_development_dependency "simplecov"
   spec.add_development_dependency "timecop"
-  spec.add_development_dependency "appraisal"
 
   spec.add_dependency "activesupport"
-  spec.add_dependency "sidekiq", ">= 3.4.2"
   spec.add_dependency "concurrent-ruby"
+  spec.add_dependency "sidekiq", ">= 3.4.2"
+  spec.metadata["rubygems_mfa_required"] = "true"
 end

data/spec/modules/batch_spec.rb

@@ -1,70 +1,72 @@
-require 'spec_helper'
+# frozen_string_literal: true
+
+require "spec_helper"
 
 describe Sidekiq::Grouping::Batch do
-  subject { Sidekiq::Grouping::Batch }
+  subject(:batch_service) { described_class }
 
-  context 'adding' do
-    it 'must enqueue unbatched worker' do
-       RegularWorker.perform_async('bar')
-       expect(RegularWorker).to have_enqueued_sidekiq_job("bar")
+  context "when adding" do
+    it "must enqueue unbatched worker" do
+      RegularWorker.perform_async("bar")
+      expect(RegularWorker).to have_enqueued_sidekiq_job("bar")
     end
 
-    it 'must not enqueue batched worker' do
-      BatchedSizeWorker.perform_async('bar')
-      expect_batch(BatchedSizeWorker, 'batched_size')
+    it "must not enqueue batched worker based on batch size setting" do
+      BatchedSizeWorker.perform_async("bar")
+      expect_batch(BatchedSizeWorker, "batched_size")
     end
 
-    it 'must not enqueue batched worker' do
-      BatchedIntervalWorker.perform_async('bar')
-      expect_batch(BatchedIntervalWorker, 'batched_interval')
+    it "must not enqueue batched worker based on interval setting" do
+      BatchedIntervalWorker.perform_async("bar")
+      expect_batch(BatchedIntervalWorker, "batched_interval")
     end
 
-    it 'must not enqueue batched worker' do
-      BatchedBothWorker.perform_async('bar')
-      expect_batch(BatchedBothWorker, 'batched_both')
+    it "must not enqueue batched worker based on both settings" do
+      BatchedBothWorker.perform_async("bar")
+      expect_batch(BatchedBothWorker, "batched_both")
     end
   end
 
-  context 'checking if should flush' do
-    it 'must flush if limit exceeds for limit worker' do
-      batch = subject.new(BatchedSizeWorker.name, 'batched_size')
+  context "when checking if should flush" do
+    it "must flush if limit exceeds for limit worker", :aggregate_failures do
+      batch = batch_service.new(BatchedSizeWorker.name, "batched_size")
 
-      expect(batch.could_flush?).to be_falsy
-      BatchedSizeWorker.perform_async('bar')
-      expect(batch.could_flush?).to be_falsy
-      4.times { BatchedSizeWorker.perform_async('bar') }
-      expect(batch.could_flush?).to be_truthy
+      expect(batch).not_to be_could_flush
+      BatchedSizeWorker.perform_async("bar")
+      expect(batch).not_to be_could_flush
+      4.times { BatchedSizeWorker.perform_async("bar") }
+      expect(batch).to be_could_flush
     end
 
-    it 'must flush if limit exceeds for both worker' do
-      batch = subject.new(BatchedBothWorker.name, 'batched_both')
+    it "must flush if limit exceeds for both worker", :aggregate_failures do
+      batch = batch_service.new(BatchedBothWorker.name, "batched_both")
 
-      expect(batch.could_flush?).to be_falsy
-      BatchedBothWorker.perform_async('bar')
-      expect(batch.could_flush?).to be_falsy
-      4.times { BatchedBothWorker.perform_async('bar') }
-      expect(batch.could_flush?).to be_truthy
+      expect(batch).not_to be_could_flush
+      BatchedBothWorker.perform_async("bar")
+      expect(batch).not_to be_could_flush
+      4.times { BatchedBothWorker.perform_async("bar") }
+      expect(batch).to be_could_flush
     end
 
-    it 'must flush if limit okay but time came' do
-      batch = subject.new(BatchedIntervalWorker.name, 'batched_interval')
+    it "must flush if limit okay but time came", :aggregate_failures do
+      batch = batch_service.new(BatchedIntervalWorker.name, "batched_interval")
 
-      expect(batch.could_flush?).to be_falsy
-      BatchedIntervalWorker.perform_async('bar')
-      expect(batch.could_flush?).to be_falsy
+      expect(batch).not_to be_could_flush
+      BatchedIntervalWorker.perform_async("bar")
+      expect(batch).not_to be_could_flush
       expect(batch.size).to eq(1)
 
       Timecop.travel(2.hours.since)
 
-      expect(batch.could_flush?).to be_truthy
+      expect(batch).to be_could_flush
     end
   end
 
-  context 'flushing' do
-    it 'must put wokrer to queue on flush' do
-      batch = subject.new(BatchedSizeWorker.name, 'batched_size')
+  context "when flushing" do
+    it "must put worker to queue on flush", :aggregate_failures do
+      batch = batch_service.new(BatchedSizeWorker.name, "batched_size")
 
-      expect(batch.could_flush?).to be_falsy
+      expect(batch).not_to be_could_flush
       10.times { |n| BatchedSizeWorker.perform_async("bar#{n}") }
       batch.flush
       expect(BatchedSizeWorker).to(
@@ -74,65 +76,93 @@ describe Sidekiq::Grouping::Batch do
     end
   end
 
-  context 'with similar args' do
-    context 'option batch_unique = true' do
-      it 'enqueues once' do
-        batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
-        3.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+  context "with similar args" do
+    context "when option batch_unique = true" do
+      it "enqueues once" do
+        batch = batch_service.new(
+          BatchedUniqueArgsWorker.name,
+          "batched_unique_args"
+        )
+        3.times { BatchedUniqueArgsWorker.perform_async("bar", 1) }
         expect(batch.size).to eq(1)
       end
 
-      it 'enqueues once each unique set of args' do
-        batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
-        3.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
-        6.times { BatchedUniqueArgsWorker.perform_async('baz', 1) }
-        3.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
-        2.times { BatchedUniqueArgsWorker.perform_async('baz', 3) }
-        7.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+      it "enqueues once each unique set of args" do
+        batch = batch_service.new(
+          BatchedUniqueArgsWorker.name,
+          "batched_unique_args"
+        )
+        3.times { BatchedUniqueArgsWorker.perform_async("bar", 1) }
+        6.times { BatchedUniqueArgsWorker.perform_async("baz", 1) }
+        3.times { BatchedUniqueArgsWorker.perform_async("bar", 1) }
+        2.times { BatchedUniqueArgsWorker.perform_async("baz", 3) }
+        7.times { BatchedUniqueArgsWorker.perform_async("bar", 1) }
         expect(batch.size).to eq(3)
       end
 
-      context 'flushing' do
-
-        it 'works' do
-          batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
-          2.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
-          2.times { BatchedUniqueArgsWorker.perform_async('baz', 1) }
-          batch.flush
-          expect(batch.size).to eq(0)
-        end
-
-        it 'allows to enqueue again after flush' do
-          batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
-          2.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
-          2.times { BatchedUniqueArgsWorker.perform_async('baz', 1) }
-          batch.flush
-          BatchedUniqueArgsWorker.perform_async('bar', 1)
-          BatchedUniqueArgsWorker.perform_async('baz', 1)
-          expect(batch.size).to eq(2)
-        end
+      it "flushes the workers" do
+        batch = batch_service.new(
+          BatchedUniqueArgsWorker.name,
+          "batched_unique_args"
+        )
+        2.times { BatchedUniqueArgsWorker.perform_async("bar", 1) }
+        2.times { BatchedUniqueArgsWorker.perform_async("baz", 1) }
+        batch.flush
+        expect(batch.size).to eq(0)
       end
 
+      it "allows to enqueue again after flush" do
+        batch = batch_service.new(
+          BatchedUniqueArgsWorker.name,
+          "batched_unique_args"
+        )
+        2.times { BatchedUniqueArgsWorker.perform_async("bar", 1) }
+        2.times { BatchedUniqueArgsWorker.perform_async("baz", 1) }
+        batch.flush
+        BatchedUniqueArgsWorker.perform_async("bar", 1)
+        BatchedUniqueArgsWorker.perform_async("baz", 1)
+        expect(batch.size).to eq(2)
+      end
     end
 
-    context 'batch_unique is not specified' do
-      it 'enqueues all' do
-        batch = subject.new(BatchedSizeWorker.name, 'batched_size')
-        3.times { BatchedSizeWorker.perform_async('bar', 1) }
+    context "when batch_unique is not specified" do
+      it "enqueues all" do
+        batch = batch_service.new(BatchedSizeWorker.name, "batched_size")
+        3.times { BatchedSizeWorker.perform_async("bar", 1) }
         expect(batch.size).to eq(3)
       end
     end
   end
 
+  context "when inline mode" do
+    it "must pass args to worker as array" do
+      Sidekiq::Testing.inline! do
+        expect_any_instance_of(BatchedSizeWorker)
+          .to receive(:perform).with([[1]])
+
+        BatchedSizeWorker.perform_async(1)
+      end
+    end
+
+    it "must not pass args to worker as array" do
+      Sidekiq::Testing.inline! do
+        expect_any_instance_of(RegularWorker).to receive(:perform).with(1)
+
+        RegularWorker.perform_async(1)
+      end
+    end
+  end
+
   private
-  def expect_batch(klass, queue)
-    expect(klass).to_not have_enqueued_sidekiq_job("bar")
-    batch = subject.new(klass.name, queue)
-    stats = subject.all
+
+  def expect_batch(klass, queue) # rubocop:disable Metrics/AbcSize
+    expect(klass).not_to have_enqueued_sidekiq_job("bar")
+    batch = batch_service.new(klass.name, queue)
+    stats = batch_service.all
     expect(batch.size).to eq(1)
     expect(stats.size).to eq(1)
     expect(stats.first.worker_class).to eq(klass.name)
     expect(stats.first.queue).to eq(queue)
-    expect(batch.pluck).to eq [['bar']]
+    expect(batch.pluck).to eq [["bar"]]
   end
 end

data/spec/modules/redis_spec.rb

@@ -1,47 +1,44 @@
-require 'spec_helper'
+# frozen_string_literal: true
+
+require "spec_helper"
 
 describe Sidekiq::Grouping::Redis do
-  subject { Sidekiq::Grouping::Redis.new }
+  include Sidekiq::Grouping::RedisDispatcher
+
+  subject(:redis_service) { described_class.new }
 
   let(:queue_name)    { "my_queue" }
   let(:key)           { "batching:#{queue_name}" }
   let(:unique_key)    { "batching:#{queue_name}:unique_messages" }
 
   describe "#push_msg" do
-    it "adds message to queue" do
-      subject.push_msg(queue_name, 'My message')
-      expect(redis { |c| c.llen key }).to eq 1
-      expect(redis { |c| c.lrange key, 0, 1 }).to eq ['My message']
-      expect(redis { |c| c.smembers unique_key}).to eq []
+    it "adds message to queue", :aggregate_failures do
+      redis_service.push_msg(queue_name, "My message")
+      expect(redis_call(:llen, key)).to eq 1
+      expect(redis_call(:lrange, key, 0, 1)).to eq ["My message"]
+      expect(redis_call(:smembers, unique_key)).to eq []
     end
 
     it "remembers unique message if specified" do
-      subject.push_msg(queue_name, 'My message', true)
-      expect(redis { |c| c.smembers unique_key}).to eq ['My message']
+      redis_service.push_msg(queue_name, "My message", remember_unique: true)
+      expect(redis_call(:smembers, unique_key)).to eq ["My message"]
     end
   end
 
   describe "#pluck" do
     it "removes messages from queue" do
-      subject.push_msg(queue_name, "Message 1")
-      subject.push_msg(queue_name, "Message 2")
-      subject.pluck(queue_name, 2)
-      expect(redis { |c| c.llen key }).to eq 0
+      redis_service.push_msg(queue_name, "Message 1")
+      redis_service.push_msg(queue_name, "Message 2")
+      redis_service.pluck(queue_name, 2)
+      expect(redis_call(:llen, key)).to eq 0
     end
 
-    it "forgets unique messages" do
-      subject.push_msg(queue_name, "Message 1", true)
-      subject.push_msg(queue_name, "Message 2", true)
-      expect(redis { |c| c.scard unique_key }).to eq 2
-      subject.pluck(queue_name, 2)
-      expect(redis { |c| c.smembers unique_key }).to eq []
+    it "forgets unique messages", :aggregate_failures do
+      redis_service.push_msg(queue_name, "Message 1", remember_unique: true)
+      redis_service.push_msg(queue_name, "Message 2", remember_unique: true)
+      expect(redis_call(:scard, unique_key)).to eq 2
+      redis_service.pluck(queue_name, 2)
+      expect(redis_call(:smembers, unique_key)).to eq []
     end
   end
-
-  private
-
-  def redis(&block)
-    Sidekiq.redis(&block)
-  end
-
 end