sidekiq-grouping 1.0.10 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd82358a1dc3ea13dcebc3d113272316e3c334fa3e562dad7eb2828590626988
-  data.tar.gz: 90a9116faf2392b7b5d81ce80346afd3741c897bf60b215eaac43f954e5fcb38
+  metadata.gz: be68edcca66201c059ee0a9d18c163cf899a00e1d5013a64ca72206dc9b48e9e
+  data.tar.gz: 55bfb74ecd86ce73a4fedd115e8adfb1a8531ab3460e008a74ce6c4b9c111892
 SHA512:
-  metadata.gz: f08a2fe6ed7811eefae95487143f0db3beb5343c3c3b11b2e7472b805b9eb0a5b6394c84b933219bf2648e6706a463bfc2cd30f1f1e623f48e665d9dec452b0d
-  data.tar.gz: ca99dbc220c9e757549bfd5659ac0673db96f1eadb394b2a396eb7581df482156c83bb107eee98e91744f0f7d93663d393d25f1b7996aac2df84f5e09d69baa3
+  metadata.gz: 709bbcf1bf236c18e330adba5fd5c0c23a5cce73f869fcc152c3b846f306c28189c3942bb6bfde1875f1895e0a799e7367e1e4bddd4abb8c6147f16490101841
+  data.tar.gz: d46cb5d01ccaaae7c52f7768e8f8159a8be287e58852f6c167d4ebfbf01e37da70640b40b4476c868fa01a4874d717bbb4c0cbe9108418d1180c4a2c9d95c3ce

data/.github/workflows/lint.yml

@@ -0,0 +1,37 @@
+---
+name: Lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  rubocop:
+    name: Rubocop
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Ruby ${{ matrix.ruby_version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler: 2
+          ruby-version: 2.7
+
+      - uses: actions/cache@v3
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-${{ matrix.ruby-version }}-${{ hashFiles('./*.gemspec') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-
+
+      - name: Bundle install
+        run: |
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3
+
+      - name: Run Rubocop
+        run: bundle exec rubocop

data/.github/workflows/test.yml

@@ -0,0 +1,49 @@
+---
+name: Test
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  rspec:
+    name: Rspec
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis:6
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+    strategy:
+      matrix:
+        include:
+          - { ruby_version: 2.7 }
+          - { ruby_version: 3.0 }
+          - { ruby_version: 3.1 }
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Ruby ${{ matrix.ruby_version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler: 2
+          ruby-version: ${{ matrix.ruby_version }}
+
+      # Appraisal doesn't support vendored install
+      # See: https://github.com/thoughtbot/appraisal/issues/173
+      #      https://github.com/thoughtbot/appraisal/pull/174
+      - name: Bundle install
+        run: |
+          bundle install --jobs 4 --retry 3
+          bundle exec appraisal install
+
+      - name: Run tests
+        run: bundle exec appraisal rspec

data/.rubocop.yml

@@ -1,8 +1,15 @@
+---
 require: rubocop-rspec
 
 AllCops:
+  TargetRubyVersion: 2.7.0
+  SuggestExtensions: false
+  NewCops: enable
   Include:
     - ./Gemfile
+    - ./Rakefile
+    - '*.gemspec'
+    - '**/*.rb'
 
 Documentation:
   Enabled: false
@@ -10,8 +17,29 @@ Documentation:
 Style/StringLiterals:
   EnforcedStyle: double_quotes
 
-Style/ClassAndModuleChildren:
-  EnforcedStyle: compact
-
 RSpec/FilePath:
   Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/AnyInstance:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 15
+
+Metrics/ClassLength:
+  Max: 150
+
+Layout/LineLength:
+  Max: 80
+
+Layout/FirstArgumentIndentation:
+  EnforcedStyle: consistent
+
+Layout/FirstMethodArgumentLineBreak:
+  Enabled: true
+
+Layout/MultilineMethodArgumentLineBreaks:
+  Enabled: true

data/Appraisals

@@ -14,6 +14,10 @@ appraise 'sidekiq-5.0' do
   gem 'sidekiq', '~> 5.0.0'
 end
 
-appraise 'sidekiq-master' do
-  gem 'sidekiq', github: 'mperham/sidekiq'
+appraise 'sidekiq-6.0' do
+  gem 'sidekiq', '~> 6.0.0'
+end
+
+appraise 'sidekiq-6.5' do
+  gem 'sidekiq', '~> 6.5.0'
 end

data/README.md

@@ -1,3 +1,5 @@
+[![Gem Version](https://badge.fury.io/rb/sidekiq-grouping.svg)](https://rubygems.org/gems/sidekiq-grouping)
+
 # Sidekiq::Grouping
 
 <a href="https://evilmartians.com/?utm_source=sidekiq-grouping-gem">
@@ -23,7 +25,7 @@ class ElasticBulkIndexWorker
   include Sidekiq::Worker
 
   sidekiq_options(
-    queue: :elasic_bulks,
+    queue: :elastic_bulks,
     batch_flush_size: 30,     # Jobs will be combined when queue size exceeds 30
     batch_flush_interval: 60, # Jobs will be combined every 60 seconds
     retry: 5

data/gemfiles/sidekiq_6.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", "~> 6.0.0"
+
+gemspec path: "../"

data/gemfiles/sidekiq_6.5.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", "~> 6.5.0"
+
+gemspec path: "../"

data/lefthook.yml

@@ -0,0 +1,15 @@
+---
+# Git hooks configuration
+#
+# See: github.com/evilmartians/lefthook
+
+pre-commit:
+  commands:
+    rubocop:
+      glob: "*.rb"
+      run: bundle exec rubocop -A {staged_files} && git add {staged_files}
+
+pre-push:
+  commands:
+    rspec:
+      run: bundle exec appraisal rspec

data/lib/sidekiq/grouping/batch.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module Sidekiq
   module Grouping
     class Batch
-      def initialize(worker_class, queue, redis_pool = nil)
+      def initialize(worker_class, queue, _redis_pool = nil)
         @worker_class = worker_class
         @queue = queue
         @name = "#{worker_class.underscore}:#{queue}"
@@ -12,11 +14,18 @@ module Sidekiq
 
       def add(msg)
         msg = msg.to_json
-        @redis.push_msg(@name, msg, enqueue_similar_once?) if should_add? msg
+        return unless should_add? msg
+
+        @redis.push_msg(
+          @name,
+          msg,
+          remember_unique: enqueue_similar_once?
+        )
       end
 
-      def should_add? msg
+      def should_add?(msg)
         return true unless enqueue_similar_once?
+
         !@redis.enqueued?(@name, msg)
       end
 
@@ -25,19 +34,19 @@ module Sidekiq
       end
 
       def chunk_size
-        worker_class_options['batch_size'] ||
+        worker_class_options["batch_size"] ||
           Sidekiq::Grouping::Config.max_batch_size
       end
 
       def pluck_size
-        worker_class_options['batch_flush_size'] ||
+        worker_class_options["batch_flush_size"] ||
           chunk_size
       end
 
       def pluck
-        if @redis.lock(@name)
-          @redis.pluck(@name, pluck_size).map { |value| JSON.parse(value) }
-        end
+        return unless @redis.lock(@name)
+
+        @redis.pluck(@name, pluck_size).map { |value| JSON.parse(value) }
       end
 
       def flush
@@ -46,9 +55,9 @@ module Sidekiq
 
         chunk.each_slice(chunk_size) do |subchunk|
           Sidekiq::Client.push(
-            'class' => @worker_class,
-            'queue' => @queue,
-            'args' => [true, subchunk]
+            "class" => @worker_class,
+            "queue" => @queue,
+            "args" => [true, subchunk]
           )
         end
         set_current_time_as_last
@@ -74,10 +83,11 @@ module Sidekiq
       end
 
       def next_execution_time
-        if interval = worker_class_options['batch_flush_interval']
-          last_time = last_execution_time
-          last_time + interval.seconds if last_time
-        end
+        interval = worker_class_options["batch_flush_interval"]
+        return unless interval
+
+        last_time = last_execution_time
+        last_time + interval.seconds if last_time
       end
 
       def delete
@@ -99,13 +109,13 @@ module Sidekiq
         if last_time.blank?
           set_current_time_as_last
           false
-        else
-          next_time < Time.now if next_time
+        elsif next_time
+          next_time < Time.now
         end
       end
 
       def enqueue_similar_once?
-        worker_class_options['batch_unique'] == true
+        worker_class_options["batch_unique"] == true
       end
 
       def set_current_time_as_last
@@ -122,7 +132,7 @@ module Sidekiq
         end
 
         def extract_worker_klass_and_queue(name)
-          klass, queue = name.split(':')
+          klass, queue = name.split(":")
           [klass.camelize, queue]
         end
       end

data/lib/sidekiq/grouping/config.rb

@@ -1,29 +1,39 @@
-module Sidekiq::Grouping::Config
-  include ActiveSupport::Configurable
+# frozen_string_literal: true
 
-  def self.options
-    Sidekiq.options[:grouping] || Sidekiq.options["grouping"] || {} # sidekiq 5.x use symbol in keys
-  end
+module Sidekiq
+  module Grouping
+    module Config
+      include ActiveSupport::Configurable
 
-  # Queue size overflow check polling interval
-  config_accessor :poll_interval do
-    options[:poll_interval] || 3
-  end
+      def self.options
+        if Sidekiq.respond_to?(:[])
+          Sidekiq[:grouping] || Sidekiq["grouping"] || {}
+        else
+          Sidekiq.options[:grouping] || Sidekiq.options["grouping"] || {}
+        end
+      end
 
-  # Maximum batch size
-  config_accessor :max_batch_size do
-    options[:max_batch_size] || 1000
-  end
+      # Queue size overflow check polling interval
+      config_accessor :poll_interval do
+        options[:poll_interval] || 3
+      end
 
-  # Batch queue flush lock timeout
-  config_accessor :lock_ttl do
-    options[:lock_ttl] || 1
-  end
+      # Maximum batch size
+      config_accessor :max_batch_size do
+        options[:max_batch_size] || 1000
+      end
+
+      # Batch queue flush lock timeout
+      config_accessor :lock_ttl do
+        options[:lock_ttl] || 1
+      end
 
-  # Option to override how Sidekiq::Grouping know about tests env
-  config_accessor :tests_env do
-    options[:tests_env] || (
-      defined?(::Rails) && Rails.respond_to?(:env) && Rails.env.test?
-    )
+      # Option to override how Sidekiq::Grouping know about tests env
+      config_accessor :tests_env do
+        options[:tests_env] || (
+          defined?(::Rails) && Rails.respond_to?(:env) && Rails.env.test?
+        )
+      end
+    end
   end
 end

data/lib/sidekiq/grouping/flusher.rb

@@ -1,42 +1,54 @@
-class Sidekiq::Grouping::Flusher
-  def flush
-    batches = Sidekiq::Grouping::Batch.all.map do |batch|
-      batch if batch.could_flush?
-    end
-    flush_batches(batches)
-  end
+# frozen_string_literal: true
 
-  def force_flush_for_test!
-    unless Sidekiq::Grouping::Config.tests_env
-      Sidekiq::Grouping.logger.warn(
-        "**************************************************"
-      )
-      Sidekiq::Grouping.logger.warn([
-        "⛔️ force_flush_for_test! for testing API, ",
-        "but this is not the test environment. ",
-        "Please check your environment or ",
-        "change 'tests_env' to cover this one"
-      ].join)
-      Sidekiq::Grouping.logger.warn(
-        "**************************************************"
-      )
-    end
-    flush_batches(Sidekiq::Grouping::Batch.all)
-  end
+module Sidekiq
+  module Grouping
+    class Flusher
+      def flush
+        batches = Sidekiq::Grouping::Batch.all.map do |batch|
+          batch if batch.could_flush?
+        end
+        flush_batches(batches)
+      end
 
-  private
+      def force_flush_for_test!
+        unless Sidekiq::Grouping::Config.tests_env
+          Sidekiq::Grouping.logger.warn(
+            "**************************************************"
+          )
+          Sidekiq::Grouping.logger.warn(
+            "⛔️ force_flush_for_test! for testing API, " \
+            "but this is not the test environment. " \
+            "Please check your environment or " \
+            "change 'tests_env' to cover this one"
+          )
+          Sidekiq::Grouping.logger.warn(
+            "**************************************************"
+          )
+        end
+        flush_batches(Sidekiq::Grouping::Batch.all)
+      end
 
-  def flush_batches(batches)
-    batches.compact!
-    flush_concrete(batches)
-  end
+      private
+
+      def flush_batches(batches)
+        batches.compact!
+        flush_concrete(batches)
+      end
 
-  def flush_concrete(batches)
-    return if batches.empty?
-    names = batches.map { |batch| "#{batch.worker_class} in #{batch.queue}" }
-    Sidekiq::Grouping.logger.info(
-      "[Sidekiq::Grouping] Trying to flush batched queues: #{names.join(',')}"
-    ) unless Sidekiq::Grouping::Config.tests_env
-    batches.each(&:flush)
+      def flush_concrete(batches)
+        return if batches.empty?
+
+        names = batches.map do |batch|
+          "#{batch.worker_class} in #{batch.queue}"
+        end
+        unless Sidekiq::Grouping::Config.tests_env
+          Sidekiq::Grouping.logger.info(
+            "[Sidekiq::Grouping] Trying to flush batched queues: " \
+            "#{names.join(',')}"
+          )
+        end
+        batches.each(&:flush)
+      end
+    end
   end
 end

data/lib/sidekiq/grouping/flusher_observer.rb

@@ -1,13 +1,19 @@
-class Sidekiq::Grouping::FlusherObserver
-  def update(time, _result, ex)
-    if ex.is_a?(Concurrent::TimeoutError)
-      Sidekiq::Grouping.logger.error(
-        "[Sidekiq::Grouping] (#{time}) Execution timed out\n"
-      )
-    elsif ex.present?
-      Sidekiq::Grouping.logger.error(
-        "[Sidekiq::Grouping] Execution failed with error #{ex}\n"
-      )
+# frozen_string_literal: true
+
+module Sidekiq
+  module Grouping
+    class FlusherObserver
+      def update(time, _result, exception)
+        if exception.is_a?(Concurrent::TimeoutError)
+          Sidekiq::Grouping.logger.error(
+            "[Sidekiq::Grouping] (#{time}) Execution timed out\n"
+          )
+        elsif exception.present?
+          Sidekiq::Grouping.logger.error(
+            "[Sidekiq::Grouping] Execution failed with error #{exception}\n"
+          )
+        end
+      end
     end
   end
 end

data/lib/sidekiq/grouping/middleware.rb

@@ -1,43 +1,56 @@
+# frozen_string_literal: true
+
 module Sidekiq
   module Grouping
     class Middleware
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       def call(worker_class, msg, queue, redis_pool = nil)
-        return yield if (defined?(Sidekiq::Testing) && Sidekiq::Testing.inline?)
-
-        worker_class = worker_class.camelize.constantize if worker_class.is_a?(String)
+        if worker_class.is_a?(String)
+          worker_class = worker_class.camelize.constantize
+        end
         options = worker_class.get_sidekiq_options
 
         batch =
-          options.key?('batch_flush_size') ||
-          options.key?('batch_flush_interval') ||
-          options.key?('batch_size')
+          options.key?("batch_flush_size") ||
+          options.key?("batch_flush_interval") ||
+          options.key?("batch_size")
 
         passthrough =
-          msg['args'] &&
-          msg['args'].is_a?(Array) &&
-          msg['args'].try(:first) == true
+          msg["args"].is_a?(Array) &&
+          msg["args"].try(:first) == true
 
         retrying = msg["failed_at"].present?
 
         return yield unless batch
 
-        if !(passthrough || retrying)
-          add_to_batch(worker_class, queue, msg, redis_pool)
-        else
-          msg['args'].shift if passthrough
+        if inline_mode?
+          wrapped_args = [[msg["args"]]]
+          msg["args"] = wrapped_args
+          return yield
+        end
+
+        if passthrough || retrying
+          msg["args"].shift if passthrough
           yield
+        else
+          add_to_batch(worker_class, queue, msg, redis_pool)
         end
       end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
 
       private
 
       def add_to_batch(worker_class, queue, msg, redis_pool = nil)
         Sidekiq::Grouping::Batch
           .new(worker_class.name, queue, redis_pool)
-          .add(msg['args'])
+          .add(msg["args"])
 
         nil
       end
+
+      def inline_mode?
+        defined?(Sidekiq::Testing) && Sidekiq::Testing.inline?
+      end
     end
   end
 end

data/lib/sidekiq/grouping/redis.rb

@@ -1,22 +1,29 @@
+# frozen_string_literal: true
+
 module Sidekiq
   module Grouping
     class Redis
-
       PLUCK_SCRIPT = <<-SCRIPT
-        local pluck_values = redis.call('lrange', KEYS[1], 0, ARGV[1] - 1)
-        redis.call('ltrim', KEYS[1], ARGV[1], -1)
-        for k, v in pairs(pluck_values) do
-          redis.call('srem', KEYS[2], v)
+        local pluck_values = redis.call('lpop', KEYS[1], ARGV[1]) or {}
+        if #pluck_values > 0 then
+          redis.call('srem', KEYS[2], unpack(pluck_values))
         end
         return pluck_values
       SCRIPT
 
-      def push_msg(name, msg, remember_unique = false)
+      def push_msg(name, msg, remember_unique: false)
         redis do |conn|
-          conn.multi do
-            conn.sadd(ns('batches'), name)
-            conn.rpush(ns(name), msg)
-            conn.sadd(unique_messages_key(name), msg) if remember_unique
+          conn.multi do |pipeline|
+            sadd = pipeline.respond_to?(:sadd?) ? :sadd? : :sadd
+            pipeline.public_send(sadd, ns("batches"), name)
+            pipeline.rpush(ns(name), msg)
+            if remember_unique
+              pipeline.public_send(
+                sadd,
+                unique_messages_key(name),
+                msg
+              )
+            end
           end
         end
       end
@@ -32,7 +39,7 @@ module Sidekiq
       end
 
       def batches
-        redis { |conn| conn.smembers(ns('batches')) }
+        redis { |conn| conn.smembers(ns("batches")) }
       end
 
       def pluck(name, limit)
@@ -46,7 +53,9 @@ module Sidekiq
       end
 
       def set_last_execution_time(name, time)
-        redis { |conn| conn.set(ns("last_execution_time:#{name}"), time.to_json) }
+        redis do |conn|
+          conn.set(ns("last_execution_time:#{name}"), time.to_json)
+        end
       end
 
       def lock(name)
@@ -60,13 +69,13 @@ module Sidekiq
         redis do |conn|
           conn.del(ns("last_execution_time:#{name}"))
           conn.del(ns(name))
-          conn.srem(ns('batches'), name)
+          conn.srem(ns("batches"), name)
         end
       end
 
       private
 
-      def unique_messages_key name
+      def unique_messages_key(name)
         ns("#{name}:unique_messages")
       end
 

data/lib/sidekiq/grouping/version.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Sidekiq
   module Grouping
-    VERSION = "1.0.10"
+    VERSION = "1.2.0"
   end
 end