sidekiq-grouping 1.0.8 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f9c96db0ae5af0c9a398f9e2a0718b55ede63741
-  data.tar.gz: 69adc754a26f5d4ba89f876f5e39803ba06d7e1f
+SHA256:
+  metadata.gz: 4dbe0fe595dfbe28a39a687925d5078d7c1df9d7793df783cf4ed448910136f9
+  data.tar.gz: a2f48474c1ec7ec9a632999f17cf6159d15490dfa6d06ff4eb5b85221fec776e
 SHA512:
-  metadata.gz: 80ae9e4599902073a2fd7336622f9b15b59f8cd3da25121b6d319dd2cb71f2aa2df56d9e14dbfe682f7b5b1d6d806c90a73cbaadd68f28bcf6f1e1655fa736aa
-  data.tar.gz: 086a0e13a585b06858d34752866099c9efa914139decaac83832dca5571d048ef8532a48cfe0395a92370f4c3554643afcb0f7a93d2330ab0fb517c757deae86
+  metadata.gz: fb1a5cc739670f30b05cd477c3797eca72d0469307ebf4e8ea13841d49ed050b2a0814f11e29b81ce5ce7d039e99f6661110b528d8ccf22cd609654fd445b4b0
+  data.tar.gz: 31f85b457287ae7e1dee1c52e008099f9806cd520132c7bad048a42d4c67ee0550cb51610d95059b3135c7a39b2865058f2d2092623864b71c23010bebc03017

data/.gitignore

@@ -4,6 +4,7 @@
 .config
 .yardoc
 Gemfile.lock
+gemfiles/*.lock
 InstalledFiles
 _yardoc
 coverage

data/.travis.yml

@@ -1,7 +1,18 @@
 language: ruby
 rvm:
   - 2.2
+  - 2.3.1
+  - 2.4
+  - 2.5
+
 cache: bundler
 
 services:
   - redis-server
+
+gemfile:
+  - gemfiles/sidekiq_4.0.gemfile
+  - gemfiles/sidekiq_4.1.gemfile
+  - gemfiles/sidekiq_4.2.gemfile
+  - gemfiles/sidekiq_5.0.gemfile
+  - gemfiles/sidekiq_master.gemfile

data/Appraisals

@@ -0,0 +1,19 @@
+appraise 'sidekiq-4.0' do
+  gem 'sidekiq', '~> 4.0.0'
+end
+
+appraise 'sidekiq-4.1' do
+  gem 'sidekiq', '~> 4.1.0'
+end
+
+appraise 'sidekiq-4.2' do
+  gem 'sidekiq', '~> 4.2.0'
+end
+
+appraise 'sidekiq-5.0' do
+  gem 'sidekiq', '~> 5.0.0'
+end
+
+appraise 'sidekiq-master' do
+  gem 'sidekiq', github: 'mperham/sidekiq'
+end

data/README.md

@@ -111,6 +111,8 @@ This jobs will be grouped into the single job with the single argument:
   # => [[5]]
   ```
 
+- `tests_env` is used to silence some logging in test environments (see below). Default: true if `Rails.env.test?`, false otherwise.
+
 ## Web UI
 
 ![Web UI](web.png)
@@ -123,15 +125,75 @@ require "sidekiq/grouping/web"
 
 ## Configuration
 
+Specify grouping configuration inside of sidekiq.yml:
+
+```yml
+grouping:
+  :poll_interval: 5       # Amount of time between polling batches
+  :max_batch_size: 5000   # Maximum batch size allowed
+  :lock_ttl: 1            # Batch queue flush lock timeout job enqueues
+```
+
+Or set it in your code:
+
 ```ruby
-Sidekiq::Grouping::Config.poll_interval = 5     # Amount of time between polling batches
-Sidekiq::Grouping::Config.max_batch_size = 5000 # Maximum batch size allowed
-Sidekiq::Grouping::Config.lock_ttl = 1          # Batch queue flush lock timeout job enqueues
+Sidekiq::Grouping::Config.poll_interval = 5
+Sidekiq::Grouping::Config.max_batch_size = 5000
+Sidekiq::Grouping::Config.lock_ttl = 1
 ```
 
-## TODO
+Note that you should set poll_interval option inside of sidekiq.yml to take effect. Setting this param in your ruby code won't change actual polling frequency.
+
+## Testing with Sidekiq::Testing.fake!
+
+Sidekiq::Grouping uses internal queues for grouping tasks. If you need to force flush internal queues into normal Sidekiq queues, use `Sidekiq::Grouping.force_flush_for_test!`.
+
+See example:
+
+```ruby
+# worker
+class GroupedWorker
+
+  include Sidekiq::Worker
+  sidekiq_options(
+    queue: :custom_queue,
+    retry: 5,
+    batch_flush_size: 9,
+    batch_flush_interval: 10,
+    batch_size: 3,
+    batch_unique: true
+  )
 
-1. Add support redis_pool option.
+  def perform(grouped_arguments)
+    # ... important payload
+  end
+
+end
+
+# test itself
+RSpec.describe GroupedWorker, type: :worker do
+
+  describe '#perform' do
+    it 'calls perform with array of arguments' do
+      Sidekiq::Testing.fake! do
+        described_class.perform_async(1)
+        described_class.perform_async(1)
+        described_class.perform_async(2)
+        described_class.perform_async(2)
+
+        # All 4 above asks will be put to :custom_queue despite of :batch_flush_size is set to 9.
+        Sidekiq::Grouping.force_flush_for_test!
+
+        last_job = described_class.jobs.last
+        expect(last_job['args']).to eq([[[1], [2]]])
+        expect(last_job['queue']).to eq('custom_queue')
+      end
+    end
+  end
+
+end
+
+```
 
 ## Installation
 

data/gemfiles/sidekiq_4.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", "~> 4.0.0"
+
+gemspec path: "../"

data/gemfiles/sidekiq_4.1.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", "~> 4.1.0"
+
+gemspec path: "../"

data/gemfiles/sidekiq_4.2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", "~> 4.2.0"
+
+gemspec path: "../"

data/gemfiles/sidekiq_5.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", "~> 5.0.0"
+
+gemspec path: "../"

data/gemfiles/sidekiq_master.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "sidekiq", github: "mperham/sidekiq"
+
+gemspec path: "../"

data/lib/sidekiq/grouping/config.rb

@@ -1,15 +1,29 @@
 module Sidekiq::Grouping::Config
   include ActiveSupport::Configurable
 
+  def self.options
+    Sidekiq.options[:grouping] || Sidekiq.options["grouping"] || {} # sidekiq 5.x use symbol in keys
+  end
+
   # Queue size overflow check polling interval
-  config_accessor :poll_interval
-  config.poll_interval = 3
+  config_accessor :poll_interval do
+    options[:poll_interval] || 3
+  end
 
   # Maximum batch size
-  config_accessor :max_batch_size
-  config.max_batch_size = 1000
+  config_accessor :max_batch_size do
+    options[:max_batch_size] || 1000
+  end
 
   # Batch queue flush lock timeout
-  config_accessor :lock_ttl
-  config.lock_ttl = 1
+  config_accessor :lock_ttl do
+    options[:lock_ttl] || 1
+  end
+
+  # Option to override how Sidekiq::Grouping know about tests env
+  config_accessor :tests_env do
+    options[:tests_env] || (
+      defined?(::Rails) && Rails.respond_to?(:env) && Rails.env.test?
+    )
+  end
 end

data/lib/sidekiq/grouping/flusher.rb

@@ -3,18 +3,40 @@ class Sidekiq::Grouping::Flusher
     batches = Sidekiq::Grouping::Batch.all.map do |batch|
       batch if batch.could_flush?
     end
-    batches.compact!
-    flush_concrete(batches)
+    flush_batches(batches)
+  end
+
+  def force_flush_for_test!
+    unless Sidekiq::Grouping::Config.tests_env
+      Sidekiq::Grouping.logger.warn(
+        "**************************************************"
+      )
+      Sidekiq::Grouping.logger.warn([
+        "⛔️ force_flush_for_test! for testing API, ",
+        "but this is not the test environment. ",
+        "Please check your environment or ",
+        "change 'tests_env' to cover this one"
+      ].join)
+      Sidekiq::Grouping.logger.warn(
+        "**************************************************"
+      )
+    end
+    flush_batches(Sidekiq::Grouping::Batch.all)
   end
 
   private
 
+  def flush_batches(batches)
+    batches.compact!
+    flush_concrete(batches)
+  end
+
   def flush_concrete(batches)
     return if batches.empty?
     names = batches.map { |batch| "#{batch.worker_class} in #{batch.queue}" }
     Sidekiq::Grouping.logger.info(
       "[Sidekiq::Grouping] Trying to flush batched queues: #{names.join(',')}"
-    )
+    ) unless Sidekiq::Grouping::Config.tests_env
     batches.each(&:flush)
   end
 end

data/lib/sidekiq/grouping/redis.rb

@@ -13,10 +13,10 @@ module Sidekiq
 
       def push_msg(name, msg, remember_unique = false)
         redis do |conn|
-          conn.multi do
-            conn.sadd(ns('batches'), name)
-            conn.rpush(ns(name), msg)
-            conn.sadd(unique_messages_key(name), msg) if remember_unique
+          conn.multi do |pipeline|
+            pipeline.sadd(ns("batches"), name)
+            pipeline.rpush(ns(name), msg)
+            pipeline.sadd(unique_messages_key(name), msg) if remember_unique
           end
         end
       end
@@ -32,7 +32,7 @@ module Sidekiq
       end
 
       def batches
-        redis { |conn| conn.smembers(ns('batches')) }
+        redis { |conn| conn.smembers(ns("batches")) }
       end
 
       def pluck(name, limit)

data/lib/sidekiq/grouping/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Grouping
-    VERSION = "1.0.8"
+    VERSION = "1.1.0"
   end
 end

data/lib/sidekiq/grouping/web.rb

@@ -11,11 +11,11 @@ module Sidekiq
           erb File.read(File.join(VIEWS, 'index.erb')), locals: {view_path: VIEWS}
         end
 
-        app.post "/grouping/*/delete" do |name|
-          worker_class, queue = Sidekiq::Grouping::Batch.extract_worker_klass_and_queue(name)
+        app.post "/grouping/:name/delete" do
+          worker_class, queue = Sidekiq::Grouping::Batch.extract_worker_klass_and_queue(params['name'])
           batch = Sidekiq::Grouping::Batch.new(worker_class, queue)
           batch.delete
-          redirect "#{root_path}/grouping"
+          redirect "#{root_path}grouping"
         end
       end
 
@@ -25,4 +25,3 @@ end
 
 Sidekiq::Web.register(Sidekiq::Grouping::Web)
 Sidekiq::Web.tabs["Grouping"] = "grouping"
-

data/lib/sidekiq/grouping.rb

@@ -1,3 +1,4 @@
+require "active_support"
 require "active_support/core_ext/string"
 require "active_support/configurable"
 require "active_support/core_ext/numeric/time"
@@ -19,6 +20,10 @@ module Sidekiq::Grouping
       @logger ||= Sidekiq.logger
     end
 
+    def force_flush_for_test!
+      Sidekiq::Grouping::Flusher.new.force_flush_for_test!
+    end
+
     def start!
       interval = Sidekiq::Grouping::Config.poll_interval
       @observer = Sidekiq::Grouping::FlusherObserver.new

data/sidekiq-grouping.gemspec

@@ -18,12 +18,13 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "bundler", "> 1.5"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "rspec-sidekiq"
   spec.add_development_dependency "timecop"
+  spec.add_development_dependency "appraisal"
 
   spec.add_dependency "activesupport"
   spec.add_dependency "sidekiq", ">= 3.4.2"

data/spec/modules/batch_spec.rb

@@ -6,7 +6,7 @@ describe Sidekiq::Grouping::Batch do
   context 'adding' do
     it 'must enqueue unbatched worker' do
        RegularWorker.perform_async('bar')
-       expect(RegularWorker).to have_enqueued_job('bar')
+       expect(RegularWorker).to have_enqueued_sidekiq_job("bar")
     end
 
     it 'must not enqueue batched worker' do
@@ -67,7 +67,9 @@ describe Sidekiq::Grouping::Batch do
       expect(batch.could_flush?).to be_falsy
       10.times { |n| BatchedSizeWorker.perform_async("bar#{n}") }
       batch.flush
-      expect(BatchedSizeWorker).to have_enqueued_job([["bar0"], ["bar1"]])
+      expect(BatchedSizeWorker).to(
+        have_enqueued_sidekiq_job([["bar0"], ["bar1"]])
+      )
       expect(batch.size).to eq(7)
     end
   end
@@ -124,7 +126,7 @@ describe Sidekiq::Grouping::Batch do
 
   private
   def expect_batch(klass, queue)
-    expect(klass).to_not have_enqueued_job('bar')
+    expect(klass).to_not have_enqueued_sidekiq_job("bar")
     batch = subject.new(klass.name, queue)
     stats = subject.all
     expect(batch.size).to eq(1)

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-grouping
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.1.0
 platform: ruby
 authors:
 - Victor Sokolov
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-12 00:00:00.000000000 Z
+date: 2022-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '1.5'
 - !ruby/object:Gem::Dependency
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +150,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - gzigzigzeo@gmail.com
 executables: []
@@ -146,10 +160,16 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
+- Appraisals
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- gemfiles/sidekiq_4.0.gemfile
+- gemfiles/sidekiq_4.1.gemfile
+- gemfiles/sidekiq_4.2.gemfile
+- gemfiles/sidekiq_5.0.gemfile
+- gemfiles/sidekiq_master.gemfile
 - lib/sidekiq/grouping.rb
 - lib/sidekiq/grouping/batch.rb
 - lib/sidekiq/grouping/config.rb
@@ -170,7 +190,7 @@ homepage: http://github.com/gzigzigzeo/sidekiq-grouping
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -185,9 +205,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Allows identical sidekiq jobs to be processed with a single background call
 test_files: