sidekiq-grouping 0.0.6 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae9e71d0d79bd9af61ed8c49fca7122219849a37
-  data.tar.gz: c5469b43d3780b36fbe343eca132c86d3d200c34
+  metadata.gz: c53ede8cde6f7b04ee9b0ae343f988d564a62944
+  data.tar.gz: 22864e990a6ffeaf86ac5ef169072b00d054ef5f
 SHA512:
-  metadata.gz: db9e641eb35867c42e2b17eb073b921c42ad992b4846452aff33afb927ea76df2811f7262f89a578c45dbd5d48a0f0e8a0a30be5ea531a8169f1cd9ea7a2ecb8
-  data.tar.gz: 1fde67a0b91497362100c2608582aaa52e36478a960ec33d9adc99f6552dbe4d2966b713ffa8ef87a3c77391f968894c34c9694801a717d6e68b28e03f148ba7
+  metadata.gz: ea56db9c75826ec9fd2cb990ef9cd553049a5200464f3722954a53f0678f353b88ab2477be3ce9c031dc4951460eac4a34618e7dc41c9b6d306887ba2b4460a0
+  data.tar.gz: 678f49766262a89976638afae0a47c91e1361b2aeaf80a82a5e0f129019e9f6f76c80b6f8f378774adbc65780eaa8b7f6249132a5a4e733d329c6b1835534757

data/.travis.yml

@@ -2,7 +2,6 @@ language: ruby
 rvm:
   - 2.2
 cache: bundler
-sudo: false
 
 services:
   - redis-server

data/README.md

@@ -1,14 +1,16 @@
 # Sidekiq::Grouping
 
-Lets you batch similar tasks to run them all as a single task.
+<a href="https://evilmartians.com/?utm_source=sidekiq-grouping-gem">
+<img src="https://evilmartians.com/badges/sponsored-by-evil-martians.svg" alt="Sponsored by Evil Martians" width="236" height="54">
+</a>
 
-Allows identical sidekiq jobs to be processed with a single background call.
+Allows to combine similar sidekiq jobs into groups to process them at once.
 
 Useful for:
 * Grouping asynchronous API index calls into bulks for bulk updating/indexing.
 * Periodical batch updating of recently changing database counters.
 
-Sponsored by [Evil Martians](http://evilmartians.com)
+*NOTE:* As of 1.0 `batch_size` renamed to `batch_flush_size`.
 
 ## Usage
 
@@ -19,9 +21,9 @@ class ElasticBulkIndexWorker
   include Sidekiq::Worker
 
   sidekiq_options(
-    queue: :batched_by_size,
-    batch_size: 30,           # Jobs will be combined to groups of 30 items
-    batch_flush_interval: 60, # Combined jobs will be executed at least every 60 seconds
+    queue: :elasic_bulks,
+    batch_flush_size: 30,     # Jobs will be combined when queue size exceeds 30
+    batch_flush_interval: 60, # Jobs will be combined every 60 seconds
     retry: 5
   )
 
@@ -35,13 +37,15 @@ end
 Perform a jobs:
 
 ```ruby
+# At least 30 times
+
 ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 5, _type: 'user' } })
 ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 6, _type: 'user' } })
 ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 7, _type: 'user' } })
 ...
 ```
 
-This jobs will be grouped into a single job which will be performed with the single argument containing:
+This jobs will be grouped into the single job with the single argument:
 
 ```ruby
 [
@@ -52,7 +56,58 @@ This jobs will be grouped into a single job which will be performed with the sin
 ]
 ```
 
-This will happen for every 30 jobs in a row or every 60 seconds.
+## Control grouping
+
+- If `batch_flush_size` option is set - grouping will be performed when batched queue size exceeds this value or `Sidekiq::Grouping::Config.max_batch_size` (1000 by default).
+- If `batch_flush_interval` option is set - grouping will be performed every given interval.
+- If both are set - grouping will be performed when first condition become true. For example, if `batch_flush_interval` is set to 60 seconds and `batch_flush_size` is set to 5 - group task will be enqueued even if just 3 jobs are in the queue at the end of the minute. In the other hand, if 5 jobs were enqueued during 10 seconds - they will be grouped and enqueued immediately.
+
+## Options
+
+- `batch_unique` prevents enqueue of jobs with identical arguments.
+
+  ```ruby
+  class FooWorker
+    include Sidekiq::Worker
+
+    sidekiq_options batch_flush_interval: 10, batch_unique: true
+
+    def perform(n)
+      puts n
+    end
+  end
+
+  FooWorker.perform_async(1)
+  FooWorker.perform_async(1)
+  FooWorker.perform_async(2)
+  FooWorker.perform_async(2)
+
+  # => [[1], [2]]
+  ```
+
+- `batch_size` is used to control single group size.
+
+  ```ruby
+  class FooWorker
+    include Sidekiq::Worker
+
+    sidekiq_options batch_flush_size: 5, batch_size: 2
+
+    def perform(n)
+      puts n
+    end
+  end
+
+  FooWorker.perform_async(1)
+  FooWorker.perform_async(2)
+  FooWorker.perform_async(3)
+  FooWorker.perform_async(4)
+  FooWorker.perform_async(5)
+
+  # => [[1], [2]]
+  # => [[3], [4]]
+  # => [[5]]
+  ```
 
 ## Web UI
 
@@ -69,13 +124,12 @@ require "sidekiq/grouping/web"
 ```ruby
 Sidekiq::Grouping::Config.poll_interval = 5     # Amount of time between polling batches
 Sidekiq::Grouping::Config.max_batch_size = 5000 # Maximum batch size allowed
-Sidekiq::Grouping::Config.lock_ttl = 1          # Timeout of lock set when batched job enqueues
+Sidekiq::Grouping::Config.lock_ttl = 1          # Batch queue flush lock timeout job enqueues
 ```
 
 ## TODO
 
 1. Add support redis_pool option.
-2. Make able to work together with sidekiq-unique-jobs.
 
 ## Installation
 

data/lib/sidekiq/grouping.rb

@@ -1,18 +1,18 @@
 require 'active_support/core_ext/string'
 require 'active_support/configurable'
 require 'active_support/core_ext/numeric/time'
-
-require 'sidekiq/grouping/config'
-require 'sidekiq/grouping/redis'
-require 'sidekiq/grouping/batch'
-require 'sidekiq/grouping/middleware'
-require 'sidekiq/grouping/logging'
-require 'sidekiq/grouping/actor'
-require 'sidekiq/grouping/supervisor'
 require 'sidekiq/grouping/version'
 
 module Sidekiq
   module Grouping
+    autoload :Config, 'sidekiq/grouping/config'
+    autoload :Redis, 'sidekiq/grouping/redis'
+    autoload :Batch, 'sidekiq/grouping/batch'
+    autoload :Middleware, 'sidekiq/grouping/middleware'
+    autoload :Logging, 'sidekiq/grouping/logging'
+    autoload :Actor, 'sidekiq/grouping/actor'
+    autoload :Supervisor, 'sidekiq/grouping/supervisor'
+
     class << self
       attr_writer :logger
 

data/lib/sidekiq/grouping/actor.rb

@@ -9,6 +9,7 @@ module Sidekiq
       end
 
       private
+      
       def start_polling
         interval = Sidekiq::Grouping::Config.poll_interval
         info "Start polling of queue batches every #{interval} seconds"

data/lib/sidekiq/grouping/batch.rb

@@ -27,25 +27,32 @@ module Sidekiq
 
       def chunk_size
         worker_class_options['batch_size'] ||
-        Sidekiq::Grouping::Config.max_batch_size
+          Sidekiq::Grouping::Config.max_batch_size
+      end
+
+      def pluck_size
+        worker_class_options['batch_flush_size'] ||
+          Sidekiq::Grouping::Config.max_batch_size
       end
 
       def pluck
         if @redis.lock(@name)
-          @redis.pluck(@name, chunk_size).map { |value| JSON.parse(value) }
+          @redis.pluck(@name, pluck_size).map { |value| JSON.parse(value) }
         end
       end
 
       def flush
         chunk = pluck
-        if chunk
-          set_current_time_as_last
+        return unless chunk
+
+        chunk.each_slice(chunk_size) do |subchunk|
           Sidekiq::Client.push(
             'class' => @worker_class,
             'queue' => @queue,
-            'args' => [true, chunk]
+            'args' => [true, subchunk]
           )
         end
+        set_current_time_as_last
       end
 
       def worker_class_constant
@@ -79,9 +86,11 @@ module Sidekiq
       end
 
       private
+
       def could_flush_on_overflow?
-        worker_class_options['batch_size'] &&
-        size >= worker_class_options['batch_size']
+        return true if size >= Sidekiq::Grouping::Config.max_batch_size
+        worker_class_options['batch_flush_size'] &&
+          size >= worker_class_options['batch_flush_size']
       end
 
       def could_flush_on_time?
@@ -94,9 +103,7 @@ module Sidekiq
           set_current_time_as_last
           false
         else
-          if next_time
-            next_time < Time.now
-          end
+          next_time < Time.now if next_time
         end
       end
 
@@ -122,7 +129,6 @@ module Sidekiq
           [klass.classify, queue]
         end
       end
-
     end
   end
 end

data/lib/sidekiq/grouping/config.rb

@@ -3,15 +3,15 @@ module Sidekiq
     module Config
       include ActiveSupport::Configurable
 
-      # Interval batch queue polling
+      # Queue size overflow check polling interval
       config_accessor :poll_interval
       self.config.poll_interval = 3
 
       # Maximum batch size
       config_accessor :max_batch_size
-      self.config.max_batch_size = 500
+      self.config.max_batch_size = 1000
 
-      # Batch queue lock timeout (set during flush)
+      # Batch queue flush lock timeout
       config_accessor :lock_ttl
       self.config.lock_ttl = 1
     end

data/lib/sidekiq/grouping/middleware.rb

@@ -6,7 +6,7 @@ module Sidekiq
         options = worker_class.get_sidekiq_options
 
         batch =
-          options.keys.include?('batch_size') ||
+          options.keys.include?('batch_flush_size') ||
           options.keys.include?('batch_flush_interval')
 
         passthrough =
@@ -25,8 +25,12 @@ module Sidekiq
       end
 
       private
+
       def add_to_batch(worker_class, queue, msg, redis_pool = nil)
-        Sidekiq::Grouping::Batch.new(worker_class.name, queue, redis_pool).add(msg['args'])
+        Sidekiq::Grouping::Batch
+          .new(worker_class.name, queue, redis_pool)
+          .add(msg['args'])
+
         nil
       end
     end

data/lib/sidekiq/grouping/supervisor.rb

@@ -4,10 +4,17 @@ module Sidekiq
       class << self
         include Sidekiq::Grouping::Logging
 
+        if Celluloid::VERSION >= '0.17'
+        def run!
+          info 'Sidekiq::Grouping starts supervision'
+          Sidekiq::Grouping::Actor.supervise as: :sidekiq_grouping
+        end
+        else
         def run!
           info 'Sidekiq::Grouping starts supervision'
           Sidekiq::Grouping::Actor.supervise_as(:sidekiq_grouping)
         end
+        end
       end
     end
   end

data/lib/sidekiq/grouping/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Grouping
-    VERSION = '0.0.6'
+    VERSION = '1.0.1'
   end
 end

data/spec/modules/batch_spec.rb

@@ -65,9 +65,9 @@ describe Sidekiq::Grouping::Batch do
       batch = subject.new(BatchedSizeWorker.name, 'batched_size')
 
       expect(batch.could_flush?).to be_falsy
-      10.times { BatchedSizeWorker.perform_async('bar') }
+      10.times { |n| BatchedSizeWorker.perform_async("bar#{n}") }
       batch.flush
-      expect(BatchedSizeWorker).to have_enqueued_job([["bar"], ["bar"], ["bar"]])
+      expect(BatchedSizeWorker).to have_enqueued_job([["bar0"], ["bar1"]])
       expect(batch.size).to eq(7)
     end
   end

data/spec/spec_helper.rb

@@ -21,6 +21,7 @@ Sidekiq.logger = nil
 
 RSpec::Sidekiq.configure do |config|
   config.clear_all_enqueued_jobs = true
+  config.warn_when_jobs_not_processed_by_sidekiq = false
 end
 
 RSpec.configure do |config|

data/spec/support/test_workers.rb

@@ -8,7 +8,7 @@ end
 class BatchedSizeWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: :batched_size, batch_size: 3
+  sidekiq_options queue: :batched_size, batch_flush_size: 3, batch_size: 2
 
   def perform(foo)
   end
@@ -26,7 +26,9 @@ end
 class BatchedBothWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: :batched_both, batch_flush_interval: 3600, batch_size: 3
+  sidekiq_options(
+    queue: :batched_both, batch_flush_interval: 3600, batch_flush_size: 3
+  )
 
   def perform(foo)
   end
@@ -35,7 +37,9 @@ end
 class BatchedUniqueArgsWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: :batched_unique_args, batch_size: 3, batch_unique: true
+  sidekiq_options(
+    queue: :batched_unique_args, batch_flush_size: 3, batch_unique: true
+  )
 
   def perform(foo)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-grouping
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 1.0.1
 platform: ruby
 authors:
 - Victor Sokolov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-24 00:00:00.000000000 Z
+date: 2015-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -186,7 +186,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Allows identical sidekiq jobs to be processed with a single background call