sidekiq-grouping 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ae9e71d0d79bd9af61ed8c49fca7122219849a37
+  data.tar.gz: c5469b43d3780b36fbe343eca132c86d3d200c34
+SHA512:
+  metadata.gz: db9e641eb35867c42e2b17eb073b921c42ad992b4846452aff33afb927ea76df2811f7262f89a578c45dbd5d48a0f0e8a0a30be5ea531a8169f1cd9ea7a2ecb8
+  data.tar.gz: 1fde67a0b91497362100c2608582aaa52e36478a960ec33d9adc99f6552dbe4d2966b713ffa8ef87a3c77391f968894c34c9694801a717d6e68b28e03f148ba7

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 2.2
+cache: bundler
+sudo: false
+
+services:
+  - redis-server

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sidekiq-grouping.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Victor Sokolov
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,100 @@
+# Sidekiq::Grouping
+
+Lets you batch similar tasks to run them all as a single task.
+
+Allows identical sidekiq jobs to be processed with a single background call.
+
+Useful for:
+* Grouping asynchronous API index calls into bulks for bulk updating/indexing.
+* Periodical batch updating of recently changing database counters.
+
+Sponsored by [Evil Martians](http://evilmartians.com)
+
+## Usage
+
+Create a worker:
+
+```ruby
+class ElasticBulkIndexWorker
+  include Sidekiq::Worker
+
+  sidekiq_options(
+    queue: :batched_by_size,
+    batch_size: 30,           # Jobs will be combined to groups of 30 items
+    batch_flush_interval: 60, # Combined jobs will be executed at least every 60 seconds
+    retry: 5
+  )
+
+  def perform(group)
+    client = Elasticsearch::Client.new
+    client.bulk(body: group.flatten)
+  end
+end
+```
+
+Perform a jobs:
+
+```ruby
+ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 5, _type: 'user' } })
+ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 6, _type: 'user' } })
+ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 7, _type: 'user' } })
+...
+```
+
+This jobs will be grouped into a single job which will be performed with the single argument containing:
+
+```ruby
+[
+  [{ delete: { _index: 'test', _id: 5, _type: 'user' } }],
+  [{ delete: { _index: 'test', _id: 6, _type: 'user' } }],
+  [{ delete: { _index: 'test', _id: 7, _type: 'user' } }]
+  ...
+]
+```
+
+This will happen for every 30 jobs in a row or every 60 seconds.
+
+## Web UI
+
+![Web UI](web.png)
+
+Add this line to your `config/routes.rb` to activate web UI:
+
+```ruby
+require "sidekiq/grouping/web"
+```
+
+## Configuration
+
+```ruby
+Sidekiq::Grouping::Config.poll_interval = 5     # Amount of time between polling batches
+Sidekiq::Grouping::Config.max_batch_size = 5000 # Maximum batch size allowed
+Sidekiq::Grouping::Config.lock_ttl = 1          # Timeout of lock set when batched job enqueues
+```
+
+## TODO
+
+1. Add support redis_pool option.
+2. Make able to work together with sidekiq-unique-jobs.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sidekiq-grouping'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-grouping
+
+## Contributing
+
+1. Fork it ( http://github.com/gzigzigzeo/sidekiq-grouping/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,4 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec

data/lib/sidekiq/grouping.rb

@@ -0,0 +1,40 @@
+require 'active_support/core_ext/string'
+require 'active_support/configurable'
+require 'active_support/core_ext/numeric/time'
+
+require 'sidekiq/grouping/config'
+require 'sidekiq/grouping/redis'
+require 'sidekiq/grouping/batch'
+require 'sidekiq/grouping/middleware'
+require 'sidekiq/grouping/logging'
+require 'sidekiq/grouping/actor'
+require 'sidekiq/grouping/supervisor'
+require 'sidekiq/grouping/version'
+
+module Sidekiq
+  module Grouping
+    class << self
+      attr_writer :logger
+
+      def logger
+        @logger ||= Sidekiq.logger
+      end
+    end
+  end
+end
+
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Grouping::Middleware
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Grouping::Middleware
+  end
+end
+
+if Sidekiq.server?
+  Sidekiq::Grouping::Supervisor.run!
+end

data/lib/sidekiq/grouping/actor.rb

@@ -0,0 +1,47 @@
+module Sidekiq
+  module Grouping
+    class Actor
+      include Sidekiq::Grouping::Logging
+      include ::Celluloid
+
+      def initialize
+        link_to_sidekiq_manager
+      end
+
+      private
+      def start_polling
+        interval = Sidekiq::Grouping::Config.poll_interval
+        info "Start polling of queue batches every #{interval} seconds"
+        every(interval) { flush_batches }
+      end
+
+      def flush_batches
+        batches = []
+
+        Sidekiq::Grouping::Batch.all.map do |batch|
+          if batch.could_flush?
+            batches << batch
+          end
+        end
+
+        flush(batches)
+      end
+
+      def link_to_sidekiq_manager
+        Sidekiq::CLI.instance.launcher.manager.link(current_actor)
+        start_polling
+      rescue NoMethodError
+        debug "Can't link #{self.class.name}. Sidekiq::Manager not running. Retrying in 5 seconds ..."
+        after(5) { link_to_sidekiq_manager }
+      end
+
+      def flush(batches)
+        if batches.any?
+          names = batches.map { |batch| "#{batch.worker_class} in #{batch.queue}" }
+          info "Trying to flush batched queues: #{names.join(',')}"
+          batches.each { |batch| batch.flush }
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/grouping/batch.rb

@@ -0,0 +1,128 @@
+module Sidekiq
+  module Grouping
+    class Batch
+
+      def initialize(worker_class, queue, redis_pool = nil)
+        @worker_class = worker_class
+        @queue = queue
+        @name = "#{worker_class.underscore}:#{queue}"
+        @redis = Sidekiq::Grouping::Redis.new
+      end
+
+      attr_reader :name, :worker_class, :queue
+
+      def add(msg)
+        msg = msg.to_json
+        @redis.push_msg(@name, msg, enqueue_similar_once?) if should_add? msg
+      end
+
+      def should_add? msg
+        return true unless enqueue_similar_once?
+        !@redis.enqueued?(@name, msg)
+      end
+
+      def size
+        @redis.batch_size(@name)
+      end
+
+      def chunk_size
+        worker_class_options['batch_size'] ||
+        Sidekiq::Grouping::Config.max_batch_size
+      end
+
+      def pluck
+        if @redis.lock(@name)
+          @redis.pluck(@name, chunk_size).map { |value| JSON.parse(value) }
+        end
+      end
+
+      def flush
+        chunk = pluck
+        if chunk
+          set_current_time_as_last
+          Sidekiq::Client.push(
+            'class' => @worker_class,
+            'queue' => @queue,
+            'args' => [true, chunk]
+          )
+        end
+      end
+
+      def worker_class_constant
+        @worker_class.constantize
+      end
+
+      def worker_class_options
+        worker_class_constant.get_sidekiq_options
+      rescue NameError
+        {}
+      end
+
+      def could_flush?
+        could_flush_on_overflow? || could_flush_on_time?
+      end
+
+      def last_execution_time
+        last_time = @redis.get_last_execution_time(@name)
+        Time.parse(last_time) if last_time
+      end
+
+      def next_execution_time
+        if interval = worker_class_options['batch_flush_interval']
+          last_time = last_execution_time
+          last_time + interval.seconds if last_time
+        end
+      end
+
+      def delete
+        @redis.delete(@name)
+      end
+
+      private
+      def could_flush_on_overflow?
+        worker_class_options['batch_size'] &&
+        size >= worker_class_options['batch_size']
+      end
+
+      def could_flush_on_time?
+        return false if size.zero?
+
+        last_time = last_execution_time
+        next_time = next_execution_time
+
+        if last_time.blank?
+          set_current_time_as_last
+          false
+        else
+          if next_time
+            next_time < Time.now
+          end
+        end
+      end
+
+      def enqueue_similar_once?
+        worker_class_options['batch_unique'] == true
+      end
+
+      def set_current_time_as_last
+        @redis.set_last_execution_time(@name, Time.now)
+      end
+
+      class << self
+        def all
+          redis = Sidekiq::Grouping::Redis.new
+
+          redis.batches.map do |name|
+            new(*extract_worker_klass_and_queue(name))
+          end
+        end
+
+        def extract_worker_klass_and_queue(name)
+          klass, queue = name.split(':')
+          [klass.classify, queue]
+        end
+      end
+
+    end
+  end
+end

data/lib/sidekiq/grouping/config.rb

@@ -0,0 +1,19 @@
+module Sidekiq
+  module Grouping
+    module Config
+      include ActiveSupport::Configurable
+
+      # Interval batch queue polling
+      config_accessor :poll_interval
+      self.config.poll_interval = 3
+
+      # Maximum batch size
+      config_accessor :max_batch_size
+      self.config.max_batch_size = 500
+
+      # Batch queue lock timeout (set during flush)
+      config_accessor :lock_ttl
+      self.config.lock_ttl = 1
+    end
+  end
+end

data/lib/sidekiq/grouping/logging.rb

@@ -0,0 +1,13 @@
+module Sidekiq
+  module Grouping
+    module Logging
+      %w(fatal error warn info debug).each do |level|
+        level = level.to_sym
+
+        define_method(level) do |msg|
+          Sidekiq::Grouping.logger.public_send(level, "[Sidekiq::Grouping] #{msg}")
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/grouping/middleware.rb

@@ -0,0 +1,34 @@
+module Sidekiq
+  module Grouping
+    class Middleware
+      def call(worker_class, msg, queue, redis_pool = nil)
+        worker_class = worker_class.classify.constantize if worker_class.is_a?(String)
+        options = worker_class.get_sidekiq_options
+
+        batch =
+          options.keys.include?('batch_size') ||
+          options.keys.include?('batch_flush_interval')
+
+        passthrough =
+          msg['args'] &&
+          msg['args'].is_a?(Array) &&
+          msg['args'].try(:first) == true
+
+        if batch && not(passthrough)
+          add_to_batch(worker_class, queue, msg, redis_pool)
+        else
+          if batch && passthrough
+            msg['args'].shift
+          end
+          yield
+        end
+      end
+
+      private
+      def add_to_batch(worker_class, queue, msg, redis_pool = nil)
+        Sidekiq::Grouping::Batch.new(worker_class.name, queue, redis_pool).add(msg['args'])
+        nil
+      end
+    end
+  end
+end

data/lib/sidekiq/grouping/redis.rb

@@ -0,0 +1,86 @@
+module Sidekiq
+  module Grouping
+    class Redis
+
+      PLUCK_SCRIPT = <<-SCRIPT
+        local pluck_values = redis.call('lrange', KEYS[1], 0, ARGV[1] - 1)
+        redis.call('ltrim', KEYS[1], ARGV[1], -1)
+        for k, v in pairs(pluck_values) do
+          redis.call('srem', KEYS[2], v)
+        end
+        return pluck_values
+      SCRIPT
+
+      def push_msg(name, msg, remember_unique = false)
+        redis do |conn|
+          conn.multi do
+            conn.sadd(ns('batches'), name)
+            conn.rpush(ns(name), msg)
+            conn.sadd(unique_messages_key(name), msg) if remember_unique
+          end
+        end
+      end
+
+      def enqueued?(name, msg)
+        redis do |conn|
+          conn.sismember(unique_messages_key(name), msg)
+        end
+      end
+
+      def batch_size(name)
+        redis { |conn| conn.llen(ns(name)) }
+      end
+
+      def batches
+        redis { |conn| conn.smembers(ns('batches')) }
+      end
+
+      def pluck(name, limit)
+        keys = [ns(name), unique_messages_key(name)]
+        args = [limit]
+        redis { |conn| conn.eval PLUCK_SCRIPT, keys, args }
+      end
+
+      def get_last_execution_time(name)
+        redis { |conn| conn.get(ns("last_execution_time:#{name}")) }
+      end
+
+      def set_last_execution_time(name, time)
+        redis { |conn| conn.set(ns("last_execution_time:#{name}"), time.to_json) }
+      end
+
+      def lock(name)
+        redis do |conn|
+          id = ns("lock:#{name}")
+          conn.setnx(id, true).tap do |obtained|
+            if obtained
+              conn.expire(id, Sidekiq::Grouping::Config.lock_ttl)
+            end
+          end
+        end
+      end
+
+      def delete(name)
+        redis do |conn|
+          conn.del(ns("last_execution_time:#{name}"))
+          conn.del(ns(name))
+          conn.srem(ns('batches'), name)
+        end
+      end
+
+      private
+
+      def unique_messages_key name
+        ns("#{name}:unique_messages")
+      end
+
+      def ns(key = nil)
+        "batching:#{key}"
+      end
+
+      def redis(&block)
+        Sidekiq.redis(&block)
+      end
+    end
+  end
+end

data/lib/sidekiq/grouping/supervisor.rb

@@ -0,0 +1,14 @@
+module Sidekiq
+  module Grouping
+    module Supervisor
+      class << self
+        include Sidekiq::Grouping::Logging
+
+        def run!
+          info 'Sidekiq::Grouping starts supervision'
+          Sidekiq::Grouping::Actor.supervise_as(:sidekiq_grouping)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/grouping/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Grouping
+    VERSION = '0.0.6'
+  end
+end

data/lib/sidekiq/grouping/views/index.erb

@@ -0,0 +1,41 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3>Grouped jobs</h3>
+  </div>
+</header>
+
+<div class="container">
+  <div class="row">
+    <div class="col-sm-12">
+      <% if true %>
+      <table class="table table-striped table-bordered table-white" style="width: 100%; margin: 0; table-layout:fixed;">
+        <thead>
+          <th style="width: 50%">Worker</th>
+          <th style="width: 30%">Queue</th>
+          <th style="width: 10%">Count</th>
+          <th style="width: 30%">Last execution time</th>
+          <th style="width: 30%">Next enqueue</th>
+          <th style="width: 10%">Actions</th>
+        </thead>
+        <% @batches.each do |batch| %>
+        <tr>
+          <td><%= batch.worker_class %></td>
+          <td><%= batch.queue %></td>
+          <td><%= batch.size %></td>
+          <td><%= batch.last_execution_time || "&ndash;"%></td>
+          <td><%= batch.next_execution_time || "&ndash;"%></td>
+          <td>
+            <form action="<%= "#{root_path}grouping/#{batch.name}/delete" %>" method="post">
+              <input class="btn btn-danger btn-xs" type="submit" name="delete" value="Delete" data-confirm="Are you sure you want to delete this batch?" />
+            </form>
+          </td>
+        </tr>
+        <% end %>
+      </table>
+      <% else %>
+      <div class="alert alert-success">No recurring jobs found.</div>
+      <% end %>
+    </div>
+  </div>
+</div>
+

data/lib/sidekiq/grouping/web.rb

@@ -0,0 +1,28 @@
+require 'sidekiq/web'
+
+module Sidetiq
+  module Grouping
+    module Web
+      VIEWS = File.expand_path('views', File.dirname(__FILE__))
+
+      def self.registered(app)
+        app.get "/grouping" do
+          @batches = Sidekiq::Grouping::Batch.all
+          erb File.read(File.join(VIEWS, 'index.erb')), locals: {view_path: VIEWS}
+        end
+
+        app.post "/grouping/:name/delete" do
+          worker_class, queue = Sidekiq::Grouping::Batch.extract_worker_klass_and_queue(params['name'])
+          batch = Sidekiq::Grouping::Batch.new(worker_class, queue)
+          batch.delete
+          redirect "#{root_path}/grouping"
+        end
+      end
+
+    end
+  end
+end
+
+Sidekiq::Web.register(Sidetiq::Grouping::Web)
+Sidekiq::Web.tabs["Grouping"] = "grouping"
+

data/sidekiq-grouping.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq/grouping/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidekiq-grouping"
+  spec.version       = Sidekiq::Grouping::VERSION
+  spec.authors       = ["Victor Sokolov"]
+  spec.email         = ["gzigzigzeo@gmail.com"]
+  spec.summary       = %q{Allows identical sidekiq jobs to be processed with a single background call}
+  spec.homepage      = "http://github.com/gzigzigzeo/sidekiq-grouping"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "rspec-sidekiq"
+  spec.add_development_dependency "activesupport"
+  spec.add_development_dependency "timecop"
+
+  spec.add_dependency "sidekiq"
+  spec.add_dependency "celluloid"
+end

data/spec/modules/batch_spec.rb

@@ -0,0 +1,136 @@
+require 'spec_helper'
+
+describe Sidekiq::Grouping::Batch do
+  subject { Sidekiq::Grouping::Batch }
+
+  context 'adding' do
+    it 'must enqueue unbatched worker' do
+       RegularWorker.perform_async('bar')
+       expect(RegularWorker).to have_enqueued_job('bar')
+    end
+
+    it 'must not enqueue batched worker' do
+      BatchedSizeWorker.perform_async('bar')
+      expect_batch(BatchedSizeWorker, 'batched_size')
+    end
+
+    it 'must not enqueue batched worker' do
+      BatchedIntervalWorker.perform_async('bar')
+      expect_batch(BatchedIntervalWorker, 'batched_interval')
+    end
+
+    it 'must not enqueue batched worker' do
+      BatchedBothWorker.perform_async('bar')
+      expect_batch(BatchedBothWorker, 'batched_both')
+    end
+  end
+
+  context 'checking if should flush' do
+    it 'must flush if limit exceeds for limit worker' do
+      batch = subject.new(BatchedSizeWorker.name, 'batched_size')
+
+      expect(batch.could_flush?).to be_falsy
+      BatchedSizeWorker.perform_async('bar')
+      expect(batch.could_flush?).to be_falsy
+      4.times { BatchedSizeWorker.perform_async('bar') }
+      expect(batch.could_flush?).to be_truthy
+    end
+
+    it 'must flush if limit exceeds for both worker' do
+      batch = subject.new(BatchedBothWorker.name, 'batched_both')
+
+      expect(batch.could_flush?).to be_falsy
+      BatchedBothWorker.perform_async('bar')
+      expect(batch.could_flush?).to be_falsy
+      4.times { BatchedBothWorker.perform_async('bar') }
+      expect(batch.could_flush?).to be_truthy
+    end
+
+    it 'must flush if limit okay but time came' do
+      batch = subject.new(BatchedIntervalWorker.name, 'batched_interval')
+
+      expect(batch.could_flush?).to be_falsy
+      BatchedIntervalWorker.perform_async('bar')
+      expect(batch.could_flush?).to be_falsy
+      expect(batch.size).to eq(1)
+
+      Timecop.travel(2.hours.since)
+
+      expect(batch.could_flush?).to be_truthy
+    end
+  end
+
+  context 'flushing' do
+    it 'must put wokrer to queue on flush' do
+      batch = subject.new(BatchedSizeWorker.name, 'batched_size')
+
+      expect(batch.could_flush?).to be_falsy
+      10.times { BatchedSizeWorker.perform_async('bar') }
+      batch.flush
+      expect(BatchedSizeWorker).to have_enqueued_job([["bar"], ["bar"], ["bar"]])
+      expect(batch.size).to eq(7)
+    end
+  end
+
+  context 'with similar args' do
+    context 'option batch_unique = true' do
+      it 'enqueues once' do
+        batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
+        3.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+        expect(batch.size).to eq(1)
+      end
+
+      it 'enqueues once each unique set of args' do
+        batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
+        3.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+        6.times { BatchedUniqueArgsWorker.perform_async('baz', 1) }
+        3.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+        2.times { BatchedUniqueArgsWorker.perform_async('baz', 3) }
+        7.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+        expect(batch.size).to eq(3)
+      end
+
+      context 'flushing' do
+
+        it 'works' do
+          batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
+          2.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+          2.times { BatchedUniqueArgsWorker.perform_async('baz', 1) }
+          batch.flush
+          expect(batch.size).to eq(0)
+        end
+
+        it 'allows to enqueue again after flush' do
+          batch = subject.new(BatchedUniqueArgsWorker.name, 'batched_unique_args')
+          2.times { BatchedUniqueArgsWorker.perform_async('bar', 1) }
+          2.times { BatchedUniqueArgsWorker.perform_async('baz', 1) }
+          batch.flush
+          BatchedUniqueArgsWorker.perform_async('bar', 1)
+          BatchedUniqueArgsWorker.perform_async('baz', 1)
+          expect(batch.size).to eq(2)
+        end
+      end
+
+    end
+
+    context 'batch_unique is not specified' do
+      it 'enqueues all' do
+        batch = subject.new(BatchedSizeWorker.name, 'batched_size')
+        3.times { BatchedSizeWorker.perform_async('bar', 1) }
+        expect(batch.size).to eq(3)
+      end
+    end
+  end
+
+  private
+  def expect_batch(klass, queue)
+    expect(klass).to_not have_enqueued_job('bar')
+    batch = subject.new(klass.name, queue)
+    stats = subject.all
+    expect(batch.size).to eq(1)
+    expect(stats.size).to eq(1)
+    expect(stats.first.worker_class).to eq(klass.name)
+    expect(stats.first.queue).to eq(queue)
+    expect(batch.pluck).to eq [['bar']]
+  end
+end

data/spec/modules/redis_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe Sidekiq::Grouping::Redis do
+  subject { Sidekiq::Grouping::Redis.new }
+
+  let(:queue_name)    { "my_queue" }
+  let(:key)           { "batching:#{queue_name}" }
+  let(:unique_key)    { "batching:#{queue_name}:unique_messages" }
+
+  describe "#push_msg" do
+    it "adds message to queue" do
+      subject.push_msg(queue_name, 'My message')
+      expect(redis { |c| c.llen key }).to eq 1
+      expect(redis { |c| c.lrange key, 0, 1 }).to eq ['My message']
+      expect(redis { |c| c.smembers unique_key}).to eq []
+    end
+
+    it "remembers unique message if specified" do
+      subject.push_msg(queue_name, 'My message', true)
+      expect(redis { |c| c.smembers unique_key}).to eq ['My message']
+    end
+  end
+
+  describe "#pluck" do
+    it "removes messages from queue" do
+      subject.push_msg(queue_name, "Message 1")
+      subject.push_msg(queue_name, "Message 2")
+      subject.pluck(queue_name, 2)
+      expect(redis { |c| c.llen key }).to eq 0
+    end
+
+    it "forgets unique messages" do
+      subject.push_msg(queue_name, "Message 1", true)
+      subject.push_msg(queue_name, "Message 2", true)
+      expect(redis { |c| c.scard unique_key }).to eq 2
+      subject.pluck(queue_name, 2)
+      expect(redis { |c| c.smembers unique_key }).to eq []
+    end
+  end
+
+  private
+
+  def redis(&block)
+    Sidekiq.redis(&block)
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,43 @@
+$LOAD_PATH << "." unless $LOAD_PATH.include?(".")
+
+require 'rubygems'
+require 'bundler/setup'
+require 'timecop'
+require 'simplecov'
+require 'celluloid/autostart'
+require 'sidekiq'
+require 'rspec-sidekiq'
+require 'support/test_workers'
+
+SimpleCov.start do
+  add_filter 'spec'
+end
+
+require 'sidekiq/grouping'
+
+Sidekiq::Grouping.logger = nil
+Sidekiq.redis = { namespace: ENV['namespace'] }
+Sidekiq.logger = nil
+
+RSpec::Sidekiq.configure do |config|
+  config.clear_all_enqueued_jobs = true
+end
+
+RSpec.configure do |config|
+  config.order = :random
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  config.before :each do
+    Sidekiq.redis do |conn|
+      keys = conn.keys '*batching*'
+      keys.each { |key| conn.del key }
+    end
+  end
+
+  config.after :each do
+    Timecop.return
+  end
+end
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')

data/spec/support/test_workers.rb

@@ -0,0 +1,42 @@
+class RegularWorker
+  include Sidekiq::Worker
+
+  def perform(foo)
+  end
+end
+
+class BatchedSizeWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :batched_size, batch_size: 3
+
+  def perform(foo)
+  end
+end
+
+class BatchedIntervalWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :batched_interval, batch_flush_interval: 3600
+
+  def perform(foo)
+  end
+end
+
+class BatchedBothWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :batched_both, batch_flush_interval: 3600, batch_size: 3
+
+  def perform(foo)
+  end
+end
+
+class BatchedUniqueArgsWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :batched_unique_args, batch_size: 3, batch_unique: true
+
+  def perform(foo)
+  end
+end

metadata

@@ -0,0 +1,197 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-grouping
+version: !ruby/object:Gem::Version
+  version: 0.0.6
+platform: ruby
+authors:
+- Victor Sokolov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- gzigzigzeo@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sidekiq/grouping.rb
+- lib/sidekiq/grouping/actor.rb
+- lib/sidekiq/grouping/batch.rb
+- lib/sidekiq/grouping/config.rb
+- lib/sidekiq/grouping/logging.rb
+- lib/sidekiq/grouping/middleware.rb
+- lib/sidekiq/grouping/redis.rb
+- lib/sidekiq/grouping/supervisor.rb
+- lib/sidekiq/grouping/version.rb
+- lib/sidekiq/grouping/views/index.erb
+- lib/sidekiq/grouping/web.rb
+- sidekiq-grouping.gemspec
+- spec/modules/batch_spec.rb
+- spec/modules/redis_spec.rb
+- spec/spec_helper.rb
+- spec/support/test_workers.rb
+- web.png
+homepage: http://github.com/gzigzigzeo/sidekiq-grouping
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: Allows identical sidekiq jobs to be processed with a single background call
+test_files:
+- spec/modules/batch_spec.rb
+- spec/modules/redis_spec.rb
+- spec/spec_helper.rb
+- spec/support/test_workers.rb