sidekiq-merger 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 668c85fb3075147efdfbfacd83b1f6cad5766249
+  data.tar.gz: ef35056d67ce049cf7b0c3c328cd04f91b01a77c
+SHA512:
+  metadata.gz: 031c84fa7cbf1cebe5e21244a1e7bfa85aa3d3c0a496652b825490aa2a24716cbf4bde327d4ef3e19bd099a403ef7104aa7c2d23e6d60a680ad908b7ea875628
+  data.tar.gz: ab858f4ad9cfafe67f7a837f42df0770b329d14b5a756d2d3c0e0109e6bb10aac23694a24522cead863ea09430a8a3ad7a2e195bf23b90ec5fb0c1e0f5daac05

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,90 @@
+AllCops:
+  DisabledByDefault: true
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/SpaceBeforeBlockBraces:
+  Enabled: true
+  EnforcedStyle: 'space'
+
+Style/SpaceInsideBrackets:
+  Enabled: true
+
+Style/SpaceInsideHashLiteralBraces:
+  Enabled: true
+
+Style/SpaceInsideBlockBraces:
+  Enabled: true
+
+Style/SpaceAroundEqualsInParameterDefault:
+  Enabled: true
+
+Style/SpaceBeforeComma:
+  Enabled: false
+
+Style/SpaceAroundOperators:
+  Enabled: true
+
+Style/SpaceAfterComma:
+  Enabled: true
+
+Style/ExtraSpacing:
+  Enabled: true
+  AllowForAlignment: true
+
+Lint/DuplicateMethods:
+  Enabled: true
+
+Metrics/LineLength:
+  Enabled: true
+  Max: 200
+  AllowHeredoc: true
+  AllowURI: true
+  URISchemes: http, https
+
+Metrics/MethodLength:
+  Enabled: true
+  Max: 25
+
+Metrics/ClassLength:
+  Enabled: true
+  Max: 160
+
+Metrics/ModuleLength:
+  Enabled: true
+  Max: 160
+
+Style/ClassAndModuleChildren:
+  EnforcedStyle: compact
+  Exclude:
+    - lib/sidekiq/merger/version.rb
+
+Rails/PluralizationGrammar:
+  Enabled: true
+
+Style/AlignArray:
+  Enabled: true
+
+Style/AlignHash:
+  Enabled: true
+
+Style/BlockEndNewline:
+  Enabled: false
+
+Style/DoubleNegation:
+  Enabled: false
+
+Metrics/AbcSize:
+  Max: 60
+
+Metrics/CyclomaticComplexity:
+  Max: 12
+
+Metrics/PerceivedComplexity:
+  Max: 12

data/.travis.yml

@@ -0,0 +1,10 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.2.6
+  - 2.3.3
+  - 2.4.0
+before_install: gem install bundler -v 1.13.6
+script:
+  - "bundle exec rake spec"
+  - "bundle exec rubocop -D"

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in sidekiq-lazy-batch.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 dtaniwaki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# sidekiq-merger
+
+[![Gem Version][gem-image]][gem-link]
+[![Dependency Status][deps-image]][deps-link]
+[![Build Status][build-image]][build-link]
+[![Coverage Status][cov-image]][cov-link]
+[![Code Climate][gpa-image]][gpa-link]
+
+Merge sidekiq jobs occurring within specific period.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-merger'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-merger
+
+## Usage
+
+Add merger option into your workers.
+
+```
+class YourWorker
+  include Sidekiq::Worker
+
+  sidekiq_options merger: { key: -> (args) { args[0] } }
+
+  def perform(all_args)
+    # Do something
+  end
+end
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## Copyright
+
+Copyright (c) 2017 dtaniwaki. See [LICENSE](LICENSE) for details.
+
+[gem-image]:   https://badge.fury.io/rb/sidekiq-merger.svg
+[gem-link]:    http://badge.fury.io/rb/sidekiq-merger
+[build-image]: https://secure.travis-ci.org/dtaniwaki/sidekiq-merger.svg
+[build-link]:  http://travis-ci.org/dtaniwaki/sidekiq-merger
+[deps-image]:  https://gemnasium.com/dtaniwaki/sidekiq-merger.svg
+[deps-link]:   https://gemnasium.com/dtaniwaki/sidekiq-merger
+[cov-image]:   https://coveralls.io/repos/dtaniwaki/sidekiq-merger/badge.png
+[cov-link]:    https://coveralls.io/r/dtaniwaki/sidekiq-merger
+[gpa-image]:   https://codeclimate.com/github/dtaniwaki/sidekiq-merger.svg
+[gpa-link]:    https://codeclimate.com/github/dtaniwaki/sidekiq-merger
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "sidekiq-merger"
+
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/sidekiq-merger.rb

@@ -0,0 +1,15 @@
+require_relative "sidekiq/merger"
+
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Merger::Middleware
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.client_middleware do |chain|
+    chain.add Sidekiq::Merger::Middleware
+  end
+end
+
+Sidekiq::Merger.start! if Sidekiq.server?

data/lib/sidekiq/merger.rb

@@ -0,0 +1,33 @@
+require "sidekiq"
+require "concurrent"
+require_relative "merger/version"
+require_relative "merger/middleware"
+require_relative "merger/config"
+require_relative "merger/flusher"
+require_relative "merger/logging_observer"
+
+module Sidekiq::Merger
+  class << self
+    attr_accessor :logger
+  end
+
+  self.logger ||= Sidekiq.logger
+
+  def logger
+    self.class.logger
+  end
+
+  def start!
+    interval = Sidekiq::Merger::Config.poll_interval
+    observer = Sidekiq::Merger::LoggingObserver.new(logger)
+    flusher = Sidekiq::Merger::Flusher.new(logger)
+    task = Concurrent::TimerTask.new(
+      execution_interval: interval
+    ) { flusher.flush }
+    task.add_observer(observer)
+    logger.info(
+      "[Sidekiq::Merger] Started polling batches every #{interval} seconds"
+    )
+    task.execute
+  end
+end

data/lib/sidekiq/merger/batch.rb

@@ -0,0 +1,100 @@
+require_relative "redis"
+require "active_support/core_ext/hash/indifferent_access"
+
+class Sidekiq::Merger::Batch
+  class << self
+    def all
+      redis = Sidekiq::Merger::Redis.new
+
+      redis.all.map do |full_batch_key|
+        keys = full_batch_key.split(":")
+        raise "Invalid batch key" if keys.size < 3
+        worker_class = keys[0].camelize.constantize
+        queue = keys[1]
+        batch_key = keys[2]
+        new(worker_class, queue, batch_key, redis: redis)
+      end
+    end
+
+    def initialize_with_args(worker_class, queue, args, options = {})
+      new(worker_class, queue, batch_key(worker_class, args), options)
+    end
+
+    def batch_key(worker_class, args)
+      options = get_options(worker_class)
+      batch_key = options["key"]
+      if batch_key.respond_to?(:call)
+        batch_key.call(args)
+      else
+        batch_key
+      end
+    end
+
+    def get_options(worker_class)
+      (worker_class.get_sidekiq_options["merger"] || {}).with_indifferent_access
+    end
+  end
+
+  attr_reader :worker_class, :queue, :batch_key
+
+  def initialize(worker_class, queue, batch_key, redis: Sidekiq::Merger::Redis.new)
+    @worker_class = worker_class
+    @queue = queue
+    @batch_key = batch_key
+    @redis = redis
+  end
+
+  def add(args, execution_time)
+    @redis.push(full_batch_key, args, execution_time)
+  end
+
+  def delete(args)
+    @redis.delete(full_batch_key, args)
+  end
+
+  def size
+    @redis.batch_size(full_batch_key)
+  end
+
+  def flush
+    msgs = []
+
+    if @redis.lock(full_batch_key, Sidekiq::Merger::Config.lock_ttl)
+      msgs = @redis.pluck(full_batch_key)
+    end
+
+    unless msgs.empty?
+      Sidekiq::Client.push(
+        "class" => worker_class,
+        "queue" => queue,
+        "args" => msgs
+      )
+    end
+  end
+
+  def can_flush?
+    !execution_time.nil? && execution_time < Time.now
+  end
+
+  def full_batch_key
+    @full_batch_key ||= [worker_class.name.to_s.underscore, queue, batch_key].join(":")
+  end
+
+  def execution_time
+    @execution_time ||= @redis.execution_time(full_batch_key)
+  end
+
+  def ==(other)
+    self.worker_class == other.worker_class &&
+    self.queue == other.queue &&
+    self.batch_key == other.batch_key
+  end
+
+  private
+
+  def options
+    @options ||= self.class.get_options(worker_class)
+  rescue NameError
+    {}
+  end
+end

data/lib/sidekiq/merger/config.rb

@@ -0,0 +1,17 @@
+require "active_support/configurable"
+
+module Sidekiq::Merger::Config
+  include ActiveSupport::Configurable
+
+  def self.options
+    Sidekiq.options["merger"] || {}
+  end
+
+  config_accessor :poll_interval do
+    options[:poll_interval] || 5
+  end
+
+  config_accessor :lock_ttl do
+    options[:lock_ttl] || 1
+  end
+end

data/lib/sidekiq/merger/flusher.rb

@@ -0,0 +1,15 @@
+class Sidekiq::Merger::Flusher
+  def initialize(logger)
+    @logger = logger
+  end
+
+  def flush
+    batches = Sidekiq::Merger::Batch.all.select(&:can_flush?)
+    unless batches.empty?
+      @logger.info(
+        "[Sidekiq::Merger] Trying to flush batched queues: #{batches.map(&:full_batch_key).join(",")}"
+      )
+      batches.each(&:flush)
+    end
+  end
+end

data/lib/sidekiq/merger/logging_observer.rb

@@ -0,0 +1,17 @@
+class Sidekiq::Merger::LoggingObserver
+  def initialize(logger)
+    @logger = logger
+  end
+
+  def update(time, _result, ex)
+    if ex.is_a?(Concurrent::TimeoutError)
+      @logger.error(
+        "[Sidekiq::Grouping] (#{time}) Execution timed out\n"
+      )
+    elsif ex.present?
+      @logger.error(
+        "[Sidekiq::Grouping] Execution failed with error #{ex}\n"
+      )
+    end
+  end
+end

data/lib/sidekiq/merger/middleware.rb

@@ -0,0 +1,18 @@
+require_relative "batch"
+
+class Sidekiq::Merger::Middleware
+  def call(worker_class, msg, queue, _redis_pool = nil)
+    return yield if defined?(Sidekiq::Testing) && Sidekiq::Testing.inline?
+
+    worker_class = worker_class.camelize.constantize if worker_class.is_a?(String)
+    options = worker_class.get_sidekiq_options
+
+    if !msg["at"].nil? && options.key?("merger")
+      Sidekiq::Merger::Batch
+        .initialize_with_args(worker_class, queue, msg["args"])
+        .add(msg["args"], msg["at"])
+    else
+      yield
+    end
+  end
+end

data/lib/sidekiq/merger/redis.rb

@@ -0,0 +1,106 @@
+require "active_support/core_ext/module/delegation"
+
+class Sidekiq::Merger::Redis
+  class << self
+    KEY_PREFIX = "sidekiq-merger".freeze
+
+    def purge!
+      redis do |conn|
+        conn.eval
+        script = <<-SCRIPT
+          for i=1, #ARGV do
+            redis.call('del', unpack(redis.call('keys', ARGV[i])))
+          end
+          return true
+        SCRIPT
+        conn.eval(script, [], [batches_key, msg_key("*"), lock_key("*")])
+      end
+    end
+
+    def batches_key
+      "#{KEY_PREFIX}:batches"
+    end
+
+    def msg_key(key)
+      "#{KEY_PREFIX}:msg:#{key}"
+    end
+
+    def time_key(key)
+      "#{KEY_PREFIX}:time:#{key}"
+    end
+
+    def lock_key(key)
+      "#{KEY_PREFIX}:lock:#{key}"
+    end
+
+    def redis(&block)
+      Sidekiq.redis(&block)
+    end
+  end
+
+  def push(key, msg, execution_time)
+    redis do |conn|
+      conn.multi do
+        conn.sadd(batches_key, key)
+        conn.setnx(time_key(key), execution_time.to_json)
+        conn.sadd(msg_key(key), msg.to_json)
+      end
+    end
+  end
+
+  def delete(key, msg)
+    redis { |conn| conn.srem(msg_key(key), msg.to_json) }
+  end
+
+  def execution_time(key)
+    redis { |conn| Time.parse(conn.get(time_key(key))) rescue nil }
+  end
+
+  def batch_size(key)
+    redis { |conn| conn.scard(msg_key(key)) }
+  end
+
+  def exists?(key, msg)
+    redis { |conn| conn.sismember(msg_key(key), msg.to_json) }
+  end
+
+  def all
+    redis { |conn| conn.smembers(batches_key) }
+  end
+
+  def lock(key, ttl)
+    redis { |conn| conn.set(lock_key(key), true, nx: true, ex: ttl) }
+  end
+
+  def get(key)
+    msgs = []
+    redis do |conn|
+      msgs = conn.smembers(msg_key(key))
+    end
+    msgs.map { |msg| JSON.parse(msg) }
+  end
+
+  def pluck(key)
+    msgs = []
+    redis do |conn|
+      msgs = conn.smembers(msg_key(key))
+      conn.del(msg_key(key))
+      conn.del(time_key(key))
+      conn.srem(batches_key, key)
+    end
+    msgs.map { |msg| JSON.parse(msg) }
+  end
+
+  def delete_all(key)
+    redis do |conn|
+      conn.del(msg_key(key))
+      conn.del(time_key(key))
+      conn.del(lock_key(key))
+      conn.srem(batches_key, key)
+    end
+  end
+
+  private
+
+  delegate :batches_key, :msg_key, :time_key, :lock_key, :redis, to: "self.class"
+end

data/lib/sidekiq/merger/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Merger
+    VERSION = "0.0.1".freeze
+  end
+end

data/sidekiq-merger.gemspec

@@ -0,0 +1,39 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "sidekiq/merger/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidekiq-merger"
+  spec.version       = Sidekiq::Merger::VERSION
+  spec.platform      = Gem::Platform::RUBY
+  spec.authors       = ["dtaniwaki"]
+  spec.email         = ["daisuketaniwaki@gmail.com"]
+
+  spec.summary       = "Sidekiq merger plugin"
+  spec.description   = "Merge sidekiq jobs."
+  spec.homepage      = "https://github.com/dtaniwaki/sidekiq-merger"
+  spec.license = "MIT"
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  spec.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = [">= 2.2.2", "< 2.5"]
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "timecop"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "coveralls"
+
+  spec.add_dependency "sidekiq", ">= 3.4.0"
+  spec.add_dependency "concurrent-ruby"
+  spec.add_dependency "activesupport", ">= 3.2.0"
+end

data/spec/sidekiq/merger/batch_spec.rb

@@ -0,0 +1,124 @@
+require "spec_helper"
+
+describe Sidekiq::Merger::Batch do
+  subject { described_class.new(worker_class, queue, "foo", redis: redis) }
+  let(:redis) { Sidekiq::Merger::Redis.new }
+  let(:queue) { "queue" }
+  let(:now) { Time.now }
+  let(:execution_time) { now + 10.seconds }
+  let(:options) { { key: -> (args) { args.to_json } } }
+  let(:worker_class) do
+    local_options = options
+    Class.new do
+      include Sidekiq::Worker
+
+      sidekiq_options merger: local_options
+
+      def self.name
+        "name"
+      end
+
+      def perform(args)
+      end
+    end
+  end
+  before { Timecop.freeze(now) }
+
+  describe ".all" do
+    it "returns all the keys" do
+      redis.redis do |conn|
+        conn.sadd("sidekiq-merger:batches", "string:foo:xxx")
+        conn.sadd("sidekiq-merger:batches", "numeric:bar:yyy")
+      end
+
+      expect(described_class.all).to contain_exactly(
+        described_class.new(String, "foo", "xxx"),
+        described_class.new(Numeric, "bar", "yyy")
+      )
+    end
+
+    context "including invalid key" do
+      it "raises an error" do
+        redis.redis do |conn|
+          conn.sadd("sidekiq-merger:batches", "string:foo:xxx")
+          conn.sadd("sidekiq-merger:batches", "invalid")
+        end
+        expect {
+          described_class.all
+        }.to raise_error RuntimeError, "Invalid batch key"
+      end
+    end
+  end
+
+  describe ".initialize_with_args" do
+    it "provides batch_key from args" do
+      expect(described_class).to receive(:new).with(worker_class, queue, "[1,2,3]", anything)
+      described_class.initialize_with_args(worker_class, queue, [1, 2, 3])
+    end
+    it "passes options" do
+      expect(described_class).to receive(:new).with(worker_class, queue, anything, { redis: 1 })
+      described_class.initialize_with_args(worker_class, queue, anything, redis: 1)
+    end
+  end
+
+  describe "#add" do
+    it "adds the args in lazy batch" do
+      expect(redis).to receive(:push).with("name:queue:foo", [1, 2, 3], execution_time)
+      subject.add([1, 2, 3], execution_time)
+    end
+  end
+
+  describe "#delete" do
+    it "adds the args in lazy batch" do
+      expect(redis).to receive(:delete).with("name:queue:foo", [1, 2, 3])
+      subject.delete([1, 2, 3])
+    end
+  end
+
+  describe "#size" do
+  end
+
+  describe "#flush" do
+    before do
+      subject.add([1, 2, 3], execution_time)
+      subject.add([2, 3, 4], execution_time)
+    end
+    it "flushes all the args" do
+      expect(Sidekiq::Client).to receive(:push).with(
+        "class" => worker_class,
+        "queue" => queue,
+        "args" => [[1, 2, 3], [2, 3, 4]]
+      )
+
+      subject.flush
+    end
+  end
+
+  describe "#can_flush?" do
+    let(:options) { { flush_interval: 10.seconds } }
+    context "it has not get anything in batch" do
+      it "returns false" do
+        expect(subject.can_flush?).to eq false
+      end
+    end
+    context "it has not passed the interval time" do
+      it "returns false" do
+        subject.add([], execution_time)
+        expect(subject.can_flush?).to eq false
+      end
+    end
+    context "it has passed the interval time" do
+      it "returns true" do
+        subject.add([], execution_time)
+        Timecop.travel(10.seconds)
+        expect(subject.can_flush?).to eq true
+      end
+    end
+  end
+
+  describe "#full_batch_key" do
+    it "returns full batch key" do
+      expect(subject.full_batch_key).to eq "name:queue:foo"
+    end
+  end
+end

data/spec/sidekiq/merger/flusher_spec.rb

@@ -0,0 +1,18 @@
+require "spec_helper"
+
+describe Sidekiq::Merger::Flusher do
+  subject { described_class.new(Sidekiq.logger) }
+
+  describe "#call" do
+    let(:active_batch) { double(full_batch_key: "active", can_flush?: true, flush: nil) }
+    let(:inactive_batch) { double(full_batch_key: "inactive", can_flush?: false, flush: nil) }
+    let(:batches) { [active_batch, inactive_batch] }
+    it "adds the args to the batch" do
+      allow(Sidekiq::Merger::Batch).to receive(:all).and_return batches
+      expect(active_batch).to receive(:flush)
+      expect(inactive_batch).not_to receive(:flush)
+
+      subject.flush
+    end
+  end
+end

data/spec/sidekiq/merger/middleware_spec.rb

@@ -0,0 +1,50 @@
+require "spec_helper"
+
+describe Sidekiq::Merger::Middleware do
+  subject { described_class.new }
+  let(:flusher) { Sidekiq::Merger::Flusher.new(Sidekiq.logger) }
+  let(:queue) { "queue" }
+  let(:now) { Time.now }
+  let(:options) { { key: -> (args) { "key" } } }
+  let(:worker_class) do
+    local_options = options
+    Class.new do
+      include Sidekiq::Worker
+
+      sidekiq_options merger: local_options
+
+      def self.name
+        "name"
+      end
+
+      def perform(args)
+      end
+    end
+  end
+  before :example do
+    allow(Object).to receive(:const_get).with("Name").and_return worker_class
+  end
+
+  describe "#call" do
+    it "adds the args to the batch" do
+      subject.call(worker_class, { "args" => [1, 2, 3], "at" => now + 10.seconds }, queue) {}
+      subject.call(worker_class, { "args" => [2, 3, 4], "at" => now + 15.seconds }, queue) {}
+      flusher.flush
+      expect(worker_class.jobs.size).to eq 0
+      Timecop.travel(10.seconds)
+      flusher.flush
+      expect(worker_class.jobs.size).to eq 1
+      job = worker_class.jobs[0]
+      expect(job["queue"]).to eq queue
+      expect(job["args"]).to eq [[1, 2, 3], [2, 3, 4]]
+    end
+    context "without at msg" do
+      it "does not add the args to the batch" do
+        subject.call(worker_class, { "args" => [1, 2, 3] }, queue) {}
+        subject.call(worker_class, { "args" => [2, 3, 4] }, queue) {}
+        flusher.flush
+        expect(worker_class.jobs.size).to eq 0
+      end
+    end
+  end
+end

data/spec/sidekiq/merger/redis_spec.rb

@@ -0,0 +1,140 @@
+require "spec_helper"
+
+describe Sidekiq::Merger::Redis do
+  subject { described_class.new }
+  let(:now) { Time.now }
+  let(:execution_time) { now + 10.seconds }
+  before { Timecop.freeze(now) }
+
+  describe ".purge" do
+    it "cleans up all the keys" do
+      described_class.redis do |conn|
+        conn.sadd("sidekiq-merger:batches", "test")
+        conn.set("sidekiq-merger:msg:foo", "test")
+        conn.set("sidekiq-merger:lock:foo", "test")
+      end
+
+      described_class.purge!
+
+      described_class.redis do |conn|
+        expect(conn.smembers("sidekiq-merger:batches")).to be_empty
+        expect(conn.keys("sidekiq-merger:msg:*")).to be_empty
+        expect(conn.keys("sidekiq-merger:lock:*")).to be_empty
+      end
+    end
+  end
+
+  describe "#push" do
+    it "pushes the args" do
+      subject.push("foo", [1, 2, 3], execution_time)
+      described_class.redis do |conn|
+        expect(conn.smembers("sidekiq-merger:batches")).to contain_exactly "foo"
+        expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly "sidekiq-merger:time:foo"
+        expect(conn.keys("sidekiq-merger:msg:*")).to contain_exactly "sidekiq-merger:msg:foo"
+        expect(conn.smembers("sidekiq-merger:msg:foo")).to contain_exactly "[1,2,3]"
+      end
+    end
+    it "sets the execution time" do
+      subject.push("foo", [1, 2, 3], execution_time)
+      described_class.redis do |conn|
+        expect(conn.get("sidekiq-merger:time:foo")).to eq execution_time.to_json
+      end
+    end
+
+    context "the batch key already exists" do
+      before do
+        subject.push("foo", [1, 2, 3], execution_time)
+      end
+      it "pushes the args" do
+        subject.push("foo", [2, 3, 4], execution_time + 1.hour)
+        described_class.redis do |conn|
+          expect(conn.smembers("sidekiq-merger:batches")).to contain_exactly "foo"
+          expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly "sidekiq-merger:time:foo"
+          expect(conn.keys("sidekiq-merger:msg:*")).to contain_exactly "sidekiq-merger:msg:foo"
+          expect(conn.smembers("sidekiq-merger:msg:foo")).to contain_exactly "[1,2,3]", "[2,3,4]"
+        end
+      end
+      it "does not update the execution time" do
+        subject.push("foo", [2, 3, 4], execution_time + 1.hour)
+        described_class.redis do |conn|
+          expect(conn.get("sidekiq-merger:time:foo")).to eq execution_time.to_json
+        end
+      end
+    end
+
+    context "the args has already been pushed" do
+      before do
+        subject.push("foo", [1, 2, 3], execution_time)
+      end
+      it "does not push the args" do
+        subject.push("foo", [1, 2, 3], execution_time + 1.hour)
+        described_class.redis do |conn|
+          expect(conn.smembers("sidekiq-merger:batches")).to contain_exactly "foo"
+          expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly "sidekiq-merger:time:foo"
+          expect(conn.keys("sidekiq-merger:msg:*")).to contain_exactly "sidekiq-merger:msg:foo"
+          expect(conn.smembers("sidekiq-merger:msg:foo")).to contain_exactly "[1,2,3]"
+        end
+      end
+      it "does not update the execution time" do
+        subject.push("foo", [1, 2, 3], execution_time + 1.hour)
+        described_class.redis do |conn|
+          expect(conn.get("sidekiq-merger:time:foo")).to eq execution_time.to_json
+        end
+      end
+    end
+
+    context "other batch key already exists" do
+      before do
+        subject.push("foo", [1, 2, 3], execution_time)
+      end
+      it "does not interfere the other batch" do
+        subject.push("bar", [2, 3, 4], execution_time + 1.hour)
+        described_class.redis do |conn|
+          expect(conn.smembers("sidekiq-merger:batches")).to contain_exactly "foo", "bar"
+          expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly "sidekiq-merger:time:foo", "sidekiq-merger:time:bar"
+          expect(conn.keys("sidekiq-merger:msg:*")).to contain_exactly "sidekiq-merger:msg:foo", "sidekiq-merger:msg:bar"
+          expect(conn.smembers("sidekiq-merger:msg:foo")).to contain_exactly "[1,2,3]"
+          expect(conn.smembers("sidekiq-merger:msg:bar")).to contain_exactly "[2,3,4]"
+        end
+      end
+      it "sets the execution time" do
+        subject.push("bar", [2, 3, 4], execution_time + 1.hour)
+        described_class.redis do |conn|
+          expect(conn.get("sidekiq-merger:time:foo")).to eq execution_time.to_json
+          expect(conn.get("sidekiq-merger:time:bar")).to eq (execution_time + 1.hour).to_json
+        end
+      end
+    end
+  end
+
+  describe "#delete" do
+  end
+
+  describe "#batch_size" do
+  end
+
+  describe "#exists?" do
+  end
+
+  describe "#all" do
+  end
+
+  describe "#lock" do
+  end
+
+  describe "#get" do
+  end
+
+  describe "#pluck" do
+    before do
+      subject.push("bar", [1, 2, 3], execution_time)
+      subject.push("bar", [2, 3, 4], execution_time)
+    end
+    it "plucks all the args" do
+      expect(subject.pluck("bar")).to eq [[1, 2, 3], [2, 3, 4]]
+    end
+  end
+
+  describe "#delete_all" do
+  end
+end

data/spec/sidekiq/merger_spec.rb

@@ -0,0 +1,7 @@
+require "spec_helper"
+
+describe Sidekiq::Merger do
+  it "has a version number" do
+    expect(Sidekiq::Merger::VERSION).not_to be nil
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,66 @@
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+require "rubygems"
+require "sidekiq/testing"
+require "active_support/core_ext/numeric/time"
+require "timecop"
+require "simplecov"
+require "coveralls"
+
+pid = Process.pid
+SimpleCov.at_exit do
+  SimpleCov.result.format! if Process.pid == pid
+end
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+  SimpleCov::Formatter::HTMLFormatter,
+  Coveralls::SimpleCov::Formatter
+]
+SimpleCov.start
+
+require "sidekiq/merger"
+
+Dir[File.join(__dir__, "support", "**", "*.rb")].each { |f| require f }
+
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    # be_bigger_than(2).and_smaller_than(4).description
+    #   # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #   # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  config.order = :random
+
+  Kernel.srand config.seed
+
+  config.before :suite do
+    Sidekiq.logger.level = Logger::ERROR
+  end
+
+  config.before :example do
+    Timecop.return
+    Sidekiq::Merger::Redis.redis do |conn|
+      conn.flushall
+    end
+  end
+end

metadata

@@ -0,0 +1,237 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-merger
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- dtaniwaki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-01-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+description: Merge sidekiq jobs.
+email:
+- daisuketaniwaki@gmail.com
+executables:
+- console
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/sidekiq-merger.rb
+- lib/sidekiq/merger.rb
+- lib/sidekiq/merger/batch.rb
+- lib/sidekiq/merger/config.rb
+- lib/sidekiq/merger/flusher.rb
+- lib/sidekiq/merger/logging_observer.rb
+- lib/sidekiq/merger/middleware.rb
+- lib/sidekiq/merger/redis.rb
+- lib/sidekiq/merger/version.rb
+- sidekiq-merger.gemspec
+- spec/sidekiq/merger/batch_spec.rb
+- spec/sidekiq/merger/flusher_spec.rb
+- spec/sidekiq/merger/middleware_spec.rb
+- spec/sidekiq/merger/redis_spec.rb
+- spec/sidekiq/merger_spec.rb
+- spec/spec_helper.rb
+- spec/support/matchers.rb
+homepage: https://github.com/dtaniwaki/sidekiq-merger
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.2
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '2.5'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Sidekiq merger plugin
+test_files:
+- spec/sidekiq/merger/batch_spec.rb
+- spec/sidekiq/merger/flusher_spec.rb
+- spec/sidekiq/merger/middleware_spec.rb
+- spec/sidekiq/merger/redis_spec.rb
+- spec/sidekiq/merger_spec.rb
+- spec/spec_helper.rb
+- spec/support/matchers.rb