sidekiq-batching 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 680d50e306eddc51995164a7d565e6cdc777ceea
4
+ data.tar.gz: 73340f1b7215bff463cfa8f09e05fdf9de82474a
5
+ SHA512:
6
+ metadata.gz: 60e20f03424b583ce84ce28bc5dce944cbc94495ca8ca6d6840ccfbbb2ee1ed58ad08ed4cd03e2c2f8a22d9a89ead2a4474e633744ac5a2a61d16f701971ebde
7
+ data.tar.gz: 8e0038fc5d3d8ce9d512a915f71ad261aedfad579e083062597db428a947eec157e66b1250fc6644534bd387707e5e2d7a53f64d43bbb306e2a07548b254ec99
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in sidekiq-batching.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Victor Sokolov
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,92 @@
1
+ # Sidekiq::Batching
2
+
3
+ Allows identical sidekiq jobs to be processed with a single background call.
4
+
5
+ Useful for:
6
+ * Grouping asynchronous API index calls into bulks for bulk updating/indexing.
7
+ * Periodical batch updating of recently changing database counters.
8
+
9
+ ## Usage
10
+
11
+ Create a worker:
12
+
13
+ ```ruby
14
+ class ElasticBulkIndexWorker
15
+ include Sidekiq::Worker
16
+
17
+ sidekiq_options(
18
+ queue: :batched_by_size,
19
+ batch_size: 30,
20
+ batch_flush_interval: 30,
21
+ retry: 5
22
+ )
23
+
24
+ def perform(group)
25
+ client = Elasticsearch::Client.new
26
+ client.bulk(body: group.flatten)
27
+ end
28
+ end
29
+ ```
30
+
31
+ Perform a jobs:
32
+
33
+ ```ruby
34
+ ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 5, _type: 'user' } })
35
+ ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 6, _type: 'user' } })
36
+ ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 7, _type: 'user' } })
37
+ ...
38
+ ```
39
+
40
+ This jobs will be grouped into a single job which will be performed with the single argument containing:
41
+
42
+ ```ruby
43
+ [
44
+ [{ delete: { _index: 'test', _id: 5, _type: 'user' } }],
45
+ [{ delete: { _index: 'test', _id: 6, _type: 'user' } }],
46
+ [{ delete: { _index: 'test', _id: 7, _type: 'user' } }]
47
+ ...
48
+ ]
49
+ ```
50
+
51
+ This will happen for every 30 jobs in a row or every 30 seconds.
52
+
53
+ Add this line to your `config/routes.rb` to activate web UI:
54
+
55
+ ```ruby
56
+ require "sidekiq/batching/web"
57
+ ```
58
+
59
+ ## Configuration
60
+
61
+ ```ruby
62
+ Sidekiq::Batching::Config.poll_interval = 5 # Amount of time between polling batches
63
+ Sidekiq::Batching::Config.max_batch_size = 5000 # Maximum batch size allowed
64
+ Sidekiq::Batching::Config.lock_ttl = 1 # Timeout of lock set when batched job enqueues
65
+ ```
66
+
67
+ ## Notes
68
+
69
+ 1. Did not tested with sidekiq 3.
70
+ 1. Does not support sidekiq 3 redis_pool option.
71
+
72
+ ## Installation
73
+
74
+ Add this line to your application's Gemfile:
75
+
76
+ gem 'sidekiq-batching'
77
+
78
+ And then execute:
79
+
80
+ $ bundle
81
+
82
+ Or install it yourself as:
83
+
84
+ $ gem install sidekiq-batching
85
+
86
+ ## Contributing
87
+
88
+ 1. Fork it ( http://github.com/gzigzigzeo/sidekiq-batching/fork )
89
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
90
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
91
+ 4. Push to the branch (`git push origin my-new-feature`)
92
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,40 @@
1
+ require 'active_support/core_ext/string'
2
+ require 'active_support/configurable'
3
+ require 'active_support/core_ext/numeric/time'
4
+
5
+ require 'sidekiq/batching/config'
6
+ require 'sidekiq/batching/redis'
7
+ require 'sidekiq/batching/batch'
8
+ require 'sidekiq/batching/middleware'
9
+ require 'sidekiq/batching/logging'
10
+ require 'sidekiq/batching/actor'
11
+ require 'sidekiq/batching/supervisor'
12
+ require 'sidekiq/batching/version'
13
+
14
+ module Sidekiq
15
+ module Batching
16
+ class << self
17
+ attr_writer :logger
18
+
19
+ def logger
20
+ @logger ||= Sidekiq.logger
21
+ end
22
+ end
23
+ end
24
+ end
25
+
26
+ Sidekiq.configure_client do |config|
27
+ config.client_middleware do |chain|
28
+ chain.add Sidekiq::Batching::Middleware
29
+ end
30
+ end
31
+
32
+ Sidekiq.configure_server do |config|
33
+ config.client_middleware do |chain|
34
+ chain.add Sidekiq::Batching::Middleware
35
+ end
36
+ end
37
+
38
+ if Sidekiq.server?
39
+ Sidekiq::Batching::Supervisor.run!
40
+ end
@@ -0,0 +1,47 @@
1
+ module Sidekiq
2
+ module Batching
3
+ class Actor
4
+ include Sidekiq::Batching::Logging
5
+ include Celluloid
6
+
7
+ def initialize
8
+ link_to_sidekiq_manager
9
+ start_polling
10
+ end
11
+
12
+ private
13
+ def start_polling
14
+ interval = Sidekiq::Batching::Config.poll_interval
15
+ info "Start polling of queue batches every #{interval} seconds"
16
+ every(interval) { flush_batches }
17
+ end
18
+
19
+ def flush_batches
20
+ batches = []
21
+
22
+ Sidekiq::Batching::Batch.all.map do |batch|
23
+ if batch.could_flush?
24
+ batches << batch
25
+ end
26
+ end
27
+
28
+ flush(batches)
29
+ end
30
+
31
+ def link_to_sidekiq_manager
32
+ Sidekiq::CLI.instance.launcher.manager.link(current_actor)
33
+ rescue NoMethodError
34
+ debug "Can't link #{self.class.name}. Sidekiq::Manager not running. Retrying in 5 seconds ..."
35
+ after(5) { link_to_sidekiq_manager }
36
+ end
37
+
38
+ def flush(batches)
39
+ if batches.any?
40
+ names = batches.map { |batch| "#{batch.worker_class} in #{batch.queue}" }
41
+ info "Trying to flush batched queues: #{names.join(',')}"
42
+ batches.each { |batch| batch.flush }
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,117 @@
1
+ module Sidekiq
2
+ module Batching
3
+ class Batch
4
+
5
+ def initialize(worker_class, queue, redis_pool = nil)
6
+ @worker_class = worker_class
7
+ @queue = queue
8
+ @name = "#{worker_class.underscore}:#{queue}"
9
+ @redis = Sidekiq::Batching::Redis.new
10
+ end
11
+
12
+ attr_reader :name, :worker_class, :queue
13
+
14
+ def add(msg)
15
+ @redis.push_msg(@name, msg.to_json)
16
+ end
17
+
18
+ def size
19
+ @redis.batch_size(@name)
20
+ end
21
+
22
+ def chunk_size
23
+ worker_class_options['batch_size'] ||
24
+ Sidekiq::Batching::Config.max_batch_size
25
+ end
26
+
27
+ def pluck
28
+ if @redis.lock(@name)
29
+ @redis.pluck(@name, chunk_size).map { |value| JSON.parse(value) }
30
+ end
31
+ end
32
+
33
+ def flush
34
+ chunk = pluck
35
+ if chunk
36
+ Sidekiq::Client.push(
37
+ 'class' => @worker_class,
38
+ 'queue' => @queue,
39
+ 'args' => [true, chunk]
40
+ )
41
+ end
42
+ end
43
+
44
+ def worker_class_constant
45
+ @worker_class.constantize
46
+ end
47
+
48
+ def worker_class_options
49
+ worker_class_constant.get_sidekiq_options
50
+ rescue NameError
51
+ {}
52
+ end
53
+
54
+ def could_flush?
55
+ could_flush_on_overflow? || could_flush_on_time?
56
+ end
57
+
58
+ def last_execution_time
59
+ last_time = @redis.get_last_execution_time(@name)
60
+ Time.parse(last_time) if last_time
61
+ end
62
+
63
+ def next_execution_time
64
+ if interval = worker_class_options['batch_flush_interval']
65
+ last_time = last_execution_time
66
+ last_time + interval.seconds if last_time
67
+ end
68
+ end
69
+
70
+ def delete
71
+ @redis.delete(@name)
72
+ end
73
+
74
+ private
75
+ def could_flush_on_overflow?
76
+ worker_class_options['batch_size'] &&
77
+ size >= worker_class_options['batch_size']
78
+ end
79
+
80
+ def could_flush_on_time?
81
+ return false if size.zero?
82
+
83
+ last_time = last_execution_time
84
+ next_time = next_execution_time
85
+
86
+ if last_time.blank?
87
+ set_current_time_as_last
88
+ false
89
+ else
90
+ if next_time
91
+ next_time < Time.now
92
+ end
93
+ end
94
+ end
95
+
96
+ def set_current_time_as_last
97
+ @redis.set_last_execution_time(@name, Time.now)
98
+ end
99
+
100
+ class << self
101
+ def all
102
+ redis = Sidekiq::Batching::Redis.new
103
+
104
+ redis.batches.map do |name|
105
+ new(*extract_worker_klass_and_queue(name))
106
+ end
107
+ end
108
+
109
+ def extract_worker_klass_and_queue(name)
110
+ klass, queue = name.split(':')
111
+ [klass.classify, queue]
112
+ end
113
+ end
114
+
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,19 @@
1
+ module Sidekiq
2
+ module Batching
3
+ module Config
4
+ include ActiveSupport::Configurable
5
+
6
+ # Interval batch queue polling
7
+ config_accessor :poll_interval
8
+ self.config.poll_interval = 3
9
+
10
+ # Maximum batch size
11
+ config_accessor :max_batch_size
12
+ self.config.max_batch_size = 500
13
+
14
+ # Batch queue lock timeout (set during flush)
15
+ config_accessor :lock_ttl
16
+ self.config.lock_ttl = 1
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,13 @@
1
+ module Sidekiq
2
+ module Batching
3
+ module Logging
4
+ %w(fatal error warn info debug).each do |level|
5
+ level = level.to_sym
6
+
7
+ define_method(level) do |msg|
8
+ Sidekiq::Batching.logger.public_send(level, "[Sidekiq::Batching] #{msg}")
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,34 @@
1
+ module Sidekiq
2
+ module Batching
3
+ class Middleware
4
+ def call(worker_class, msg, queue, redis_pool = nil)
5
+ worker_class = worker_class.classify.constantize if worker_class.is_a?(String)
6
+ options = worker_class.get_sidekiq_options
7
+
8
+ batch =
9
+ options.keys.include?('batch_size') ||
10
+ options.keys.include?('batch_flush_interval')
11
+
12
+ passthrough =
13
+ msg['args'] &&
14
+ msg['args'].is_a?(Array) &&
15
+ msg['args'].try(:first) == true
16
+
17
+ if batch && not(passthrough)
18
+ add_to_batch(worker_class, queue, msg, redis_pool)
19
+ else
20
+ if batch && passthrough
21
+ msg['args'].shift
22
+ end
23
+ yield
24
+ end
25
+ end
26
+
27
+ private
28
+ def add_to_batch(worker_class, queue, msg, redis_pool = nil)
29
+ Sidekiq::Batching::Batch.new(worker_class.name, queue, redis_pool).add(msg['args'])
30
+ nil
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,67 @@
1
+ module Sidekiq
2
+ module Batching
3
+ class Redis
4
+ def push_msg(name, msg)
5
+ redis do |conn|
6
+ conn.sadd(ns('batches'), name)
7
+ conn.rpush(ns(name), msg)
8
+ end
9
+ end
10
+
11
+ def batch_size(name)
12
+ redis { |conn| conn.llen(ns(name)) }
13
+ end
14
+
15
+ def batches
16
+ redis { |conn| conn.smembers(ns('batches')) }
17
+ end
18
+
19
+ def pluck(name, limit)
20
+ redis do |conn|
21
+ result = conn.pipelined do
22
+ conn.lrange(ns(name), 0, limit - 1)
23
+ conn.ltrim(ns(name), limit, -1)
24
+ end
25
+
26
+ result.first
27
+ end
28
+ end
29
+
30
+ def get_last_execution_time(name)
31
+ redis { |conn| conn.get(ns("last_execution_time:#{name}")) }
32
+ end
33
+
34
+ def set_last_execution_time(name, time)
35
+ redis { |conn| conn.set(ns("last_execution_time:#{name}"), time.to_json) }
36
+ end
37
+
38
+ def lock(name)
39
+ redis do |conn|
40
+ id = ns("lock:#{name}")
41
+ conn.setnx(id, true).tap do |obtained|
42
+ if obtained
43
+ conn.expire(id, Sidekiq::Batching::Config.lock_ttl)
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ def delete(name)
50
+ redis do |conn|
51
+ conn.del(ns("last_execution_time:#{name}"))
52
+ conn.del(ns(name))
53
+ conn.srem(ns('batches'), name)
54
+ end
55
+ end
56
+
57
+ private
58
+ def ns(key = nil)
59
+ "batching:#{key}"
60
+ end
61
+
62
+ def redis(&block)
63
+ Sidekiq.redis(&block)
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,14 @@
1
+ module Sidekiq
2
+ module Batching
3
+ module Supervisor
4
+ class << self
5
+ include Sidekiq::Batching::Logging
6
+
7
+ def run!
8
+ info 'Sidekiq::Batching starts supervision'
9
+ Sidekiq::Batching::Actor.supervise_as(:sidekiq_batching)
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,5 @@
1
+ module Sidekiq
2
+ module Batching
3
+ VERSION = "0.0.1"
4
+ end
5
+ end
@@ -0,0 +1,41 @@
1
+ <header class="row">
2
+ <div class="col-sm-5">
3
+ <h3>Batched jobs</h3>
4
+ </div>
5
+ </header>
6
+
7
+ <div class="container">
8
+ <div class="row">
9
+ <div class="col-sm-12">
10
+ <% if true %>
11
+ <table class="table table-striped table-bordered table-white" style="width: 100%; margin: 0; table-layout:fixed;">
12
+ <thead>
13
+ <th style="width: 50%">Worker</th>
14
+ <th style="width: 30%">Queue</th>
15
+ <th style="width: 10%">Count</th>
16
+ <th style="width: 30%">Last execution time</th>
17
+ <th style="width: 30%">Next enqueue</th>
18
+ <th style="width: 10%">Actions</th>
19
+ </thead>
20
+ <% @batches.each do |batch| %>
21
+ <tr>
22
+ <td><%= batch.worker_class %></td>
23
+ <td><%= batch.queue %></td>
24
+ <td><%= batch.size %></td>
25
+ <td><%= batch.last_execution_time || "&ndash;"%></td>
26
+ <td><%= batch.next_execution_time || "&ndash;"%></td>
27
+ <td>
28
+ <form action="<%= "#{root_path}batching/#{batch.name}/delete" %>" method="post">
29
+ <input class="btn btn-danger btn-xs" type="submit" name="delete" value="Delete" data-confirm="Are you sure you want to delete this batch?" />
30
+ </form>
31
+ </td>
32
+ </tr>
33
+ <% end %>
34
+ </table>
35
+ <% else %>
36
+ <div class="alert alert-success">No recurring jobs found.</div>
37
+ <% end %>
38
+ </div>
39
+ </div>
40
+ </div>
41
+
@@ -0,0 +1,28 @@
1
+ require 'sidekiq/web'
2
+
3
+ module Sidetiq
4
+ module Batching
5
+ module Web
6
+ VIEWS = File.expand_path('views', File.dirname(__FILE__))
7
+
8
+ def self.registered(app)
9
+ app.get "/batching" do
10
+ @batches = Sidekiq::Batching::Batch.all
11
+ erb File.read(File.join(VIEWS, 'index.erb')), locals: {view_path: VIEWS}
12
+ end
13
+
14
+ app.post "/batching/:name/delete" do
15
+ worker_class, queue = Sidekiq::Batching::Batch.extract_worker_klass_and_queue(params['name'])
16
+ batch = Sidekiq::Batching::Batch.new(worker_class, queue)
17
+ batch.delete
18
+ redirect "#{root_path}/batching"
19
+ end
20
+ end
21
+
22
+ end
23
+ end
24
+ end
25
+
26
+ Sidekiq::Web.register(Sidetiq::Batching::Web)
27
+ Sidekiq::Web.tabs["Batching"] = "batching"
28
+
@@ -0,0 +1,29 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'sidekiq/batching/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "sidekiq-batching"
8
+ spec.version = Sidekiq::Batching::VERSION
9
+ spec.authors = ["Victor Sokolov"]
10
+ spec.email = ["gzigzigzeo@gmail.com"]
11
+ spec.summary = %q{Allows identical sidekiq jobs to be processed with a single background call}
12
+ spec.homepage = "http://github.com/gzigzigzeo/sidekiq-batching"
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_development_dependency "bundler", "~> 1.5"
21
+ spec.add_development_dependency "rake"
22
+ spec.add_development_dependency "rspec"
23
+ spec.add_development_dependency "simplecov"
24
+ spec.add_development_dependency "rspec-sidekiq"
25
+ spec.add_development_dependency "activesupport"
26
+ spec.add_development_dependency "timecop"
27
+
28
+ spec.add_dependency "sidekiq"
29
+ end
@@ -0,0 +1,86 @@
1
+ require 'spec_helper'
2
+
3
+ describe Sidekiq::Batching::Batch do
4
+ subject { Sidekiq::Batching::Batch }
5
+
6
+ context 'adding' do
7
+ it 'must enqueue unbatched worker' do
8
+ RegularWorker.perform_async('bar')
9
+ expect(RegularWorker).to have_enqueued_job('bar')
10
+ end
11
+
12
+ it 'must not enqueue batched worker' do
13
+ BatchedSizeWorker.perform_async('bar')
14
+ expect_batch(BatchedSizeWorker, 'batched_size')
15
+ end
16
+
17
+ it 'must not enqueue batched worker' do
18
+ BatchedIntervalWorker.perform_async('bar')
19
+ expect_batch(BatchedIntervalWorker, 'batched_interval')
20
+ end
21
+
22
+ it 'must not enqueue batched worker' do
23
+ BatchedBothWorker.perform_async('bar')
24
+ expect_batch(BatchedBothWorker, 'batched_both')
25
+ end
26
+ end
27
+
28
+ context 'checking if should flush' do
29
+ it 'must flush if limit exceeds for limit worker' do
30
+ batch = subject.new(BatchedSizeWorker.name, 'batched_size')
31
+
32
+ expect(batch.could_flush?).to be_false
33
+ BatchedSizeWorker.perform_async('bar')
34
+ expect(batch.could_flush?).to be_false
35
+ 4.times { BatchedSizeWorker.perform_async('bar') }
36
+ expect(batch.could_flush?).to be_true
37
+ end
38
+
39
+ it 'must flush if limit exceeds for both worker' do
40
+ batch = subject.new(BatchedBothWorker.name, 'batched_both')
41
+
42
+ expect(batch.could_flush?).to be_false
43
+ BatchedBothWorker.perform_async('bar')
44
+ expect(batch.could_flush?).to be_false
45
+ 4.times { BatchedBothWorker.perform_async('bar') }
46
+ expect(batch.could_flush?).to be_true
47
+ end
48
+
49
+ it 'must flush if limit okay but time came' do
50
+ batch = subject.new(BatchedIntervalWorker.name, 'batched_interval')
51
+
52
+ expect(batch.could_flush?).to be_false
53
+ BatchedIntervalWorker.perform_async('bar')
54
+ expect(batch.could_flush?).to be_false
55
+ expect(batch.size).to eq(1)
56
+
57
+ Timecop.travel(2.hours.since)
58
+
59
+ expect(batch.could_flush?).to be_true
60
+ end
61
+ end
62
+
63
+ context 'flushing' do
64
+ it 'must put wokrer to queue on flush' do
65
+ batch = subject.new(BatchedSizeWorker.name, 'batched_size')
66
+
67
+ expect(batch.could_flush?).to be_false
68
+ 10.times { BatchedSizeWorker.perform_async('bar') }
69
+ batch.flush
70
+ expect(BatchedSizeWorker).to have_enqueued_job([["bar"], ["bar"], ["bar"]])
71
+ expect(batch.size).to eq(7)
72
+ end
73
+ end
74
+
75
+ private
76
+ def expect_batch(klass, queue)
77
+ expect(klass).to_not have_enqueued_job('bar')
78
+ batch = subject.new(klass.name, queue)
79
+ stats = subject.all
80
+ expect(batch.size).to eq(1)
81
+ expect(stats.size).to eq(1)
82
+ expect(stats.first.worker_class).to eq(klass.name)
83
+ expect(stats.first.queue).to eq(queue)
84
+ expect(batch.pluck).to eq [['bar']]
85
+ end
86
+ end
@@ -0,0 +1,44 @@
1
+ $LOAD_PATH << "." unless $LOAD_PATH.include?(".")
2
+
3
+ require 'rubygems'
4
+ require 'bundler/setup'
5
+ require 'timecop'
6
+ require 'simplecov'
7
+ require 'celluloid/autostart'
8
+ require 'sidekiq'
9
+ require 'rspec-sidekiq'
10
+ require 'support/test_workers'
11
+
12
+ SimpleCov.start do
13
+ add_filter 'spec'
14
+ end
15
+
16
+ require 'sidekiq/batching'
17
+
18
+ Sidekiq::Batching.logger = nil
19
+ Sidekiq.redis = { namespace: ENV['namespace'] }
20
+ Sidekiq.logger = nil
21
+
22
+ RSpec::Sidekiq.configure do |config|
23
+ config.clear_all_enqueued_jobs = true
24
+ end
25
+
26
+ RSpec.configure do |config|
27
+ config.order = :random
28
+ config.treat_symbols_as_metadata_keys_with_true_values = true
29
+ config.run_all_when_everything_filtered = true
30
+ config.filter_run :focus
31
+
32
+ config.before :each do
33
+ Sidekiq.redis do |conn|
34
+ keys = conn.keys '*batching*'
35
+ keys.each { |key| conn.del key }
36
+ end
37
+ end
38
+
39
+ config.after :each do
40
+ Timecop.return
41
+ end
42
+ end
43
+
44
+ $: << File.join(File.dirname(__FILE__), '..', 'lib')
@@ -0,0 +1,33 @@
1
+ class RegularWorker
2
+ include Sidekiq::Worker
3
+
4
+ def perform(foo)
5
+ end
6
+ end
7
+
8
+ class BatchedSizeWorker
9
+ include Sidekiq::Worker
10
+
11
+ sidekiq_options queue: :batched_size, batch_size: 3
12
+
13
+ def perform(foo)
14
+ end
15
+ end
16
+
17
+ class BatchedIntervalWorker
18
+ include Sidekiq::Worker
19
+
20
+ sidekiq_options queue: :batched_interval, batch_flush_interval: 3600
21
+
22
+ def perform(foo)
23
+ end
24
+ end
25
+
26
+ class BatchedBothWorker
27
+ include Sidekiq::Worker
28
+
29
+ sidekiq_options queue: :batched_both, batch_flush_interval: 3600, batch_size: 3
30
+
31
+ def perform(foo)
32
+ end
33
+ end
metadata ADDED
@@ -0,0 +1,179 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sidekiq-batching
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Victor Sokolov
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-04-12 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.5'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.5'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: simplecov
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rspec-sidekiq
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: activesupport
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: timecop
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: sidekiq
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ description:
126
+ email:
127
+ - gzigzigzeo@gmail.com
128
+ executables: []
129
+ extensions: []
130
+ extra_rdoc_files: []
131
+ files:
132
+ - ".gitignore"
133
+ - Gemfile
134
+ - LICENSE.txt
135
+ - README.md
136
+ - Rakefile
137
+ - lib/sidekiq/batching.rb
138
+ - lib/sidekiq/batching/actor.rb
139
+ - lib/sidekiq/batching/batch.rb
140
+ - lib/sidekiq/batching/config.rb
141
+ - lib/sidekiq/batching/logging.rb
142
+ - lib/sidekiq/batching/middleware.rb
143
+ - lib/sidekiq/batching/redis.rb
144
+ - lib/sidekiq/batching/supervisor.rb
145
+ - lib/sidekiq/batching/version.rb
146
+ - lib/sidekiq/batching/views/index.erb
147
+ - lib/sidekiq/batching/web.rb
148
+ - sidekiq-batching.gemspec
149
+ - spec/modules/batch_spec.rb
150
+ - spec/spec_helper.rb
151
+ - spec/support/test_workers.rb
152
+ homepage: http://github.com/gzigzigzeo/sidekiq-batching
153
+ licenses:
154
+ - MIT
155
+ metadata: {}
156
+ post_install_message:
157
+ rdoc_options: []
158
+ require_paths:
159
+ - lib
160
+ required_ruby_version: !ruby/object:Gem::Requirement
161
+ requirements:
162
+ - - ">="
163
+ - !ruby/object:Gem::Version
164
+ version: '0'
165
+ required_rubygems_version: !ruby/object:Gem::Requirement
166
+ requirements:
167
+ - - ">="
168
+ - !ruby/object:Gem::Version
169
+ version: '0'
170
+ requirements: []
171
+ rubyforge_project:
172
+ rubygems_version: 2.2.2
173
+ signing_key:
174
+ specification_version: 4
175
+ summary: Allows identical sidekiq jobs to be processed with a single background call
176
+ test_files:
177
+ - spec/modules/batch_spec.rb
178
+ - spec/spec_helper.rb
179
+ - spec/support/test_workers.rb