sidekiq-batching 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 680d50e306eddc51995164a7d565e6cdc777ceea
4
+ data.tar.gz: 73340f1b7215bff463cfa8f09e05fdf9de82474a
5
+ SHA512:
6
+ metadata.gz: 60e20f03424b583ce84ce28bc5dce944cbc94495ca8ca6d6840ccfbbb2ee1ed58ad08ed4cd03e2c2f8a22d9a89ead2a4474e633744ac5a2a61d16f701971ebde
7
+ data.tar.gz: 8e0038fc5d3d8ce9d512a915f71ad261aedfad579e083062597db428a947eec157e66b1250fc6644534bd387707e5e2d7a53f64d43bbb306e2a07548b254ec99
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in sidekiq-batching.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Victor Sokolov
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,92 @@
1
+ # Sidekiq::Batching
2
+
3
+ Allows identical sidekiq jobs to be processed with a single background call.
4
+
5
+ Useful for:
6
+ * Grouping asynchronous API index calls into bulks for bulk updating/indexing.
7
+ * Periodical batch updating of recently changing database counters.
8
+
9
+ ## Usage
10
+
11
+ Create a worker:
12
+
13
+ ```ruby
14
+ class ElasticBulkIndexWorker
15
+ include Sidekiq::Worker
16
+
17
+ sidekiq_options(
18
+ queue: :batched_by_size,
19
+ batch_size: 30,
20
+ batch_flush_interval: 30,
21
+ retry: 5
22
+ )
23
+
24
+ def perform(group)
25
+ client = Elasticsearch::Client.new
26
+ client.bulk(body: group.flatten)
27
+ end
28
+ end
29
+ ```
30
+
31
+ Perform a jobs:
32
+
33
+ ```ruby
34
+ ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 5, _type: 'user' } })
35
+ ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 6, _type: 'user' } })
36
+ ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 7, _type: 'user' } })
37
+ ...
38
+ ```
39
+
40
+ This jobs will be grouped into a single job which will be performed with the single argument containing:
41
+
42
+ ```ruby
43
+ [
44
+ [{ delete: { _index: 'test', _id: 5, _type: 'user' } }],
45
+ [{ delete: { _index: 'test', _id: 6, _type: 'user' } }],
46
+ [{ delete: { _index: 'test', _id: 7, _type: 'user' } }]
47
+ ...
48
+ ]
49
+ ```
50
+
51
+ This will happen for every 30 jobs in a row or every 30 seconds.
52
+
53
+ Add this line to your `config/routes.rb` to activate web UI:
54
+
55
+ ```ruby
56
+ require "sidekiq/batching/web"
57
+ ```
58
+
59
+ ## Configuration
60
+
61
+ ```ruby
62
+ Sidekiq::Batching::Config.poll_interval = 5 # Amount of time between polling batches
63
+ Sidekiq::Batching::Config.max_batch_size = 5000 # Maximum batch size allowed
64
+ Sidekiq::Batching::Config.lock_ttl = 1 # Timeout of lock set when batched job enqueues
65
+ ```
66
+
67
+ ## Notes
68
+
69
+ 1. Did not tested with sidekiq 3.
70
+ 1. Does not support sidekiq 3 redis_pool option.
71
+
72
+ ## Installation
73
+
74
+ Add this line to your application's Gemfile:
75
+
76
+ gem 'sidekiq-batching'
77
+
78
+ And then execute:
79
+
80
+ $ bundle
81
+
82
+ Or install it yourself as:
83
+
84
+ $ gem install sidekiq-batching
85
+
86
+ ## Contributing
87
+
88
+ 1. Fork it ( http://github.com/gzigzigzeo/sidekiq-batching/fork )
89
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
90
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
91
+ 4. Push to the branch (`git push origin my-new-feature`)
92
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,40 @@
1
+ require 'active_support/core_ext/string'
2
+ require 'active_support/configurable'
3
+ require 'active_support/core_ext/numeric/time'
4
+
5
+ require 'sidekiq/batching/config'
6
+ require 'sidekiq/batching/redis'
7
+ require 'sidekiq/batching/batch'
8
+ require 'sidekiq/batching/middleware'
9
+ require 'sidekiq/batching/logging'
10
+ require 'sidekiq/batching/actor'
11
+ require 'sidekiq/batching/supervisor'
12
+ require 'sidekiq/batching/version'
13
+
14
+ module Sidekiq
15
+ module Batching
16
+ class << self
17
+ attr_writer :logger
18
+
19
+ def logger
20
+ @logger ||= Sidekiq.logger
21
+ end
22
+ end
23
+ end
24
+ end
25
+
26
+ Sidekiq.configure_client do |config|
27
+ config.client_middleware do |chain|
28
+ chain.add Sidekiq::Batching::Middleware
29
+ end
30
+ end
31
+
32
+ Sidekiq.configure_server do |config|
33
+ config.client_middleware do |chain|
34
+ chain.add Sidekiq::Batching::Middleware
35
+ end
36
+ end
37
+
38
+ if Sidekiq.server?
39
+ Sidekiq::Batching::Supervisor.run!
40
+ end
@@ -0,0 +1,47 @@
1
+ module Sidekiq
2
+ module Batching
3
+ class Actor
4
+ include Sidekiq::Batching::Logging
5
+ include Celluloid
6
+
7
+ def initialize
8
+ link_to_sidekiq_manager
9
+ start_polling
10
+ end
11
+
12
+ private
13
+ def start_polling
14
+ interval = Sidekiq::Batching::Config.poll_interval
15
+ info "Start polling of queue batches every #{interval} seconds"
16
+ every(interval) { flush_batches }
17
+ end
18
+
19
+ def flush_batches
20
+ batches = []
21
+
22
+ Sidekiq::Batching::Batch.all.map do |batch|
23
+ if batch.could_flush?
24
+ batches << batch
25
+ end
26
+ end
27
+
28
+ flush(batches)
29
+ end
30
+
31
+ def link_to_sidekiq_manager
32
+ Sidekiq::CLI.instance.launcher.manager.link(current_actor)
33
+ rescue NoMethodError
34
+ debug "Can't link #{self.class.name}. Sidekiq::Manager not running. Retrying in 5 seconds ..."
35
+ after(5) { link_to_sidekiq_manager }
36
+ end
37
+
38
+ def flush(batches)
39
+ if batches.any?
40
+ names = batches.map { |batch| "#{batch.worker_class} in #{batch.queue}" }
41
+ info "Trying to flush batched queues: #{names.join(',')}"
42
+ batches.each { |batch| batch.flush }
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,117 @@
1
+ module Sidekiq
2
+ module Batching
3
+ class Batch
4
+
5
+ def initialize(worker_class, queue, redis_pool = nil)
6
+ @worker_class = worker_class
7
+ @queue = queue
8
+ @name = "#{worker_class.underscore}:#{queue}"
9
+ @redis = Sidekiq::Batching::Redis.new
10
+ end
11
+
12
+ attr_reader :name, :worker_class, :queue
13
+
14
+ def add(msg)
15
+ @redis.push_msg(@name, msg.to_json)
16
+ end
17
+
18
+ def size
19
+ @redis.batch_size(@name)
20
+ end
21
+
22
+ def chunk_size
23
+ worker_class_options['batch_size'] ||
24
+ Sidekiq::Batching::Config.max_batch_size
25
+ end
26
+
27
+ def pluck
28
+ if @redis.lock(@name)
29
+ @redis.pluck(@name, chunk_size).map { |value| JSON.parse(value) }
30
+ end
31
+ end
32
+
33
+ def flush
34
+ chunk = pluck
35
+ if chunk
36
+ Sidekiq::Client.push(
37
+ 'class' => @worker_class,
38
+ 'queue' => @queue,
39
+ 'args' => [true, chunk]
40
+ )
41
+ end
42
+ end
43
+
44
+ def worker_class_constant
45
+ @worker_class.constantize
46
+ end
47
+
48
+ def worker_class_options
49
+ worker_class_constant.get_sidekiq_options
50
+ rescue NameError
51
+ {}
52
+ end
53
+
54
+ def could_flush?
55
+ could_flush_on_overflow? || could_flush_on_time?
56
+ end
57
+
58
+ def last_execution_time
59
+ last_time = @redis.get_last_execution_time(@name)
60
+ Time.parse(last_time) if last_time
61
+ end
62
+
63
+ def next_execution_time
64
+ if interval = worker_class_options['batch_flush_interval']
65
+ last_time = last_execution_time
66
+ last_time + interval.seconds if last_time
67
+ end
68
+ end
69
+
70
+ def delete
71
+ @redis.delete(@name)
72
+ end
73
+
74
+ private
75
+ def could_flush_on_overflow?
76
+ worker_class_options['batch_size'] &&
77
+ size >= worker_class_options['batch_size']
78
+ end
79
+
80
+ def could_flush_on_time?
81
+ return false if size.zero?
82
+
83
+ last_time = last_execution_time
84
+ next_time = next_execution_time
85
+
86
+ if last_time.blank?
87
+ set_current_time_as_last
88
+ false
89
+ else
90
+ if next_time
91
+ next_time < Time.now
92
+ end
93
+ end
94
+ end
95
+
96
+ def set_current_time_as_last
97
+ @redis.set_last_execution_time(@name, Time.now)
98
+ end
99
+
100
+ class << self
101
+ def all
102
+ redis = Sidekiq::Batching::Redis.new
103
+
104
+ redis.batches.map do |name|
105
+ new(*extract_worker_klass_and_queue(name))
106
+ end
107
+ end
108
+
109
+ def extract_worker_klass_and_queue(name)
110
+ klass, queue = name.split(':')
111
+ [klass.classify, queue]
112
+ end
113
+ end
114
+
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,19 @@
1
+ module Sidekiq
2
+ module Batching
3
+ module Config
4
+ include ActiveSupport::Configurable
5
+
6
+ # Interval batch queue polling
7
+ config_accessor :poll_interval
8
+ self.config.poll_interval = 3
9
+
10
+ # Maximum batch size
11
+ config_accessor :max_batch_size
12
+ self.config.max_batch_size = 500
13
+
14
+ # Batch queue lock timeout (set during flush)
15
+ config_accessor :lock_ttl
16
+ self.config.lock_ttl = 1
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,13 @@
1
+ module Sidekiq
2
+ module Batching
3
+ module Logging
4
+ %w(fatal error warn info debug).each do |level|
5
+ level = level.to_sym
6
+
7
+ define_method(level) do |msg|
8
+ Sidekiq::Batching.logger.public_send(level, "[Sidekiq::Batching] #{msg}")
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,34 @@
1
+ module Sidekiq
2
+ module Batching
3
+ class Middleware
4
+ def call(worker_class, msg, queue, redis_pool = nil)
5
+ worker_class = worker_class.classify.constantize if worker_class.is_a?(String)
6
+ options = worker_class.get_sidekiq_options
7
+
8
+ batch =
9
+ options.keys.include?('batch_size') ||
10
+ options.keys.include?('batch_flush_interval')
11
+
12
+ passthrough =
13
+ msg['args'] &&
14
+ msg['args'].is_a?(Array) &&
15
+ msg['args'].try(:first) == true
16
+
17
+ if batch && not(passthrough)
18
+ add_to_batch(worker_class, queue, msg, redis_pool)
19
+ else
20
+ if batch && passthrough
21
+ msg['args'].shift
22
+ end
23
+ yield
24
+ end
25
+ end
26
+
27
+ private
28
+ def add_to_batch(worker_class, queue, msg, redis_pool = nil)
29
+ Sidekiq::Batching::Batch.new(worker_class.name, queue, redis_pool).add(msg['args'])
30
+ nil
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,67 @@
1
+ module Sidekiq
2
+ module Batching
3
+ class Redis
4
+ def push_msg(name, msg)
5
+ redis do |conn|
6
+ conn.sadd(ns('batches'), name)
7
+ conn.rpush(ns(name), msg)
8
+ end
9
+ end
10
+
11
+ def batch_size(name)
12
+ redis { |conn| conn.llen(ns(name)) }
13
+ end
14
+
15
+ def batches
16
+ redis { |conn| conn.smembers(ns('batches')) }
17
+ end
18
+
19
+ def pluck(name, limit)
20
+ redis do |conn|
21
+ result = conn.pipelined do
22
+ conn.lrange(ns(name), 0, limit - 1)
23
+ conn.ltrim(ns(name), limit, -1)
24
+ end
25
+
26
+ result.first
27
+ end
28
+ end
29
+
30
+ def get_last_execution_time(name)
31
+ redis { |conn| conn.get(ns("last_execution_time:#{name}")) }
32
+ end
33
+
34
+ def set_last_execution_time(name, time)
35
+ redis { |conn| conn.set(ns("last_execution_time:#{name}"), time.to_json) }
36
+ end
37
+
38
+ def lock(name)
39
+ redis do |conn|
40
+ id = ns("lock:#{name}")
41
+ conn.setnx(id, true).tap do |obtained|
42
+ if obtained
43
+ conn.expire(id, Sidekiq::Batching::Config.lock_ttl)
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ def delete(name)
50
+ redis do |conn|
51
+ conn.del(ns("last_execution_time:#{name}"))
52
+ conn.del(ns(name))
53
+ conn.srem(ns('batches'), name)
54
+ end
55
+ end
56
+
57
+ private
58
+ def ns(key = nil)
59
+ "batching:#{key}"
60
+ end
61
+
62
+ def redis(&block)
63
+ Sidekiq.redis(&block)
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,14 @@
1
+ module Sidekiq
2
+ module Batching
3
+ module Supervisor
4
+ class << self
5
+ include Sidekiq::Batching::Logging
6
+
7
+ def run!
8
+ info 'Sidekiq::Batching starts supervision'
9
+ Sidekiq::Batching::Actor.supervise_as(:sidekiq_batching)
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,5 @@
1
+ module Sidekiq
2
+ module Batching
3
+ VERSION = "0.0.1"
4
+ end
5
+ end
@@ -0,0 +1,41 @@
1
+ <header class="row">
2
+ <div class="col-sm-5">
3
+ <h3>Batched jobs</h3>
4
+ </div>
5
+ </header>
6
+
7
+ <div class="container">
8
+ <div class="row">
9
+ <div class="col-sm-12">
10
+ <% if true %>
11
+ <table class="table table-striped table-bordered table-white" style="width: 100%; margin: 0; table-layout:fixed;">
12
+ <thead>
13
+ <th style="width: 50%">Worker</th>
14
+ <th style="width: 30%">Queue</th>
15
+ <th style="width: 10%">Count</th>
16
+ <th style="width: 30%">Last execution time</th>
17
+ <th style="width: 30%">Next enqueue</th>
18
+ <th style="width: 10%">Actions</th>
19
+ </thead>
20
+ <% @batches.each do |batch| %>
21
+ <tr>
22
+ <td><%= batch.worker_class %></td>
23
+ <td><%= batch.queue %></td>
24
+ <td><%= batch.size %></td>
25
+ <td><%= batch.last_execution_time || "&ndash;"%></td>
26
+ <td><%= batch.next_execution_time || "&ndash;"%></td>
27
+ <td>
28
+ <form action="<%= "#{root_path}batching/#{batch.name}/delete" %>" method="post">
29
+ <input class="btn btn-danger btn-xs" type="submit" name="delete" value="Delete" data-confirm="Are you sure you want to delete this batch?" />
30
+ </form>
31
+ </td>
32
+ </tr>
33
+ <% end %>
34
+ </table>
35
+ <% else %>
36
+ <div class="alert alert-success">No recurring jobs found.</div>
37
+ <% end %>
38
+ </div>
39
+ </div>
40
+ </div>
41
+
@@ -0,0 +1,28 @@
1
+ require 'sidekiq/web'
2
+
3
+ module Sidetiq
4
+ module Batching
5
+ module Web
6
+ VIEWS = File.expand_path('views', File.dirname(__FILE__))
7
+
8
+ def self.registered(app)
9
+ app.get "/batching" do
10
+ @batches = Sidekiq::Batching::Batch.all
11
+ erb File.read(File.join(VIEWS, 'index.erb')), locals: {view_path: VIEWS}
12
+ end
13
+
14
+ app.post "/batching/:name/delete" do
15
+ worker_class, queue = Sidekiq::Batching::Batch.extract_worker_klass_and_queue(params['name'])
16
+ batch = Sidekiq::Batching::Batch.new(worker_class, queue)
17
+ batch.delete
18
+ redirect "#{root_path}/batching"
19
+ end
20
+ end
21
+
22
+ end
23
+ end
24
+ end
25
+
26
+ Sidekiq::Web.register(Sidetiq::Batching::Web)
27
+ Sidekiq::Web.tabs["Batching"] = "batching"
28
+
@@ -0,0 +1,29 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'sidekiq/batching/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "sidekiq-batching"
8
+ spec.version = Sidekiq::Batching::VERSION
9
+ spec.authors = ["Victor Sokolov"]
10
+ spec.email = ["gzigzigzeo@gmail.com"]
11
+ spec.summary = %q{Allows identical sidekiq jobs to be processed with a single background call}
12
+ spec.homepage = "http://github.com/gzigzigzeo/sidekiq-batching"
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_development_dependency "bundler", "~> 1.5"
21
+ spec.add_development_dependency "rake"
22
+ spec.add_development_dependency "rspec"
23
+ spec.add_development_dependency "simplecov"
24
+ spec.add_development_dependency "rspec-sidekiq"
25
+ spec.add_development_dependency "activesupport"
26
+ spec.add_development_dependency "timecop"
27
+
28
+ spec.add_dependency "sidekiq"
29
+ end
@@ -0,0 +1,86 @@
1
+ require 'spec_helper'
2
+
3
+ describe Sidekiq::Batching::Batch do
4
+ subject { Sidekiq::Batching::Batch }
5
+
6
+ context 'adding' do
7
+ it 'must enqueue unbatched worker' do
8
+ RegularWorker.perform_async('bar')
9
+ expect(RegularWorker).to have_enqueued_job('bar')
10
+ end
11
+
12
+ it 'must not enqueue batched worker' do
13
+ BatchedSizeWorker.perform_async('bar')
14
+ expect_batch(BatchedSizeWorker, 'batched_size')
15
+ end
16
+
17
+ it 'must not enqueue batched worker' do
18
+ BatchedIntervalWorker.perform_async('bar')
19
+ expect_batch(BatchedIntervalWorker, 'batched_interval')
20
+ end
21
+
22
+ it 'must not enqueue batched worker' do
23
+ BatchedBothWorker.perform_async('bar')
24
+ expect_batch(BatchedBothWorker, 'batched_both')
25
+ end
26
+ end
27
+
28
+ context 'checking if should flush' do
29
+ it 'must flush if limit exceeds for limit worker' do
30
+ batch = subject.new(BatchedSizeWorker.name, 'batched_size')
31
+
32
+ expect(batch.could_flush?).to be_false
33
+ BatchedSizeWorker.perform_async('bar')
34
+ expect(batch.could_flush?).to be_false
35
+ 4.times { BatchedSizeWorker.perform_async('bar') }
36
+ expect(batch.could_flush?).to be_true
37
+ end
38
+
39
+ it 'must flush if limit exceeds for both worker' do
40
+ batch = subject.new(BatchedBothWorker.name, 'batched_both')
41
+
42
+ expect(batch.could_flush?).to be_false
43
+ BatchedBothWorker.perform_async('bar')
44
+ expect(batch.could_flush?).to be_false
45
+ 4.times { BatchedBothWorker.perform_async('bar') }
46
+ expect(batch.could_flush?).to be_true
47
+ end
48
+
49
+ it 'must flush if limit okay but time came' do
50
+ batch = subject.new(BatchedIntervalWorker.name, 'batched_interval')
51
+
52
+ expect(batch.could_flush?).to be_false
53
+ BatchedIntervalWorker.perform_async('bar')
54
+ expect(batch.could_flush?).to be_false
55
+ expect(batch.size).to eq(1)
56
+
57
+ Timecop.travel(2.hours.since)
58
+
59
+ expect(batch.could_flush?).to be_true
60
+ end
61
+ end
62
+
63
+ context 'flushing' do
64
+ it 'must put wokrer to queue on flush' do
65
+ batch = subject.new(BatchedSizeWorker.name, 'batched_size')
66
+
67
+ expect(batch.could_flush?).to be_false
68
+ 10.times { BatchedSizeWorker.perform_async('bar') }
69
+ batch.flush
70
+ expect(BatchedSizeWorker).to have_enqueued_job([["bar"], ["bar"], ["bar"]])
71
+ expect(batch.size).to eq(7)
72
+ end
73
+ end
74
+
75
+ private
76
+ def expect_batch(klass, queue)
77
+ expect(klass).to_not have_enqueued_job('bar')
78
+ batch = subject.new(klass.name, queue)
79
+ stats = subject.all
80
+ expect(batch.size).to eq(1)
81
+ expect(stats.size).to eq(1)
82
+ expect(stats.first.worker_class).to eq(klass.name)
83
+ expect(stats.first.queue).to eq(queue)
84
+ expect(batch.pluck).to eq [['bar']]
85
+ end
86
+ end
@@ -0,0 +1,44 @@
1
+ $LOAD_PATH << "." unless $LOAD_PATH.include?(".")
2
+
3
+ require 'rubygems'
4
+ require 'bundler/setup'
5
+ require 'timecop'
6
+ require 'simplecov'
7
+ require 'celluloid/autostart'
8
+ require 'sidekiq'
9
+ require 'rspec-sidekiq'
10
+ require 'support/test_workers'
11
+
12
+ SimpleCov.start do
13
+ add_filter 'spec'
14
+ end
15
+
16
+ require 'sidekiq/batching'
17
+
18
+ Sidekiq::Batching.logger = nil
19
+ Sidekiq.redis = { namespace: ENV['namespace'] }
20
+ Sidekiq.logger = nil
21
+
22
+ RSpec::Sidekiq.configure do |config|
23
+ config.clear_all_enqueued_jobs = true
24
+ end
25
+
26
+ RSpec.configure do |config|
27
+ config.order = :random
28
+ config.treat_symbols_as_metadata_keys_with_true_values = true
29
+ config.run_all_when_everything_filtered = true
30
+ config.filter_run :focus
31
+
32
+ config.before :each do
33
+ Sidekiq.redis do |conn|
34
+ keys = conn.keys '*batching*'
35
+ keys.each { |key| conn.del key }
36
+ end
37
+ end
38
+
39
+ config.after :each do
40
+ Timecop.return
41
+ end
42
+ end
43
+
44
+ $: << File.join(File.dirname(__FILE__), '..', 'lib')
@@ -0,0 +1,33 @@
1
+ class RegularWorker
2
+ include Sidekiq::Worker
3
+
4
+ def perform(foo)
5
+ end
6
+ end
7
+
8
+ class BatchedSizeWorker
9
+ include Sidekiq::Worker
10
+
11
+ sidekiq_options queue: :batched_size, batch_size: 3
12
+
13
+ def perform(foo)
14
+ end
15
+ end
16
+
17
+ class BatchedIntervalWorker
18
+ include Sidekiq::Worker
19
+
20
+ sidekiq_options queue: :batched_interval, batch_flush_interval: 3600
21
+
22
+ def perform(foo)
23
+ end
24
+ end
25
+
26
+ class BatchedBothWorker
27
+ include Sidekiq::Worker
28
+
29
+ sidekiq_options queue: :batched_both, batch_flush_interval: 3600, batch_size: 3
30
+
31
+ def perform(foo)
32
+ end
33
+ end
metadata ADDED
@@ -0,0 +1,179 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sidekiq-batching
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Victor Sokolov
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-04-12 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.5'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.5'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: simplecov
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rspec-sidekiq
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: activesupport
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: timecop
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: sidekiq
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ description:
126
+ email:
127
+ - gzigzigzeo@gmail.com
128
+ executables: []
129
+ extensions: []
130
+ extra_rdoc_files: []
131
+ files:
132
+ - ".gitignore"
133
+ - Gemfile
134
+ - LICENSE.txt
135
+ - README.md
136
+ - Rakefile
137
+ - lib/sidekiq/batching.rb
138
+ - lib/sidekiq/batching/actor.rb
139
+ - lib/sidekiq/batching/batch.rb
140
+ - lib/sidekiq/batching/config.rb
141
+ - lib/sidekiq/batching/logging.rb
142
+ - lib/sidekiq/batching/middleware.rb
143
+ - lib/sidekiq/batching/redis.rb
144
+ - lib/sidekiq/batching/supervisor.rb
145
+ - lib/sidekiq/batching/version.rb
146
+ - lib/sidekiq/batching/views/index.erb
147
+ - lib/sidekiq/batching/web.rb
148
+ - sidekiq-batching.gemspec
149
+ - spec/modules/batch_spec.rb
150
+ - spec/spec_helper.rb
151
+ - spec/support/test_workers.rb
152
+ homepage: http://github.com/gzigzigzeo/sidekiq-batching
153
+ licenses:
154
+ - MIT
155
+ metadata: {}
156
+ post_install_message:
157
+ rdoc_options: []
158
+ require_paths:
159
+ - lib
160
+ required_ruby_version: !ruby/object:Gem::Requirement
161
+ requirements:
162
+ - - ">="
163
+ - !ruby/object:Gem::Version
164
+ version: '0'
165
+ required_rubygems_version: !ruby/object:Gem::Requirement
166
+ requirements:
167
+ - - ">="
168
+ - !ruby/object:Gem::Version
169
+ version: '0'
170
+ requirements: []
171
+ rubyforge_project:
172
+ rubygems_version: 2.2.2
173
+ signing_key:
174
+ specification_version: 4
175
+ summary: Allows identical sidekiq jobs to be processed with a single background call
176
+ test_files:
177
+ - spec/modules/batch_spec.rb
178
+ - spec/spec_helper.rb
179
+ - spec/support/test_workers.rb