sidekiq-batching 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +92 -0
- data/Rakefile +1 -0
- data/lib/sidekiq/batching.rb +40 -0
- data/lib/sidekiq/batching/actor.rb +47 -0
- data/lib/sidekiq/batching/batch.rb +117 -0
- data/lib/sidekiq/batching/config.rb +19 -0
- data/lib/sidekiq/batching/logging.rb +13 -0
- data/lib/sidekiq/batching/middleware.rb +34 -0
- data/lib/sidekiq/batching/redis.rb +67 -0
- data/lib/sidekiq/batching/supervisor.rb +14 -0
- data/lib/sidekiq/batching/version.rb +5 -0
- data/lib/sidekiq/batching/views/index.erb +41 -0
- data/lib/sidekiq/batching/web.rb +28 -0
- data/sidekiq-batching.gemspec +29 -0
- data/spec/modules/batch_spec.rb +86 -0
- data/spec/spec_helper.rb +44 -0
- data/spec/support/test_workers.rb +33 -0
- metadata +179 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 680d50e306eddc51995164a7d565e6cdc777ceea
|
4
|
+
data.tar.gz: 73340f1b7215bff463cfa8f09e05fdf9de82474a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 60e20f03424b583ce84ce28bc5dce944cbc94495ca8ca6d6840ccfbbb2ee1ed58ad08ed4cd03e2c2f8a22d9a89ead2a4474e633744ac5a2a61d16f701971ebde
|
7
|
+
data.tar.gz: 8e0038fc5d3d8ce9d512a915f71ad261aedfad579e083062597db428a947eec157e66b1250fc6644534bd387707e5e2d7a53f64d43bbb306e2a07548b254ec99
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Victor Sokolov
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
# Sidekiq::Batching
|
2
|
+
|
3
|
+
Allows identical sidekiq jobs to be processed with a single background call.
|
4
|
+
|
5
|
+
Useful for:
|
6
|
+
* Grouping asynchronous API index calls into bulks for bulk updating/indexing.
|
7
|
+
* Periodical batch updating of recently changing database counters.
|
8
|
+
|
9
|
+
## Usage
|
10
|
+
|
11
|
+
Create a worker:
|
12
|
+
|
13
|
+
```ruby
|
14
|
+
class ElasticBulkIndexWorker
|
15
|
+
include Sidekiq::Worker
|
16
|
+
|
17
|
+
sidekiq_options(
|
18
|
+
queue: :batched_by_size,
|
19
|
+
batch_size: 30,
|
20
|
+
batch_flush_interval: 30,
|
21
|
+
retry: 5
|
22
|
+
)
|
23
|
+
|
24
|
+
def perform(group)
|
25
|
+
client = Elasticsearch::Client.new
|
26
|
+
client.bulk(body: group.flatten)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
```
|
30
|
+
|
31
|
+
Perform a jobs:
|
32
|
+
|
33
|
+
```ruby
|
34
|
+
ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 5, _type: 'user' } })
|
35
|
+
ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 6, _type: 'user' } })
|
36
|
+
ElasticBulkIndexWorker.perform_async({ delete: { _index: 'test', _id: 7, _type: 'user' } })
|
37
|
+
...
|
38
|
+
```
|
39
|
+
|
40
|
+
This jobs will be grouped into a single job which will be performed with the single argument containing:
|
41
|
+
|
42
|
+
```ruby
|
43
|
+
[
|
44
|
+
[{ delete: { _index: 'test', _id: 5, _type: 'user' } }],
|
45
|
+
[{ delete: { _index: 'test', _id: 6, _type: 'user' } }],
|
46
|
+
[{ delete: { _index: 'test', _id: 7, _type: 'user' } }]
|
47
|
+
...
|
48
|
+
]
|
49
|
+
```
|
50
|
+
|
51
|
+
This will happen for every 30 jobs in a row or every 30 seconds.
|
52
|
+
|
53
|
+
Add this line to your `config/routes.rb` to activate web UI:
|
54
|
+
|
55
|
+
```ruby
|
56
|
+
require "sidekiq/batching/web"
|
57
|
+
```
|
58
|
+
|
59
|
+
## Configuration
|
60
|
+
|
61
|
+
```ruby
|
62
|
+
Sidekiq::Batching::Config.poll_interval = 5 # Amount of time between polling batches
|
63
|
+
Sidekiq::Batching::Config.max_batch_size = 5000 # Maximum batch size allowed
|
64
|
+
Sidekiq::Batching::Config.lock_ttl = 1 # Timeout of lock set when batched job enqueues
|
65
|
+
```
|
66
|
+
|
67
|
+
## Notes
|
68
|
+
|
69
|
+
1. Did not tested with sidekiq 3.
|
70
|
+
1. Does not support sidekiq 3 redis_pool option.
|
71
|
+
|
72
|
+
## Installation
|
73
|
+
|
74
|
+
Add this line to your application's Gemfile:
|
75
|
+
|
76
|
+
gem 'sidekiq-batching'
|
77
|
+
|
78
|
+
And then execute:
|
79
|
+
|
80
|
+
$ bundle
|
81
|
+
|
82
|
+
Or install it yourself as:
|
83
|
+
|
84
|
+
$ gem install sidekiq-batching
|
85
|
+
|
86
|
+
## Contributing
|
87
|
+
|
88
|
+
1. Fork it ( http://github.com/gzigzigzeo/sidekiq-batching/fork )
|
89
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
90
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
91
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
92
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'active_support/core_ext/string'
|
2
|
+
require 'active_support/configurable'
|
3
|
+
require 'active_support/core_ext/numeric/time'
|
4
|
+
|
5
|
+
require 'sidekiq/batching/config'
|
6
|
+
require 'sidekiq/batching/redis'
|
7
|
+
require 'sidekiq/batching/batch'
|
8
|
+
require 'sidekiq/batching/middleware'
|
9
|
+
require 'sidekiq/batching/logging'
|
10
|
+
require 'sidekiq/batching/actor'
|
11
|
+
require 'sidekiq/batching/supervisor'
|
12
|
+
require 'sidekiq/batching/version'
|
13
|
+
|
14
|
+
module Sidekiq
|
15
|
+
module Batching
|
16
|
+
class << self
|
17
|
+
attr_writer :logger
|
18
|
+
|
19
|
+
def logger
|
20
|
+
@logger ||= Sidekiq.logger
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
Sidekiq.configure_client do |config|
|
27
|
+
config.client_middleware do |chain|
|
28
|
+
chain.add Sidekiq::Batching::Middleware
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
Sidekiq.configure_server do |config|
|
33
|
+
config.client_middleware do |chain|
|
34
|
+
chain.add Sidekiq::Batching::Middleware
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
if Sidekiq.server?
|
39
|
+
Sidekiq::Batching::Supervisor.run!
|
40
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module Batching
|
3
|
+
class Actor
|
4
|
+
include Sidekiq::Batching::Logging
|
5
|
+
include Celluloid
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
link_to_sidekiq_manager
|
9
|
+
start_polling
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
def start_polling
|
14
|
+
interval = Sidekiq::Batching::Config.poll_interval
|
15
|
+
info "Start polling of queue batches every #{interval} seconds"
|
16
|
+
every(interval) { flush_batches }
|
17
|
+
end
|
18
|
+
|
19
|
+
def flush_batches
|
20
|
+
batches = []
|
21
|
+
|
22
|
+
Sidekiq::Batching::Batch.all.map do |batch|
|
23
|
+
if batch.could_flush?
|
24
|
+
batches << batch
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
flush(batches)
|
29
|
+
end
|
30
|
+
|
31
|
+
def link_to_sidekiq_manager
|
32
|
+
Sidekiq::CLI.instance.launcher.manager.link(current_actor)
|
33
|
+
rescue NoMethodError
|
34
|
+
debug "Can't link #{self.class.name}. Sidekiq::Manager not running. Retrying in 5 seconds ..."
|
35
|
+
after(5) { link_to_sidekiq_manager }
|
36
|
+
end
|
37
|
+
|
38
|
+
def flush(batches)
|
39
|
+
if batches.any?
|
40
|
+
names = batches.map { |batch| "#{batch.worker_class} in #{batch.queue}" }
|
41
|
+
info "Trying to flush batched queues: #{names.join(',')}"
|
42
|
+
batches.each { |batch| batch.flush }
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module Batching
|
3
|
+
class Batch
|
4
|
+
|
5
|
+
def initialize(worker_class, queue, redis_pool = nil)
|
6
|
+
@worker_class = worker_class
|
7
|
+
@queue = queue
|
8
|
+
@name = "#{worker_class.underscore}:#{queue}"
|
9
|
+
@redis = Sidekiq::Batching::Redis.new
|
10
|
+
end
|
11
|
+
|
12
|
+
attr_reader :name, :worker_class, :queue
|
13
|
+
|
14
|
+
def add(msg)
|
15
|
+
@redis.push_msg(@name, msg.to_json)
|
16
|
+
end
|
17
|
+
|
18
|
+
def size
|
19
|
+
@redis.batch_size(@name)
|
20
|
+
end
|
21
|
+
|
22
|
+
def chunk_size
|
23
|
+
worker_class_options['batch_size'] ||
|
24
|
+
Sidekiq::Batching::Config.max_batch_size
|
25
|
+
end
|
26
|
+
|
27
|
+
def pluck
|
28
|
+
if @redis.lock(@name)
|
29
|
+
@redis.pluck(@name, chunk_size).map { |value| JSON.parse(value) }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def flush
|
34
|
+
chunk = pluck
|
35
|
+
if chunk
|
36
|
+
Sidekiq::Client.push(
|
37
|
+
'class' => @worker_class,
|
38
|
+
'queue' => @queue,
|
39
|
+
'args' => [true, chunk]
|
40
|
+
)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def worker_class_constant
|
45
|
+
@worker_class.constantize
|
46
|
+
end
|
47
|
+
|
48
|
+
def worker_class_options
|
49
|
+
worker_class_constant.get_sidekiq_options
|
50
|
+
rescue NameError
|
51
|
+
{}
|
52
|
+
end
|
53
|
+
|
54
|
+
def could_flush?
|
55
|
+
could_flush_on_overflow? || could_flush_on_time?
|
56
|
+
end
|
57
|
+
|
58
|
+
def last_execution_time
|
59
|
+
last_time = @redis.get_last_execution_time(@name)
|
60
|
+
Time.parse(last_time) if last_time
|
61
|
+
end
|
62
|
+
|
63
|
+
def next_execution_time
|
64
|
+
if interval = worker_class_options['batch_flush_interval']
|
65
|
+
last_time = last_execution_time
|
66
|
+
last_time + interval.seconds if last_time
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def delete
|
71
|
+
@redis.delete(@name)
|
72
|
+
end
|
73
|
+
|
74
|
+
private
|
75
|
+
def could_flush_on_overflow?
|
76
|
+
worker_class_options['batch_size'] &&
|
77
|
+
size >= worker_class_options['batch_size']
|
78
|
+
end
|
79
|
+
|
80
|
+
def could_flush_on_time?
|
81
|
+
return false if size.zero?
|
82
|
+
|
83
|
+
last_time = last_execution_time
|
84
|
+
next_time = next_execution_time
|
85
|
+
|
86
|
+
if last_time.blank?
|
87
|
+
set_current_time_as_last
|
88
|
+
false
|
89
|
+
else
|
90
|
+
if next_time
|
91
|
+
next_time < Time.now
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def set_current_time_as_last
|
97
|
+
@redis.set_last_execution_time(@name, Time.now)
|
98
|
+
end
|
99
|
+
|
100
|
+
class << self
|
101
|
+
def all
|
102
|
+
redis = Sidekiq::Batching::Redis.new
|
103
|
+
|
104
|
+
redis.batches.map do |name|
|
105
|
+
new(*extract_worker_klass_and_queue(name))
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def extract_worker_klass_and_queue(name)
|
110
|
+
klass, queue = name.split(':')
|
111
|
+
[klass.classify, queue]
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module Batching
|
3
|
+
module Config
|
4
|
+
include ActiveSupport::Configurable
|
5
|
+
|
6
|
+
# Interval batch queue polling
|
7
|
+
config_accessor :poll_interval
|
8
|
+
self.config.poll_interval = 3
|
9
|
+
|
10
|
+
# Maximum batch size
|
11
|
+
config_accessor :max_batch_size
|
12
|
+
self.config.max_batch_size = 500
|
13
|
+
|
14
|
+
# Batch queue lock timeout (set during flush)
|
15
|
+
config_accessor :lock_ttl
|
16
|
+
self.config.lock_ttl = 1
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module Batching
|
3
|
+
module Logging
|
4
|
+
%w(fatal error warn info debug).each do |level|
|
5
|
+
level = level.to_sym
|
6
|
+
|
7
|
+
define_method(level) do |msg|
|
8
|
+
Sidekiq::Batching.logger.public_send(level, "[Sidekiq::Batching] #{msg}")
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module Batching
|
3
|
+
class Middleware
|
4
|
+
def call(worker_class, msg, queue, redis_pool = nil)
|
5
|
+
worker_class = worker_class.classify.constantize if worker_class.is_a?(String)
|
6
|
+
options = worker_class.get_sidekiq_options
|
7
|
+
|
8
|
+
batch =
|
9
|
+
options.keys.include?('batch_size') ||
|
10
|
+
options.keys.include?('batch_flush_interval')
|
11
|
+
|
12
|
+
passthrough =
|
13
|
+
msg['args'] &&
|
14
|
+
msg['args'].is_a?(Array) &&
|
15
|
+
msg['args'].try(:first) == true
|
16
|
+
|
17
|
+
if batch && not(passthrough)
|
18
|
+
add_to_batch(worker_class, queue, msg, redis_pool)
|
19
|
+
else
|
20
|
+
if batch && passthrough
|
21
|
+
msg['args'].shift
|
22
|
+
end
|
23
|
+
yield
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
def add_to_batch(worker_class, queue, msg, redis_pool = nil)
|
29
|
+
Sidekiq::Batching::Batch.new(worker_class.name, queue, redis_pool).add(msg['args'])
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module Batching
|
3
|
+
class Redis
|
4
|
+
def push_msg(name, msg)
|
5
|
+
redis do |conn|
|
6
|
+
conn.sadd(ns('batches'), name)
|
7
|
+
conn.rpush(ns(name), msg)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def batch_size(name)
|
12
|
+
redis { |conn| conn.llen(ns(name)) }
|
13
|
+
end
|
14
|
+
|
15
|
+
def batches
|
16
|
+
redis { |conn| conn.smembers(ns('batches')) }
|
17
|
+
end
|
18
|
+
|
19
|
+
def pluck(name, limit)
|
20
|
+
redis do |conn|
|
21
|
+
result = conn.pipelined do
|
22
|
+
conn.lrange(ns(name), 0, limit - 1)
|
23
|
+
conn.ltrim(ns(name), limit, -1)
|
24
|
+
end
|
25
|
+
|
26
|
+
result.first
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def get_last_execution_time(name)
|
31
|
+
redis { |conn| conn.get(ns("last_execution_time:#{name}")) }
|
32
|
+
end
|
33
|
+
|
34
|
+
def set_last_execution_time(name, time)
|
35
|
+
redis { |conn| conn.set(ns("last_execution_time:#{name}"), time.to_json) }
|
36
|
+
end
|
37
|
+
|
38
|
+
def lock(name)
|
39
|
+
redis do |conn|
|
40
|
+
id = ns("lock:#{name}")
|
41
|
+
conn.setnx(id, true).tap do |obtained|
|
42
|
+
if obtained
|
43
|
+
conn.expire(id, Sidekiq::Batching::Config.lock_ttl)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def delete(name)
|
50
|
+
redis do |conn|
|
51
|
+
conn.del(ns("last_execution_time:#{name}"))
|
52
|
+
conn.del(ns(name))
|
53
|
+
conn.srem(ns('batches'), name)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
def ns(key = nil)
|
59
|
+
"batching:#{key}"
|
60
|
+
end
|
61
|
+
|
62
|
+
def redis(&block)
|
63
|
+
Sidekiq.redis(&block)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module Batching
|
3
|
+
module Supervisor
|
4
|
+
class << self
|
5
|
+
include Sidekiq::Batching::Logging
|
6
|
+
|
7
|
+
def run!
|
8
|
+
info 'Sidekiq::Batching starts supervision'
|
9
|
+
Sidekiq::Batching::Actor.supervise_as(:sidekiq_batching)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
<header class="row">
|
2
|
+
<div class="col-sm-5">
|
3
|
+
<h3>Batched jobs</h3>
|
4
|
+
</div>
|
5
|
+
</header>
|
6
|
+
|
7
|
+
<div class="container">
|
8
|
+
<div class="row">
|
9
|
+
<div class="col-sm-12">
|
10
|
+
<% if true %>
|
11
|
+
<table class="table table-striped table-bordered table-white" style="width: 100%; margin: 0; table-layout:fixed;">
|
12
|
+
<thead>
|
13
|
+
<th style="width: 50%">Worker</th>
|
14
|
+
<th style="width: 30%">Queue</th>
|
15
|
+
<th style="width: 10%">Count</th>
|
16
|
+
<th style="width: 30%">Last execution time</th>
|
17
|
+
<th style="width: 30%">Next enqueue</th>
|
18
|
+
<th style="width: 10%">Actions</th>
|
19
|
+
</thead>
|
20
|
+
<% @batches.each do |batch| %>
|
21
|
+
<tr>
|
22
|
+
<td><%= batch.worker_class %></td>
|
23
|
+
<td><%= batch.queue %></td>
|
24
|
+
<td><%= batch.size %></td>
|
25
|
+
<td><%= batch.last_execution_time || "–"%></td>
|
26
|
+
<td><%= batch.next_execution_time || "–"%></td>
|
27
|
+
<td>
|
28
|
+
<form action="<%= "#{root_path}batching/#{batch.name}/delete" %>" method="post">
|
29
|
+
<input class="btn btn-danger btn-xs" type="submit" name="delete" value="Delete" data-confirm="Are you sure you want to delete this batch?" />
|
30
|
+
</form>
|
31
|
+
</td>
|
32
|
+
</tr>
|
33
|
+
<% end %>
|
34
|
+
</table>
|
35
|
+
<% else %>
|
36
|
+
<div class="alert alert-success">No recurring jobs found.</div>
|
37
|
+
<% end %>
|
38
|
+
</div>
|
39
|
+
</div>
|
40
|
+
</div>
|
41
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'sidekiq/web'
|
2
|
+
|
3
|
+
module Sidetiq
|
4
|
+
module Batching
|
5
|
+
module Web
|
6
|
+
VIEWS = File.expand_path('views', File.dirname(__FILE__))
|
7
|
+
|
8
|
+
def self.registered(app)
|
9
|
+
app.get "/batching" do
|
10
|
+
@batches = Sidekiq::Batching::Batch.all
|
11
|
+
erb File.read(File.join(VIEWS, 'index.erb')), locals: {view_path: VIEWS}
|
12
|
+
end
|
13
|
+
|
14
|
+
app.post "/batching/:name/delete" do
|
15
|
+
worker_class, queue = Sidekiq::Batching::Batch.extract_worker_klass_and_queue(params['name'])
|
16
|
+
batch = Sidekiq::Batching::Batch.new(worker_class, queue)
|
17
|
+
batch.delete
|
18
|
+
redirect "#{root_path}/batching"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
Sidekiq::Web.register(Sidetiq::Batching::Web)
|
27
|
+
Sidekiq::Web.tabs["Batching"] = "batching"
|
28
|
+
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'sidekiq/batching/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "sidekiq-batching"
|
8
|
+
spec.version = Sidekiq::Batching::VERSION
|
9
|
+
spec.authors = ["Victor Sokolov"]
|
10
|
+
spec.email = ["gzigzigzeo@gmail.com"]
|
11
|
+
spec.summary = %q{Allows identical sidekiq jobs to be processed with a single background call}
|
12
|
+
spec.homepage = "http://github.com/gzigzigzeo/sidekiq-batching"
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0")
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_development_dependency "bundler", "~> 1.5"
|
21
|
+
spec.add_development_dependency "rake"
|
22
|
+
spec.add_development_dependency "rspec"
|
23
|
+
spec.add_development_dependency "simplecov"
|
24
|
+
spec.add_development_dependency "rspec-sidekiq"
|
25
|
+
spec.add_development_dependency "activesupport"
|
26
|
+
spec.add_development_dependency "timecop"
|
27
|
+
|
28
|
+
spec.add_dependency "sidekiq"
|
29
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Sidekiq::Batching::Batch do
|
4
|
+
subject { Sidekiq::Batching::Batch }
|
5
|
+
|
6
|
+
context 'adding' do
|
7
|
+
it 'must enqueue unbatched worker' do
|
8
|
+
RegularWorker.perform_async('bar')
|
9
|
+
expect(RegularWorker).to have_enqueued_job('bar')
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'must not enqueue batched worker' do
|
13
|
+
BatchedSizeWorker.perform_async('bar')
|
14
|
+
expect_batch(BatchedSizeWorker, 'batched_size')
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'must not enqueue batched worker' do
|
18
|
+
BatchedIntervalWorker.perform_async('bar')
|
19
|
+
expect_batch(BatchedIntervalWorker, 'batched_interval')
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'must not enqueue batched worker' do
|
23
|
+
BatchedBothWorker.perform_async('bar')
|
24
|
+
expect_batch(BatchedBothWorker, 'batched_both')
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'checking if should flush' do
|
29
|
+
it 'must flush if limit exceeds for limit worker' do
|
30
|
+
batch = subject.new(BatchedSizeWorker.name, 'batched_size')
|
31
|
+
|
32
|
+
expect(batch.could_flush?).to be_false
|
33
|
+
BatchedSizeWorker.perform_async('bar')
|
34
|
+
expect(batch.could_flush?).to be_false
|
35
|
+
4.times { BatchedSizeWorker.perform_async('bar') }
|
36
|
+
expect(batch.could_flush?).to be_true
|
37
|
+
end
|
38
|
+
|
39
|
+
it 'must flush if limit exceeds for both worker' do
|
40
|
+
batch = subject.new(BatchedBothWorker.name, 'batched_both')
|
41
|
+
|
42
|
+
expect(batch.could_flush?).to be_false
|
43
|
+
BatchedBothWorker.perform_async('bar')
|
44
|
+
expect(batch.could_flush?).to be_false
|
45
|
+
4.times { BatchedBothWorker.perform_async('bar') }
|
46
|
+
expect(batch.could_flush?).to be_true
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'must flush if limit okay but time came' do
|
50
|
+
batch = subject.new(BatchedIntervalWorker.name, 'batched_interval')
|
51
|
+
|
52
|
+
expect(batch.could_flush?).to be_false
|
53
|
+
BatchedIntervalWorker.perform_async('bar')
|
54
|
+
expect(batch.could_flush?).to be_false
|
55
|
+
expect(batch.size).to eq(1)
|
56
|
+
|
57
|
+
Timecop.travel(2.hours.since)
|
58
|
+
|
59
|
+
expect(batch.could_flush?).to be_true
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
context 'flushing' do
|
64
|
+
it 'must put wokrer to queue on flush' do
|
65
|
+
batch = subject.new(BatchedSizeWorker.name, 'batched_size')
|
66
|
+
|
67
|
+
expect(batch.could_flush?).to be_false
|
68
|
+
10.times { BatchedSizeWorker.perform_async('bar') }
|
69
|
+
batch.flush
|
70
|
+
expect(BatchedSizeWorker).to have_enqueued_job([["bar"], ["bar"], ["bar"]])
|
71
|
+
expect(batch.size).to eq(7)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
private
|
76
|
+
def expect_batch(klass, queue)
|
77
|
+
expect(klass).to_not have_enqueued_job('bar')
|
78
|
+
batch = subject.new(klass.name, queue)
|
79
|
+
stats = subject.all
|
80
|
+
expect(batch.size).to eq(1)
|
81
|
+
expect(stats.size).to eq(1)
|
82
|
+
expect(stats.first.worker_class).to eq(klass.name)
|
83
|
+
expect(stats.first.queue).to eq(queue)
|
84
|
+
expect(batch.pluck).to eq [['bar']]
|
85
|
+
end
|
86
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
$LOAD_PATH << "." unless $LOAD_PATH.include?(".")
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler/setup'
|
5
|
+
require 'timecop'
|
6
|
+
require 'simplecov'
|
7
|
+
require 'celluloid/autostart'
|
8
|
+
require 'sidekiq'
|
9
|
+
require 'rspec-sidekiq'
|
10
|
+
require 'support/test_workers'
|
11
|
+
|
12
|
+
SimpleCov.start do
|
13
|
+
add_filter 'spec'
|
14
|
+
end
|
15
|
+
|
16
|
+
require 'sidekiq/batching'
|
17
|
+
|
18
|
+
Sidekiq::Batching.logger = nil
|
19
|
+
Sidekiq.redis = { namespace: ENV['namespace'] }
|
20
|
+
Sidekiq.logger = nil
|
21
|
+
|
22
|
+
RSpec::Sidekiq.configure do |config|
|
23
|
+
config.clear_all_enqueued_jobs = true
|
24
|
+
end
|
25
|
+
|
26
|
+
RSpec.configure do |config|
|
27
|
+
config.order = :random
|
28
|
+
config.treat_symbols_as_metadata_keys_with_true_values = true
|
29
|
+
config.run_all_when_everything_filtered = true
|
30
|
+
config.filter_run :focus
|
31
|
+
|
32
|
+
config.before :each do
|
33
|
+
Sidekiq.redis do |conn|
|
34
|
+
keys = conn.keys '*batching*'
|
35
|
+
keys.each { |key| conn.del key }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
config.after :each do
|
40
|
+
Timecop.return
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
$: << File.join(File.dirname(__FILE__), '..', 'lib')
|
@@ -0,0 +1,33 @@
|
|
1
|
+
class RegularWorker
|
2
|
+
include Sidekiq::Worker
|
3
|
+
|
4
|
+
def perform(foo)
|
5
|
+
end
|
6
|
+
end
|
7
|
+
|
8
|
+
class BatchedSizeWorker
|
9
|
+
include Sidekiq::Worker
|
10
|
+
|
11
|
+
sidekiq_options queue: :batched_size, batch_size: 3
|
12
|
+
|
13
|
+
def perform(foo)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class BatchedIntervalWorker
|
18
|
+
include Sidekiq::Worker
|
19
|
+
|
20
|
+
sidekiq_options queue: :batched_interval, batch_flush_interval: 3600
|
21
|
+
|
22
|
+
def perform(foo)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
class BatchedBothWorker
|
27
|
+
include Sidekiq::Worker
|
28
|
+
|
29
|
+
sidekiq_options queue: :batched_both, batch_flush_interval: 3600, batch_size: 3
|
30
|
+
|
31
|
+
def perform(foo)
|
32
|
+
end
|
33
|
+
end
|
metadata
ADDED
@@ -0,0 +1,179 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sidekiq-batching
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Victor Sokolov
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-04-12 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.5'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.5'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: simplecov
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rspec-sidekiq
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: activesupport
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: timecop
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: sidekiq
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description:
|
126
|
+
email:
|
127
|
+
- gzigzigzeo@gmail.com
|
128
|
+
executables: []
|
129
|
+
extensions: []
|
130
|
+
extra_rdoc_files: []
|
131
|
+
files:
|
132
|
+
- ".gitignore"
|
133
|
+
- Gemfile
|
134
|
+
- LICENSE.txt
|
135
|
+
- README.md
|
136
|
+
- Rakefile
|
137
|
+
- lib/sidekiq/batching.rb
|
138
|
+
- lib/sidekiq/batching/actor.rb
|
139
|
+
- lib/sidekiq/batching/batch.rb
|
140
|
+
- lib/sidekiq/batching/config.rb
|
141
|
+
- lib/sidekiq/batching/logging.rb
|
142
|
+
- lib/sidekiq/batching/middleware.rb
|
143
|
+
- lib/sidekiq/batching/redis.rb
|
144
|
+
- lib/sidekiq/batching/supervisor.rb
|
145
|
+
- lib/sidekiq/batching/version.rb
|
146
|
+
- lib/sidekiq/batching/views/index.erb
|
147
|
+
- lib/sidekiq/batching/web.rb
|
148
|
+
- sidekiq-batching.gemspec
|
149
|
+
- spec/modules/batch_spec.rb
|
150
|
+
- spec/spec_helper.rb
|
151
|
+
- spec/support/test_workers.rb
|
152
|
+
homepage: http://github.com/gzigzigzeo/sidekiq-batching
|
153
|
+
licenses:
|
154
|
+
- MIT
|
155
|
+
metadata: {}
|
156
|
+
post_install_message:
|
157
|
+
rdoc_options: []
|
158
|
+
require_paths:
|
159
|
+
- lib
|
160
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
161
|
+
requirements:
|
162
|
+
- - ">="
|
163
|
+
- !ruby/object:Gem::Version
|
164
|
+
version: '0'
|
165
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
166
|
+
requirements:
|
167
|
+
- - ">="
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: '0'
|
170
|
+
requirements: []
|
171
|
+
rubyforge_project:
|
172
|
+
rubygems_version: 2.2.2
|
173
|
+
signing_key:
|
174
|
+
specification_version: 4
|
175
|
+
summary: Allows identical sidekiq jobs to be processed with a single background call
|
176
|
+
test_files:
|
177
|
+
- spec/modules/batch_spec.rb
|
178
|
+
- spec/spec_helper.rb
|
179
|
+
- spec/support/test_workers.rb
|