sidekiq-trackable_batch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +3 -0
- data/LICENSE.txt +21 -0
- data/README.md +126 -0
- data/lib/sidekiq/overrides/client.rb +18 -0
- data/lib/sidekiq/overrides/worker.rb +29 -0
- data/lib/sidekiq/trackable_batch/initializer.rb +2 -0
- data/lib/sidekiq/trackable_batch/messages.rb +28 -0
- data/lib/sidekiq/trackable_batch/middleware.rb +26 -0
- data/lib/sidekiq/trackable_batch/persistance.rb +112 -0
- data/lib/sidekiq/trackable_batch/scripting.rb +71 -0
- data/lib/sidekiq/trackable_batch/stage.rb +29 -0
- data/lib/sidekiq/trackable_batch/success_callback.rb +18 -0
- data/lib/sidekiq/trackable_batch/tracking.rb +27 -0
- data/lib/sidekiq/trackable_batch/update_notifier.rb +19 -0
- data/lib/sidekiq/trackable_batch/workflow.rb +48 -0
- data/lib/sidekiq/trackable_batch.rb +170 -0
- metadata +144 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 481de84af018a7ec43c03b21926afe9d6137a4f6
|
4
|
+
data.tar.gz: e66a84c45a80d3967ba501bdd13473de1e5acaf4
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 5b0bdb501e87521a24dc509826c75ac5b7bc325398eabae7f5e672e5d0538356141608ab03627343bac66039569018a0afc8169c46e9722594eb45192c10dd6b
|
7
|
+
data.tar.gz: cb023a9f50ec07250c7d9e4f4ea20dc210592fa9ed8a6d09e40a5cbe6ab7f59d5462afe49993bff56deccc5a1ec852d234742b5dcfd2b799386e1ecdd57846d1
|
data/CHANGELOG.md
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 darrhiggs
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,126 @@
|
|
1
|
+
# Sidekiq::TrackableBatch
|
2
|
+
|
3
|
+
`Sidekiq::TrackableBatch` is an extension to `Sidekiq::Batch` that provides access to detailed, up-to-date progress information about a `Sidekiq::Batch` as it runs.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add the following to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'sidekiq-trackable_batch'
|
11
|
+
# and either
|
12
|
+
gem 'sidekiq-pro'
|
13
|
+
# or
|
14
|
+
gem 'sidekiq-batch' # currently unsupported
|
15
|
+
```
|
16
|
+
|
17
|
+
And then execute:
|
18
|
+
|
19
|
+
$ bundle
|
20
|
+
|
21
|
+
Or install it yourself as:
|
22
|
+
|
23
|
+
$ gem install sidekiq-trackable_batch
|
24
|
+
|
25
|
+
## Usage
|
26
|
+
|
27
|
+
Add a `.max` to existing workers:
|
28
|
+
```ruby
|
29
|
+
class MyWorker
|
30
|
+
def self.max; 42; end # some (total) amount of work
|
31
|
+
end
|
32
|
+
```
|
33
|
+
|
34
|
+
Update the class' `#perform` method to use `#update_status`:
|
35
|
+
```ruby
|
36
|
+
def perform(*args)
|
37
|
+
# do some work
|
38
|
+
update_status(value: 21) # made available through Sidekiq::Worker
|
39
|
+
# maybe do some more work
|
40
|
+
update_status(value: self.class.max, status_text: 'Done')
|
41
|
+
end
|
42
|
+
```
|
43
|
+
|
44
|
+
Create a batch using `Sidekiq::TrackableBatch`:
|
45
|
+
```ruby
|
46
|
+
trackable_batch = Sidekiq::TrackableBatch.new
|
47
|
+
# set callbacks & description etc as required
|
48
|
+
trackable_batch.jobs
|
49
|
+
5.times { MyWorker.perform_async }
|
50
|
+
end
|
51
|
+
```
|
52
|
+
|
53
|
+
Track your batch:
|
54
|
+
```ruby
|
55
|
+
Sidekiq::TrackableBatch::Tracking(trackable_batch).to_h
|
56
|
+
# => { max: 210, value: 105 }
|
57
|
+
```
|
58
|
+
|
59
|
+
All `Sidekiq::Batch` features should continue to work as before:
|
60
|
+
```ruby
|
61
|
+
Sidekiq::Batch::Status.new(trackable_batch.bid)
|
62
|
+
trackable_batch.invalidate_all
|
63
|
+
# …
|
64
|
+
```
|
65
|
+
|
66
|
+
### Really Complex Workflows with Batches
|
67
|
+
|
68
|
+
`Sidekiq::TrackableBatch` is constrained by all jobs having to be exposed to the batch during initialization. To fulfil this constraint, an updated DSL has been provided to allow nested batch creation that requires ordered execution:
|
69
|
+
```ruby
|
70
|
+
trackable_batch = Sidekiq::TrackableBatch.new do # Pass a block
|
71
|
+
# The :update callback has the same API as existing
|
72
|
+
# Sidekiq::Batch callbacks
|
73
|
+
on(:update, OrderStatusNotifier, order_id: order.id)
|
74
|
+
|
75
|
+
# updates can be pushed to another queue other than the default
|
76
|
+
self.update_queue = 'priority'
|
77
|
+
|
78
|
+
# The DSL consists of three (aliased) chainable methods:
|
79
|
+
# #start_with, #then and #finally. As with a callback, a target
|
80
|
+
# can be passed (#on_update is called by default).
|
81
|
+
# A block can be passed for inline declaration.
|
82
|
+
start_with('pick', 'Fulfilment#pick', products: order.products)
|
83
|
+
.then('pack', 'Fulfilment#pack', boxes: order.boxes)
|
84
|
+
.finally('ship', 'Fulfilment#ship', boxes: order.boxes)
|
85
|
+
# The first argument will be set as the batch's description.
|
86
|
+
|
87
|
+
# Optionally set an initial state
|
88
|
+
initial_status(status_text: 'Order sent for picking')
|
89
|
+
|
90
|
+
# Sidekiq::Batch methods are also available
|
91
|
+
on(:complete, 'Fulfilment#fulfilment_complete', order_id: order.id)
|
92
|
+
end
|
93
|
+
```
|
94
|
+
All commands, their arguments and return values are documented, and available on [rdoc.info][docs]
|
95
|
+
|
96
|
+
## DEMO
|
97
|
+
|
98
|
+
Check out the [demo app][da] ([source][dar]) to see how the [Really Complex Workflows with Batches][rcwwb] example would be created in a Rails 5 app using `Sidekiq::TrackableBatch`. The app specifically demonstrates how batch updates can be consumed through the use of the new `:update` callback, and uses ActionCable to asynchronously stream these updates to a client.
|
99
|
+
|
100
|
+
## Caveats
|
101
|
+
- ActiveJob is unsupported. [(wiki)][saj#c]
|
102
|
+
- #update_status only accepts strings to be set for anything except the value.
|
103
|
+
|
104
|
+
## TODOS
|
105
|
+
- Integrate with sidekiq UI.
|
106
|
+
- Provide mountable rack middleware à la Sidekiq Pro.
|
107
|
+
|
108
|
+
## Development
|
109
|
+
|
110
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec appraisal rake` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
111
|
+
|
112
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
113
|
+
|
114
|
+
## Contributing
|
115
|
+
|
116
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/sidekiq-trackable_batch.
|
117
|
+
|
118
|
+
## License
|
119
|
+
|
120
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
121
|
+
|
122
|
+
[da]: https://sidekiq-trackable-batch-demo.herokuapp.com/
|
123
|
+
[dar]: https://github.com/darrhiggs/sidekiq_trackable_batch_demo_app
|
124
|
+
[docs]: TODO
|
125
|
+
[rcwwb]: https://github.com/mperham/sidekiq/wiki/Really-Complex-Workflows-with-Batches
|
126
|
+
[saj#c]: https://github.com/mperham/sidekiq/wiki/Active-Job#commercial-features
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq/client'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
# @api private
|
6
|
+
class Client
|
7
|
+
alias _raw_push raw_push
|
8
|
+
|
9
|
+
def raw_push(payload)
|
10
|
+
if Thread.current[:tbatch]
|
11
|
+
Thread.current[:tbatch].job_list << payload.pop
|
12
|
+
true
|
13
|
+
else
|
14
|
+
_raw_push(payload)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq/trackable_batch/persistance'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
# Provides access to #update_status in classes that include Sidekiq::Worker
|
6
|
+
module Worker
|
7
|
+
include Sidekiq::TrackableBatch::Persistance
|
8
|
+
# @private
|
9
|
+
alias _update_status update_status
|
10
|
+
|
11
|
+
# @example Update a batch's status
|
12
|
+
# class MyWorker
|
13
|
+
# include Sidekiq::Worker
|
14
|
+
# def max; 100; end
|
15
|
+
# def perform(*)
|
16
|
+
# update_status(value: 100, more: 'info')
|
17
|
+
# end
|
18
|
+
# end
|
19
|
+
#
|
20
|
+
# @param [Hash] updates The changes to be persisted to the
|
21
|
+
# {TrackableBatch}'s current status
|
22
|
+
# @option updates [Numeric] :value Amount to increment the
|
23
|
+
# current value by.
|
24
|
+
# @option updates [String] * Any other key and value updates. (optional)
|
25
|
+
def update_status(**updates)
|
26
|
+
_update_status(batch, **updates)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# @private
|
4
|
+
class Messages
|
5
|
+
def initialize
|
6
|
+
@messages = []
|
7
|
+
end
|
8
|
+
|
9
|
+
def <<(msg)
|
10
|
+
@messages << msg
|
11
|
+
end
|
12
|
+
|
13
|
+
def max_sum
|
14
|
+
@messages.reduce(0) { |memo, msg| memo + msg['max'] }
|
15
|
+
end
|
16
|
+
|
17
|
+
def empty?
|
18
|
+
@messages.empty?
|
19
|
+
end
|
20
|
+
|
21
|
+
def to_json
|
22
|
+
@messages.map(&:to_json)
|
23
|
+
end
|
24
|
+
|
25
|
+
def clear
|
26
|
+
@messages.clear
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
class TrackableBatch < Batch
|
4
|
+
module Middleware
|
5
|
+
# @api private
|
6
|
+
class Client
|
7
|
+
def call(worker_class, msg, _queue, _redis_pool)
|
8
|
+
trackable_batch = Thread.current[:tbatch]
|
9
|
+
if trackable_batch
|
10
|
+
msg['max'] = Object.const_get(worker_class).max
|
11
|
+
out = yield
|
12
|
+
trackable_batch.register_job(out) if out
|
13
|
+
return out
|
14
|
+
end
|
15
|
+
yield
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
Sidekiq.configure_client do |config|
|
23
|
+
config.client_middleware do |chain|
|
24
|
+
chain.add Sidekiq::TrackableBatch::Middleware::Client
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq/trackable_batch/scripting'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
class TrackableBatch < Sidekiq::Batch
|
6
|
+
# @api private
|
7
|
+
# Interface for Redis persistance
|
8
|
+
module Persistance
|
9
|
+
include Scripting
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def get_status(bid)
|
14
|
+
connection { |c| c.hgetall keys(bid)[:status] }
|
15
|
+
end
|
16
|
+
|
17
|
+
def get_jobs(bid)
|
18
|
+
connection { |c| c.smembers keys(bid)[:jobs] }
|
19
|
+
end
|
20
|
+
|
21
|
+
def persist_batch(batch)
|
22
|
+
status = get_status(batch.bid)
|
23
|
+
new_max = batch.messages.max_sum + status['max'].to_i
|
24
|
+
|
25
|
+
connection do |c|
|
26
|
+
keys = keys(batch.bid)
|
27
|
+
c.multi do
|
28
|
+
c.hset keys[:status], :max, new_max
|
29
|
+
c.expire keys[:status], TTL
|
30
|
+
|
31
|
+
unless batch.messages.empty?
|
32
|
+
c.sadd keys[:jobs], batch.messages.to_json
|
33
|
+
c.expire keys[:jobs], TTL
|
34
|
+
end
|
35
|
+
|
36
|
+
if batch.update_listeners
|
37
|
+
c.set keys[:update_listeners], batch.update_listeners.to_json
|
38
|
+
c.expire keys[:update_listeners], TTL
|
39
|
+
if batch.update_queue
|
40
|
+
c.set keys[:update_queue], batch.update_queue
|
41
|
+
c.expire keys[:update_queue], TTL
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
c.hgetall keys[:status]
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
# Updates the status of a {TrackableBatch} with provided updates.
|
51
|
+
# - If the key `value` is provided the value will be incremented by that amount.
|
52
|
+
# - All other keys will create or replace a string value equal to the key's value.
|
53
|
+
# Also enqueues any update callbacks that have been registered.
|
54
|
+
def update_status(batch, updates = {})
|
55
|
+
parent_bid = batch.parent_bid
|
56
|
+
keys = if parent_bid
|
57
|
+
[
|
58
|
+
keys(parent_bid)[:status],
|
59
|
+
keys(batch.bid)[:status],
|
60
|
+
keys(parent_bid)[:update_listeners],
|
61
|
+
keys(parent_bid)[:update_queue]
|
62
|
+
]
|
63
|
+
else
|
64
|
+
[
|
65
|
+
nil,
|
66
|
+
keys(batch.bid)[:status],
|
67
|
+
keys(batch.bid)[:update_listeners],
|
68
|
+
keys(batch.bid)[:update_queue]
|
69
|
+
]
|
70
|
+
end
|
71
|
+
|
72
|
+
update_queue, update_listeners = Sidekiq.load_json(
|
73
|
+
connection do |c|
|
74
|
+
c.evalsha(
|
75
|
+
@@sha,
|
76
|
+
keys: keys,
|
77
|
+
argv: [updates.delete(:value), Sidekiq.dump_json(updates), TTL]
|
78
|
+
)
|
79
|
+
end
|
80
|
+
)
|
81
|
+
|
82
|
+
return unless update_listeners
|
83
|
+
|
84
|
+
Thread.new do # clean thread
|
85
|
+
Sidekiq::Client.push_bulk(
|
86
|
+
'queue' => update_queue,
|
87
|
+
'class' => Sidekiq::TrackableBatch::UpdateNotifier,
|
88
|
+
'args' => update_listeners.map do |update_listener|
|
89
|
+
target, args = update_listener.first
|
90
|
+
[parent_bid || batch.bid, target, args]
|
91
|
+
end
|
92
|
+
)
|
93
|
+
end.join
|
94
|
+
end
|
95
|
+
|
96
|
+
def keys(bid)
|
97
|
+
{
|
98
|
+
jobs: "TB:#{bid}:JOBS",
|
99
|
+
status: "TB:#{bid}:STATUS",
|
100
|
+
update_listeners: "TB:#{bid}:UPDATE_LISTENERS",
|
101
|
+
update_queue: "TB:#{bid}:UPDATE_QUEUE"
|
102
|
+
}
|
103
|
+
end
|
104
|
+
|
105
|
+
def connection
|
106
|
+
Sidekiq.redis do |connection|
|
107
|
+
yield connection
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
class TrackableBatch < Sidekiq::Batch
|
4
|
+
# @api private
|
5
|
+
module Scripting
|
6
|
+
SCRIPT = <<-LUA
|
7
|
+
local function hgetall(key)
|
8
|
+
local out = redis.call('HGETALL', key)
|
9
|
+
local result = {}
|
10
|
+
for i = 1, #out, 2 do
|
11
|
+
result[out[i]] = out[i + 1]
|
12
|
+
end
|
13
|
+
return result
|
14
|
+
end
|
15
|
+
|
16
|
+
local function merge_flat_dicts(base, updates)
|
17
|
+
local ret = {}
|
18
|
+
for k,v in pairs(base) do ret[k] = v end
|
19
|
+
for k,v in pairs(updates) do ret[k] = v end
|
20
|
+
return ret
|
21
|
+
end
|
22
|
+
|
23
|
+
local function dict_to_array(dict)
|
24
|
+
local ret = {}
|
25
|
+
for k,v in pairs(dict) do
|
26
|
+
table.insert(ret, k)
|
27
|
+
table.insert(ret, tostring(v))
|
28
|
+
end
|
29
|
+
return ret
|
30
|
+
end
|
31
|
+
|
32
|
+
local function update_batch_status(key, next_value, other_updates, ttl)
|
33
|
+
local other_updates = cjson.decode(other_updates)
|
34
|
+
local status = hgetall(key)
|
35
|
+
if tonumber(next_value) ~= nil then
|
36
|
+
redis.pcall('HSET', key, 'value', tonumber(status['value'] or 0) + tonumber(next_value))
|
37
|
+
end
|
38
|
+
if next(other_updates) ~= nil then
|
39
|
+
local status = hgetall(key)
|
40
|
+
redis.pcall('HMSET', key, unpack(dict_to_array(merge_flat_dicts(status, other_updates))))
|
41
|
+
end
|
42
|
+
redis.pcall('EXPIRE', key, ttl)
|
43
|
+
end
|
44
|
+
|
45
|
+
local parent_key, batch_key, update_listeners_key, update_queue_key = unpack(KEYS);
|
46
|
+
local next_value, other_updates, ttl = unpack(ARGV);
|
47
|
+
|
48
|
+
local ret = {}
|
49
|
+
|
50
|
+
update_batch_status(batch_key, next_value, other_updates, ttl)
|
51
|
+
if parent_key then update_batch_status(parent_key, next_value, other_updates, ttl) end
|
52
|
+
|
53
|
+
ret[1] = redis.pcall('GET', update_queue_key)
|
54
|
+
|
55
|
+
local update_listeners = redis.pcall('GET', update_listeners_key)
|
56
|
+
if update_listeners then
|
57
|
+
ret[2] = cjson.decode(update_listeners)
|
58
|
+
end
|
59
|
+
|
60
|
+
return cjson.encode(ret)
|
61
|
+
LUA
|
62
|
+
class << self
|
63
|
+
def load_script
|
64
|
+
Sidekiq.redis do |c|
|
65
|
+
@@sha = c.script(:load, SCRIPT)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
class TrackableBatch < Batch
|
4
|
+
# @api private
|
5
|
+
class Stage < TrackableBatch
|
6
|
+
attr_reader :job_list
|
7
|
+
|
8
|
+
def initialize(description, target, **kwargs, &block)
|
9
|
+
self.description = description
|
10
|
+
if target.respond_to?(:include?) && target.include?('#')
|
11
|
+
@target = target
|
12
|
+
end
|
13
|
+
@kwargs = kwargs
|
14
|
+
@block = block
|
15
|
+
@job_list = []
|
16
|
+
super(&nil)
|
17
|
+
end
|
18
|
+
|
19
|
+
def setup(enclosing_batch)
|
20
|
+
if @target
|
21
|
+
klass, method = @target.split('#')
|
22
|
+
Object.const_get(klass).new.send(method, self, **@kwargs)
|
23
|
+
end
|
24
|
+
instance_exec(**@kwargs, &@block) if @block
|
25
|
+
job_list.each { |job| enclosing_batch.register_job(job) }
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq/trackable_batch/persistance'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
class TrackableBatch < Batch
|
6
|
+
# @api private
|
7
|
+
class SuccessCallback
|
8
|
+
include Persistance
|
9
|
+
|
10
|
+
def on_success(_, next_stage_bid)
|
11
|
+
jobs = get_jobs(next_stage_bid)
|
12
|
+
Sidekiq::Client.new.raw_push(
|
13
|
+
jobs.map { |j| Sidekiq.load_json(j) }
|
14
|
+
)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq/trackable_batch/persistance'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
class TrackableBatch < Batch
|
6
|
+
# Access TrackableBatch progress data (status).
|
7
|
+
class Tracking
|
8
|
+
include Persistance
|
9
|
+
|
10
|
+
# @param (see TrackableBatch#initialize)
|
11
|
+
def initialize(bid)
|
12
|
+
@bid = bid
|
13
|
+
end
|
14
|
+
|
15
|
+
# Get the current status of a {TrackableBatch} as a hash. (network request)
|
16
|
+
# @return [Hash] the {TrackableBatch}'s current status
|
17
|
+
def to_h
|
18
|
+
status = get_status(@bid).reduce({}) { |m, (k, v)| m.merge k.to_sym => v }
|
19
|
+
{
|
20
|
+
max: status.delete(:max).to_i,
|
21
|
+
value: status[:value] ? status.delete(:value).to_i : nil,
|
22
|
+
**status
|
23
|
+
}
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
class TrackableBatch < Batch
|
4
|
+
# @api private
|
5
|
+
class UpdateNotifier
|
6
|
+
include Sidekiq::Worker
|
7
|
+
|
8
|
+
def perform(bid, target, args)
|
9
|
+
tracking = Tracking.new(bid).to_h
|
10
|
+
klass, method = target.split('#')
|
11
|
+
Object.const_get(klass).new.send(
|
12
|
+
method || :on_update,
|
13
|
+
tracking,
|
14
|
+
args.reduce({}) { |m, (k, v)| m.merge k.to_sym => v }
|
15
|
+
)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
class TrackableBatch < Batch
|
4
|
+
# @api private
|
5
|
+
class Workflow
|
6
|
+
attr_reader :stages
|
7
|
+
|
8
|
+
def initialize(enclosing_batch)
|
9
|
+
@enclosing_batch = enclosing_batch
|
10
|
+
@stages = []
|
11
|
+
end
|
12
|
+
|
13
|
+
def <<(stage)
|
14
|
+
@stages << stage
|
15
|
+
end
|
16
|
+
|
17
|
+
def stage(name)
|
18
|
+
@stages.detect { |stage| stage.description == name }
|
19
|
+
end
|
20
|
+
|
21
|
+
def start
|
22
|
+
Sidekiq::Client.new.raw_push(@stages.first.job_list)
|
23
|
+
end
|
24
|
+
|
25
|
+
def setup(&block)
|
26
|
+
@enclosing_batch.instance_eval(&block)
|
27
|
+
setup_callbacks
|
28
|
+
setup_jobs
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def setup_jobs
|
34
|
+
@enclosing_batch.jobs do
|
35
|
+
@stages.each do |stage|
|
36
|
+
stage.setup(@enclosing_batch)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def setup_callbacks
|
42
|
+
@stages.each_cons(2) do |stage, next_stage|
|
43
|
+
stage.on(:success, SuccessCallback, next_stage.bid)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,170 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
begin
|
3
|
+
require 'sidekiq-pro'
|
4
|
+
rescue LoadError
|
5
|
+
begin
|
6
|
+
require 'sidekiq/batch'
|
7
|
+
rescue LoadError
|
8
|
+
raise LoadError, 'Neither Sidekiq::Pro nor Sidekiq::Batch are available. ' \
|
9
|
+
'Ensure one of these libraries is made available to ' \
|
10
|
+
'Sidekiq::TrackableBatch'
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
require 'sidekiq/overrides/client'
|
15
|
+
require 'sidekiq/overrides/worker'
|
16
|
+
require 'sidekiq/trackable_batch/middleware'
|
17
|
+
require 'sidekiq/trackable_batch/messages'
|
18
|
+
require 'sidekiq/trackable_batch/persistance'
|
19
|
+
require 'sidekiq/trackable_batch/stage'
|
20
|
+
require 'sidekiq/trackable_batch/success_callback'
|
21
|
+
require 'sidekiq/trackable_batch/tracking'
|
22
|
+
require 'sidekiq/trackable_batch/update_notifier'
|
23
|
+
require 'sidekiq/trackable_batch/workflow'
|
24
|
+
|
25
|
+
module Sidekiq
|
26
|
+
# Interface for creating and tracking Sidekiq TrackableBatches
|
27
|
+
class TrackableBatch < Batch
|
28
|
+
include Persistance
|
29
|
+
|
30
|
+
# Time to live for data persisted to redis (30 Days)
|
31
|
+
TTL = 60 * 60 * 24 * 30
|
32
|
+
|
33
|
+
class << self
|
34
|
+
# Track a {TrackableBatch}
|
35
|
+
# @param trackable_batch [TrackableBatch] Instance
|
36
|
+
# @return [Tracking] Instance
|
37
|
+
def track(trackable_batch)
|
38
|
+
Tracking.new(trackable_batch.bid)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
attr_reader :messages, :update_listeners, :workflow
|
43
|
+
attr_accessor :update_queue
|
44
|
+
|
45
|
+
# Create a new TrackableBatch
|
46
|
+
# @param bid [String] An existing Batch ID
|
47
|
+
def initialize(bid = nil, &block)
|
48
|
+
@reopened = bid
|
49
|
+
@messages = Messages.new
|
50
|
+
super(bid)
|
51
|
+
return unless block_given?
|
52
|
+
@workflow = Workflow.new(self)
|
53
|
+
@workflow.setup(&block)
|
54
|
+
@workflow.start
|
55
|
+
end
|
56
|
+
|
57
|
+
# @private
|
58
|
+
# @return [true, false]
|
59
|
+
def reopened?
|
60
|
+
!@reopened.nil?
|
61
|
+
end
|
62
|
+
|
63
|
+
# @private
|
64
|
+
def register_job(msg)
|
65
|
+
@messages << msg
|
66
|
+
end
|
67
|
+
|
68
|
+
# Register a callback for a {TrackableBatch}'s status updates
|
69
|
+
#
|
70
|
+
# @example Using a class that responds to #on_update
|
71
|
+
# trackable_batch = Sidekiq::TrackableBatch.new
|
72
|
+
# trackable_batch.on(:update, Klass)
|
73
|
+
#
|
74
|
+
# @example Using a string to set a specific callback method.
|
75
|
+
# trackable_batch = Sidekiq::TrackableBatch.new do
|
76
|
+
# on(:update, 'Klass#not_on_update', more: 'information')
|
77
|
+
# end
|
78
|
+
#
|
79
|
+
# @param [Symbol] event The name of the event
|
80
|
+
# @param [Class, String] target The class and an optionally declared update method
|
81
|
+
# (defaults to #on_update)
|
82
|
+
# @param [Hash] options Any extra information required to be passed to the callback
|
83
|
+
def on(event, target, options = {})
|
84
|
+
if event == :update
|
85
|
+
@update_listeners ||= []
|
86
|
+
@update_listeners << { target => options }
|
87
|
+
else
|
88
|
+
super
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
# Chainable DSL for creating a Sidekiq::TrackableBatch with nested
|
93
|
+
# batches that require ordered execution.
|
94
|
+
#
|
95
|
+
# @example
|
96
|
+
# class Klass
|
97
|
+
# def pack(nested_batch)
|
98
|
+
# nested_batch.jobs do
|
99
|
+
# MyWorker.perform_async
|
100
|
+
# end
|
101
|
+
# end
|
102
|
+
#
|
103
|
+
# def ship(nested_batch, options)
|
104
|
+
# nested_batch.jobs do
|
105
|
+
# MyWorker.perform_async(options)
|
106
|
+
# end
|
107
|
+
# end
|
108
|
+
# end
|
109
|
+
#
|
110
|
+
# trackable_batch = Sidekiq::TrackableBatch.new do
|
111
|
+
# start_with('Pick', options: :get, passed: :along) do |options|
|
112
|
+
# jobs do
|
113
|
+
# MyWorker.perform_async(options[:passed])
|
114
|
+
# end
|
115
|
+
# end
|
116
|
+
# .then('Pack', 'Klass#pack')
|
117
|
+
# .finally('Ship', 'Klass#ship', here: :also)
|
118
|
+
# end
|
119
|
+
#
|
120
|
+
# @param [String] description The nested batch's description
|
121
|
+
# @param [String] target Class and method to call to setup the nested batch
|
122
|
+
# (required if no block is provided)
|
123
|
+
# @param [Hash] args Arguments to be forwarded to block or target
|
124
|
+
# @param [Block] block Block that executes in the context of the nested batch
|
125
|
+
# (required if no target is provided)
|
126
|
+
#
|
127
|
+
# @return receiver
|
128
|
+
def then(description, target = nil, **args, &block)
|
129
|
+
raise ArgumentError unless target || block
|
130
|
+
@workflow << Stage.new(description, target, **args, &block)
|
131
|
+
self
|
132
|
+
end
|
133
|
+
alias start_with then
|
134
|
+
alias finally then
|
135
|
+
|
136
|
+
# @param (see Sidekiq::Batch#jobs)
|
137
|
+
# @return [Array<String>] A list of JIDs for work that has been enqueued
|
138
|
+
def jobs
|
139
|
+
Thread.current[:parent_tbatch] = Thread.current[:tbatch]
|
140
|
+
Thread.current[:tbatch] = self
|
141
|
+
@jids = super
|
142
|
+
persist
|
143
|
+
@jids
|
144
|
+
ensure
|
145
|
+
Thread.current[:tbatch] = Thread.current[:parent_tbatch]
|
146
|
+
Thread.current[:parent_tbatch] = nil
|
147
|
+
end
|
148
|
+
|
149
|
+
# Set extra information in a batch's initial status
|
150
|
+
# @example Add some status text to a batch before any work is performed
|
151
|
+
# trackable_batch = Sidekiq::TrackableBatch.new
|
152
|
+
# trackable_batch.initial_status(status_text: 'Starting')
|
153
|
+
# tracking = Sidekiq::TrackableBatch.track(trackable_batch)
|
154
|
+
# tracking.to_h # => { max: nil, value: nil, status_text: 'Starting' }
|
155
|
+
#
|
156
|
+
# @param options [Hash]
|
157
|
+
def initial_status(options = {})
|
158
|
+
update_status(self, options)
|
159
|
+
end
|
160
|
+
|
161
|
+
private
|
162
|
+
|
163
|
+
def persist
|
164
|
+
persist_batch(self)
|
165
|
+
@messages.clear
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
require 'sidekiq/trackable_batch/initializer'
|
metadata
ADDED
@@ -0,0 +1,144 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sidekiq-trackable_batch
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- darrhiggs
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-07-06 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: sidekiq
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.14'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.14'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: minitest
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '5.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '5.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: appraisal
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '2.1'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '2.1'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: redcarpet
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
description: Access detailed & up-to-date progress information for `Sidekiq::Batch`
|
98
|
+
email:
|
99
|
+
- darrhiggs+os@gmail.com
|
100
|
+
executables: []
|
101
|
+
extensions: []
|
102
|
+
extra_rdoc_files: []
|
103
|
+
files:
|
104
|
+
- CHANGELOG.md
|
105
|
+
- LICENSE.txt
|
106
|
+
- README.md
|
107
|
+
- lib/sidekiq/overrides/client.rb
|
108
|
+
- lib/sidekiq/overrides/worker.rb
|
109
|
+
- lib/sidekiq/trackable_batch.rb
|
110
|
+
- lib/sidekiq/trackable_batch/initializer.rb
|
111
|
+
- lib/sidekiq/trackable_batch/messages.rb
|
112
|
+
- lib/sidekiq/trackable_batch/middleware.rb
|
113
|
+
- lib/sidekiq/trackable_batch/persistance.rb
|
114
|
+
- lib/sidekiq/trackable_batch/scripting.rb
|
115
|
+
- lib/sidekiq/trackable_batch/stage.rb
|
116
|
+
- lib/sidekiq/trackable_batch/success_callback.rb
|
117
|
+
- lib/sidekiq/trackable_batch/tracking.rb
|
118
|
+
- lib/sidekiq/trackable_batch/update_notifier.rb
|
119
|
+
- lib/sidekiq/trackable_batch/workflow.rb
|
120
|
+
homepage: https://github.com/darrhiggs/sidekiq-trackable_batch
|
121
|
+
licenses:
|
122
|
+
- MIT
|
123
|
+
metadata: {}
|
124
|
+
post_install_message:
|
125
|
+
rdoc_options: []
|
126
|
+
require_paths:
|
127
|
+
- lib
|
128
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
129
|
+
requirements:
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: '0'
|
133
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
requirements: []
|
139
|
+
rubyforge_project:
|
140
|
+
rubygems_version: 2.6.12
|
141
|
+
signing_key:
|
142
|
+
specification_version: 4
|
143
|
+
summary: Detailed `Sidekiq::Batch` progress
|
144
|
+
test_files: []
|