resque-batch 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.rspec +2 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +61 -0
- data/LICENSE.txt +21 -0
- data/README.md +112 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/lib/resque/batch.rb +1 -0
- data/lib/resque/plugins/batch.rb +146 -0
- data/lib/resque/plugins/batch/batch_job_info.rb +84 -0
- data/lib/resque/plugins/batch/job.rb +49 -0
- data/lib/resque/plugins/batch/message_handler.rb +91 -0
- data/lib/resque/plugins/batch/version.rb +7 -0
- data/lib/resque/plugins/batch/worker_job_info.rb +56 -0
- data/resque-batch.gemspec +38 -0
- data/solano.yml +4 -0
- metadata +118 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 357f8a6a48d8391a5f5b2f20e98801b757dd564fe06b76d70cd9303f353eee12
|
4
|
+
data.tar.gz: f58932446902c95dc79468fa085bc24c84188e926099a3b1c8403e1aaf4440a1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d53e9e71b59493a44c1dd5001ff995d0628277a3165b342151e073fba444b24cc458a94f6a05e738cc77a3f14f8debb8c9175b1d69f3d1427bfa7915ee390e5a
|
7
|
+
data.tar.gz: 06c66ac0875d3830c6e37880b57959893819c3be4ae82bf72c715dfdda04f8b229e9b1b5eee36971f943d7f7f3ba5a08a9e8afe32709e34163e8dc7ea946113d
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
resque-batch (0.2.1)
|
5
|
+
resque (~> 1.25)
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
byebug (9.1.0)
|
11
|
+
diff-lcs (1.3)
|
12
|
+
mono_logger (1.1.0)
|
13
|
+
multi_json (1.12.2)
|
14
|
+
mustermann (1.0.1)
|
15
|
+
rack (2.0.3)
|
16
|
+
rack-protection (2.0.0)
|
17
|
+
rack
|
18
|
+
rake (10.5.0)
|
19
|
+
redis (4.0.1)
|
20
|
+
redis-namespace (1.6.0)
|
21
|
+
redis (>= 3.0.4)
|
22
|
+
resque (1.27.4)
|
23
|
+
mono_logger (~> 1.0)
|
24
|
+
multi_json (~> 1.0)
|
25
|
+
redis-namespace (~> 1.3)
|
26
|
+
sinatra (>= 0.9.2)
|
27
|
+
vegas (~> 0.1.2)
|
28
|
+
rspec (3.6.0)
|
29
|
+
rspec-core (~> 3.6.0)
|
30
|
+
rspec-expectations (~> 3.6.0)
|
31
|
+
rspec-mocks (~> 3.6.0)
|
32
|
+
rspec-core (3.6.0)
|
33
|
+
rspec-support (~> 3.6.0)
|
34
|
+
rspec-expectations (3.6.0)
|
35
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
36
|
+
rspec-support (~> 3.6.0)
|
37
|
+
rspec-mocks (3.6.0)
|
38
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
39
|
+
rspec-support (~> 3.6.0)
|
40
|
+
rspec-support (3.6.0)
|
41
|
+
sinatra (2.0.0)
|
42
|
+
mustermann (~> 1.0)
|
43
|
+
rack (~> 2.0)
|
44
|
+
rack-protection (= 2.0.0)
|
45
|
+
tilt (~> 2.0)
|
46
|
+
tilt (2.0.8)
|
47
|
+
vegas (0.1.11)
|
48
|
+
rack (>= 1.0.0)
|
49
|
+
|
50
|
+
PLATFORMS
|
51
|
+
ruby
|
52
|
+
|
53
|
+
DEPENDENCIES
|
54
|
+
bundler (~> 1.15)
|
55
|
+
byebug
|
56
|
+
rake (~> 10.0)
|
57
|
+
resque-batch!
|
58
|
+
rspec (~> 3.0)
|
59
|
+
|
60
|
+
BUNDLED WITH
|
61
|
+
1.16.0
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 Eric Sullivan
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
# Resque - Batch
|
2
|
+
|
3
|
+
A plugin for Resque that allow for batched jobs. It provides two components: a batch and a wrapper around a normal Resque job. The wrapper handles communicating to the batch through a redis list, providing status updates as the jobs are processed. This allow the main program to call 'perform' on a batch, and wait for results that are processed by multiple resque workers.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'resque-batch', git: 'git@github.com:annkissam/resque-batch.git'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
|
18
|
+
## Usage
|
19
|
+
|
20
|
+
### Create a Batch Worker
|
21
|
+
|
22
|
+
* include 'Resque::Plugins::Batch::Job'
|
23
|
+
* the method is perform_job (not perform)
|
24
|
+
* You should return `success, message`
|
25
|
+
|
26
|
+
```ruby
|
27
|
+
class Archive
|
28
|
+
include Resque::Plugins::Batch::Job
|
29
|
+
|
30
|
+
def self.perform_job(repo_id, branch = 'master')
|
31
|
+
repo = Repository.find(repo_id)
|
32
|
+
repo.create_archive(branch)
|
33
|
+
|
34
|
+
return true, nil
|
35
|
+
end
|
36
|
+
end
|
37
|
+
```
|
38
|
+
|
39
|
+
### Create a Batch (and call perform)
|
40
|
+
|
41
|
+
```
|
42
|
+
batch = Resque::Plugins::Batch.new()
|
43
|
+
batch.enqueue(Job, 11)
|
44
|
+
batch.enqueue(Job, 12, "test2")
|
45
|
+
result = batch.perform
|
46
|
+
```
|
47
|
+
|
48
|
+
## message_handler
|
49
|
+
|
50
|
+
You can process results as they arrive w/ a message_handler:
|
51
|
+
|
52
|
+
```
|
53
|
+
batch.init_handler do |batch_jobs|
|
54
|
+
puts "Notify client it's starting"
|
55
|
+
end
|
56
|
+
|
57
|
+
batch.exit_handler do |batch_jobs|
|
58
|
+
puts "You're done!"
|
59
|
+
end
|
60
|
+
|
61
|
+
batch.job_begin_handler do |batch_jobs, job_id|
|
62
|
+
puts "Job #{job_id} started w/ params #{batch_jobs[job_id].args}"
|
63
|
+
end
|
64
|
+
|
65
|
+
batch.job_success_handler do |batch_jobs, job_id, data|
|
66
|
+
puts "Job #{job_id} succeeded w/ results #{data}"
|
67
|
+
end
|
68
|
+
|
69
|
+
result = batch.perform
|
70
|
+
```
|
71
|
+
|
72
|
+
If you need to send additional notifications there's an 'info' message
|
73
|
+
|
74
|
+
```
|
75
|
+
class Archive
|
76
|
+
include Resque::Plugins::Batch::Job
|
77
|
+
|
78
|
+
def self.perform_job(repo_id, branch = 'master')
|
79
|
+
...
|
80
|
+
@worker_job_info.info!({your: "DATA"})
|
81
|
+
...
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
batch.job_info_handler do |batch_jobs, job_id, data|
|
86
|
+
puts "Job #{job_id} sent info with your: #{data["your"]}"
|
87
|
+
end
|
88
|
+
```
|
89
|
+
|
90
|
+
|
91
|
+
## Development
|
92
|
+
|
93
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
94
|
+
|
95
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
96
|
+
|
97
|
+
### Testing
|
98
|
+
|
99
|
+
This uses Resque which requires Redis. To test:
|
100
|
+
|
101
|
+
```
|
102
|
+
$brew install redis
|
103
|
+
$redis-server /usr/local/etc/redis.conf
|
104
|
+
```
|
105
|
+
|
106
|
+
## Contributing
|
107
|
+
|
108
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/annkissam/resque-batch.
|
109
|
+
|
110
|
+
## License
|
111
|
+
|
112
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "resque/batch"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/lib/resque/batch.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'resque/plugins/batch'
|
@@ -0,0 +1,146 @@
|
|
1
|
+
require 'resque/plugins/batch'
|
2
|
+
|
3
|
+
require "resque"
|
4
|
+
require "resque/plugins/batch/version"
|
5
|
+
|
6
|
+
require "resque/plugins/batch/batch_job_info"
|
7
|
+
require "resque/plugins/batch/job"
|
8
|
+
require "resque/plugins/batch/message_handler"
|
9
|
+
require "resque/plugins/batch/worker_job_info"
|
10
|
+
|
11
|
+
module Resque
|
12
|
+
module Plugins
|
13
|
+
class Batch
|
14
|
+
JOB_HEARTBEAT = 45
|
15
|
+
JOB_HEARTBEAT_TTL = 120
|
16
|
+
|
17
|
+
attr_reader :id,
|
18
|
+
:message_handler,
|
19
|
+
:batch_jobs
|
20
|
+
|
21
|
+
def initialize(id: nil, message_handler: Resque::Plugins::Batch::MessageHandler.new)
|
22
|
+
@id = id || get_id
|
23
|
+
@message_handler = message_handler
|
24
|
+
@batch_jobs = []
|
25
|
+
end
|
26
|
+
|
27
|
+
def enqueue(klass, *args)
|
28
|
+
batch_jobs << Resque::Plugins::Batch::BatchJobInfo.new(id, batch_jobs.count, klass, *args)
|
29
|
+
end
|
30
|
+
|
31
|
+
def perform
|
32
|
+
# Make sure the incoming message queue is clear
|
33
|
+
if redis.llen(batch_key) > 0
|
34
|
+
raise "redis list #{batch_key} is not empty"
|
35
|
+
end
|
36
|
+
|
37
|
+
result = redis.multi do
|
38
|
+
batch_jobs.each_with_index do |batch_job, job_id|
|
39
|
+
klass = batch_job.klass
|
40
|
+
queue = Resque.queue_from_class(klass) || batch_queue
|
41
|
+
args = batch_job.args
|
42
|
+
args = [id, job_id] + args
|
43
|
+
|
44
|
+
if Resque.inline
|
45
|
+
begin
|
46
|
+
Resque::Job.create(queue, klass, *args)
|
47
|
+
rescue StandardError => exception
|
48
|
+
# NOTE: We still want to use the normal job messaging
|
49
|
+
end
|
50
|
+
else
|
51
|
+
Resque::Job.create(queue, klass, *args)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
unless Resque.inline
|
57
|
+
unless result.last == job_count
|
58
|
+
raise "not all jobs were queued"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
message_handler.send_message(self, :init)
|
63
|
+
|
64
|
+
last_heartbeat_check = Time.now
|
65
|
+
last_activity_check = Time.now
|
66
|
+
|
67
|
+
while(batch_jobs.any?(&:incomplete?)) do
|
68
|
+
msg = redis.lpop(batch_key)
|
69
|
+
|
70
|
+
if msg
|
71
|
+
decoded_msg = Resque.decode(msg)
|
72
|
+
|
73
|
+
if decoded_msg["job_id"]
|
74
|
+
job_id = decoded_msg["job_id"]
|
75
|
+
batch_jobs[job_id].process_job_msg(decoded_msg)
|
76
|
+
end
|
77
|
+
|
78
|
+
last_activity_check = Time.now
|
79
|
+
|
80
|
+
message_handler.send_message(self, :job, decoded_msg)
|
81
|
+
else
|
82
|
+
# Reasons there may be no message
|
83
|
+
# No Workers - check worker count
|
84
|
+
# Workers are processing another batch - register batches, check status
|
85
|
+
# A Job takes a long time - send a heartbeat
|
86
|
+
# A Job dies - send a heartbeat
|
87
|
+
|
88
|
+
if Time.now - last_heartbeat_check > JOB_HEARTBEAT
|
89
|
+
running_jobs = batch_jobs.select(&:running?)
|
90
|
+
|
91
|
+
if running_jobs.any?(&:heartbeat_running?)
|
92
|
+
last_activity_check = Time.now
|
93
|
+
end
|
94
|
+
|
95
|
+
last_heartbeat_check = Time.now
|
96
|
+
|
97
|
+
running_jobs.reject(&:heartbeat_running?).each do |batch_job|
|
98
|
+
decoded_msg = {"job_id" => batch_job.job_id, "msg" => "arrhythmia"}
|
99
|
+
batch_jobs[job_id].process_job_msg(decoded_msg)
|
100
|
+
message_handler.send_message(self, :job, decoded_msg)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
idle_duration = Time.now - last_activity_check
|
105
|
+
message_handler.send_message(self, :idle, {duration: idle_duration})
|
106
|
+
|
107
|
+
sleep(1)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
message_handler.send_message(self, :exit)
|
112
|
+
|
113
|
+
batch_jobs.all?(&:success?)
|
114
|
+
ensure
|
115
|
+
# Cleanup
|
116
|
+
redis.del(batch_key)
|
117
|
+
end
|
118
|
+
|
119
|
+
def job_count
|
120
|
+
batch_jobs.size
|
121
|
+
end
|
122
|
+
|
123
|
+
private
|
124
|
+
|
125
|
+
def redis
|
126
|
+
Resque.redis
|
127
|
+
end
|
128
|
+
|
129
|
+
def batch_queue
|
130
|
+
"batch"
|
131
|
+
end
|
132
|
+
|
133
|
+
def batch_key
|
134
|
+
"batch:#{id}"
|
135
|
+
end
|
136
|
+
|
137
|
+
# https://redis.io/commands/incr
|
138
|
+
# An atomic counter
|
139
|
+
# Used to identify the response list (batch_key)
|
140
|
+
def get_id
|
141
|
+
redis.incr("batch:id")
|
142
|
+
end
|
143
|
+
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
module Resque
|
2
|
+
module Plugins
|
3
|
+
class Batch
|
4
|
+
# A batch uses this file to store the state of each job.
|
5
|
+
# As messages are received (through redis) they're used to update this data.
|
6
|
+
class BatchJobInfo
|
7
|
+
attr_reader :batch_id,
|
8
|
+
:job_id,
|
9
|
+
:klass,
|
10
|
+
:args
|
11
|
+
|
12
|
+
attr_reader :status,
|
13
|
+
:result,
|
14
|
+
:exception,
|
15
|
+
:duration
|
16
|
+
|
17
|
+
def initialize(batch_id, job_id, klass, *args)
|
18
|
+
@batch_id = batch_id
|
19
|
+
@job_id = job_id
|
20
|
+
@klass = klass
|
21
|
+
@args = args
|
22
|
+
@status = 'pending'
|
23
|
+
end
|
24
|
+
|
25
|
+
def running?
|
26
|
+
status == 'running'
|
27
|
+
end
|
28
|
+
|
29
|
+
def success?
|
30
|
+
status == 'success'
|
31
|
+
end
|
32
|
+
|
33
|
+
def complete?
|
34
|
+
['success', 'failure', 'exception'].include?(status)
|
35
|
+
end
|
36
|
+
|
37
|
+
def incomplete?
|
38
|
+
!complete? && status != 'unknown'
|
39
|
+
end
|
40
|
+
|
41
|
+
def heartbeat_running?
|
42
|
+
redis.get(heartbeat_key) == "running"
|
43
|
+
end
|
44
|
+
|
45
|
+
# Process the msg sent from WorkerJobInfo
|
46
|
+
def process_job_msg(job_msg)
|
47
|
+
msg = job_msg["msg"]
|
48
|
+
data = job_msg["data"]
|
49
|
+
|
50
|
+
if status == 'pending' && msg == 'begin'
|
51
|
+
@status = 'running'
|
52
|
+
@start_time = Time.now
|
53
|
+
elsif (status == 'running' || status == 'unknown') && (msg == 'success' || msg == 'failure')
|
54
|
+
@status = msg
|
55
|
+
@result = data
|
56
|
+
@duration = Time.now - @start_time
|
57
|
+
elsif (status == 'running' || status == 'unknown') && msg == 'exception'
|
58
|
+
@status = 'exception'
|
59
|
+
@exception = data
|
60
|
+
@duration = Time.now - @start_time
|
61
|
+
elsif msg == "info"
|
62
|
+
# Ignore client defined messages
|
63
|
+
true
|
64
|
+
elsif msg == "arrhythmia"
|
65
|
+
@status = 'unknown'
|
66
|
+
@duration = Time.now - @start_time
|
67
|
+
else
|
68
|
+
raise "State machine Error #{job_msg}"
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
|
74
|
+
def redis
|
75
|
+
Resque.redis
|
76
|
+
end
|
77
|
+
|
78
|
+
def heartbeat_key
|
79
|
+
"batch:#{batch_id}:heartbeat:#{job_id}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Resque
|
2
|
+
module Plugins
|
3
|
+
class Batch
|
4
|
+
module Job
|
5
|
+
def self.included(base)
|
6
|
+
base.extend ClassMethods
|
7
|
+
base.class_eval do
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
module ClassMethods
|
12
|
+
def perform(batch_id, job_id, *params)
|
13
|
+
heartbeat_thread = nil
|
14
|
+
|
15
|
+
@worker_job_info = Resque::Plugins::Batch::WorkerJobInfo.new(batch_id, job_id)
|
16
|
+
|
17
|
+
@worker_job_info.heartbeat!
|
18
|
+
|
19
|
+
heartbeat_thread = Thread.new do
|
20
|
+
loop do
|
21
|
+
sleep(Resque::Plugins::Batch::JOB_HEARTBEAT)
|
22
|
+
@worker_job_info.heartbeat!
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
begin
|
27
|
+
@worker_job_info.begin!
|
28
|
+
|
29
|
+
success, data = perform_work(*params)
|
30
|
+
|
31
|
+
if success
|
32
|
+
@worker_job_info.success!(data)
|
33
|
+
else
|
34
|
+
@worker_job_info.failure!(data)
|
35
|
+
end
|
36
|
+
rescue StandardError => exception
|
37
|
+
@worker_job_info.exception!(exception)
|
38
|
+
raise exception
|
39
|
+
end
|
40
|
+
ensure
|
41
|
+
if heartbeat_thread
|
42
|
+
heartbeat_thread.exit
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
module Resque
|
2
|
+
module Plugins
|
3
|
+
class Batch
|
4
|
+
# NOTE: This is the default Message handler. It takes lambda to handle each message type.
|
5
|
+
# It could be replaced with any other class that responds to send_message(batch, type, msg = {})
|
6
|
+
|
7
|
+
# idle_duration: Set this to a number
|
8
|
+
class MessageHandler
|
9
|
+
attr_accessor :init_handler,
|
10
|
+
:exit_handler,
|
11
|
+
:idle_handler,
|
12
|
+
# :info_handler,
|
13
|
+
:job_begin_handler,
|
14
|
+
:job_success_handler,
|
15
|
+
:job_failure_handler,
|
16
|
+
:job_exception_handler,
|
17
|
+
:job_info_handler,
|
18
|
+
:job_arrhythmia_handler
|
19
|
+
|
20
|
+
def initialize(options = {})
|
21
|
+
@init_handler = options.fetch(:init, ->(_batch){})
|
22
|
+
@exit_handler = options.fetch(:exit, ->(_batch){})
|
23
|
+
@idle_handler = options.fetch(:idle, ->(_batch, msg){})
|
24
|
+
|
25
|
+
# @info_handler = options.fetch(:info, ->(_batch, msg){})
|
26
|
+
|
27
|
+
@job_begin_handler = options.fetch(:job_begin, ->(_batch, _job_id){})
|
28
|
+
@job_success_handler = options.fetch(:job_success, ->(_batch, _job_id, _data){})
|
29
|
+
@job_failure_handler = options.fetch(:job_failure, ->(_batch, _job_id, _data){})
|
30
|
+
@job_exception_handler = options.fetch(:job_exception, ->(_batch, _job_id, _data){})
|
31
|
+
@job_info_handler = options.fetch(:job_info, ->(_batch, _job_id, _data){})
|
32
|
+
@job_arrhythmia_handler = options.fetch(:job_arrhythmia, ->(_batch, _job_id){})
|
33
|
+
end
|
34
|
+
|
35
|
+
def send_message(batch, type, msg = {})
|
36
|
+
case type
|
37
|
+
when :init
|
38
|
+
send_init(batch)
|
39
|
+
when :exit
|
40
|
+
send_exit(batch)
|
41
|
+
when :idle
|
42
|
+
send_idle(batch, msg)
|
43
|
+
when :job
|
44
|
+
send_job(batch, msg)
|
45
|
+
else
|
46
|
+
raise "unknown message type: #{type}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def send_init(batch)
|
53
|
+
init_handler.call(batch)
|
54
|
+
end
|
55
|
+
|
56
|
+
def send_exit(batch)
|
57
|
+
exit_handler.call(batch)
|
58
|
+
end
|
59
|
+
|
60
|
+
def send_idle(batch, msg)
|
61
|
+
idle_handler.call(batch, msg)
|
62
|
+
end
|
63
|
+
|
64
|
+
# def send_info(batch, msg)
|
65
|
+
# end
|
66
|
+
|
67
|
+
def send_job(batch, msg)
|
68
|
+
job_id = msg["job_id"]
|
69
|
+
|
70
|
+
case msg["msg"]
|
71
|
+
when "begin"
|
72
|
+
job_begin_handler.call(batch, job_id)
|
73
|
+
when "success"
|
74
|
+
job_success_handler.call(batch, job_id, msg["data"])
|
75
|
+
when "failure"
|
76
|
+
job_failure_handler.call(batch, job_id, msg["data"])
|
77
|
+
when "exception"
|
78
|
+
job_exception_handler.call(batch, job_id, msg["data"])
|
79
|
+
when "info"
|
80
|
+
job_info_handler.call(batch, job_id, msg["data"])
|
81
|
+
when "arrhythmia"
|
82
|
+
job_arrhythmia_handler.call(batch, job_id)
|
83
|
+
else
|
84
|
+
raise "unknown msg type: #{msg["msg"]}"
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Resque
|
2
|
+
module Plugins
|
3
|
+
class Batch
|
4
|
+
# This class will be instantiated in a job and can be used to interact with the batch (by passing messages to a redis list)
|
5
|
+
class WorkerJobInfo
|
6
|
+
attr_reader :batch_id,
|
7
|
+
:job_id
|
8
|
+
|
9
|
+
def initialize(batch_id, job_id)
|
10
|
+
@batch_id = batch_id
|
11
|
+
@job_id = job_id
|
12
|
+
end
|
13
|
+
|
14
|
+
def begin!
|
15
|
+
redis.rpush(batch_key, Resque.encode(job_id: job_id, msg: 'begin'))
|
16
|
+
end
|
17
|
+
|
18
|
+
def success!(data)
|
19
|
+
redis.rpush(batch_key, Resque.encode(job_id: job_id, msg: 'success', data: data))
|
20
|
+
end
|
21
|
+
|
22
|
+
def failure!(data)
|
23
|
+
redis.rpush(batch_key, Resque.encode(job_id: job_id, msg: 'failure', data: data))
|
24
|
+
end
|
25
|
+
|
26
|
+
def exception!(exception)
|
27
|
+
redis.rpush(batch_key, Resque.encode(job_id: job_id, msg: 'exception', data: {class: exception.class.name, message: exception.message, backtrace: exception.backtrace}))
|
28
|
+
end
|
29
|
+
|
30
|
+
# NOTE: This is the only message that the client should send
|
31
|
+
def info!(data)
|
32
|
+
redis.rpush(batch_key, Resque.encode(job_id: job_id, msg: 'info', data: data))
|
33
|
+
end
|
34
|
+
|
35
|
+
def heartbeat!
|
36
|
+
redis.set(heartbeat_key, "running")
|
37
|
+
redis.expire(heartbeat_key, Resque::Plugins::Batch::JOB_HEARTBEAT_TTL)
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def redis
|
43
|
+
Resque.redis
|
44
|
+
end
|
45
|
+
|
46
|
+
def batch_key
|
47
|
+
"batch:#{batch_id}"
|
48
|
+
end
|
49
|
+
|
50
|
+
def heartbeat_key
|
51
|
+
"batch:#{batch_id}:heartbeat:#{job_id}"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "resque/plugins/batch/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "resque-batch"
|
8
|
+
spec.version = Resque::Plugins::Batch::VERSION
|
9
|
+
spec.authors = ["Eric Sullivan"]
|
10
|
+
spec.email = ["eric.sullivan@annkissam.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Batch Job functionality for Resque}
|
13
|
+
spec.description = %q{Adds methods for working with Batch Jobs}
|
14
|
+
spec.homepage = "https://github.com/annkissam/resque-batch"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
18
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
19
|
+
# if spec.respond_to?(:metadata)
|
20
|
+
# spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
|
21
|
+
# else
|
22
|
+
# raise "RubyGems 2.0 or newer is required to protect against " \
|
23
|
+
# "public gem pushes."
|
24
|
+
# end
|
25
|
+
|
26
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
27
|
+
f.match(%r{^(test|spec|features)/})
|
28
|
+
end
|
29
|
+
spec.bindir = "exe"
|
30
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
31
|
+
spec.require_paths = ["lib"]
|
32
|
+
|
33
|
+
spec.add_dependency "resque", "~> 1.25"
|
34
|
+
|
35
|
+
spec.add_development_dependency "bundler", "~> 1.15"
|
36
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
37
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
38
|
+
end
|
data/solano.yml
ADDED
metadata
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: resque-batch
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Eric Sullivan
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-08-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: resque
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.25'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.25'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.15'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.15'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.0'
|
69
|
+
description: Adds methods for working with Batch Jobs
|
70
|
+
email:
|
71
|
+
- eric.sullivan@annkissam.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- ".gitignore"
|
77
|
+
- ".rspec"
|
78
|
+
- Gemfile
|
79
|
+
- Gemfile.lock
|
80
|
+
- LICENSE.txt
|
81
|
+
- README.md
|
82
|
+
- Rakefile
|
83
|
+
- bin/console
|
84
|
+
- bin/setup
|
85
|
+
- lib/resque/batch.rb
|
86
|
+
- lib/resque/plugins/batch.rb
|
87
|
+
- lib/resque/plugins/batch/batch_job_info.rb
|
88
|
+
- lib/resque/plugins/batch/job.rb
|
89
|
+
- lib/resque/plugins/batch/message_handler.rb
|
90
|
+
- lib/resque/plugins/batch/version.rb
|
91
|
+
- lib/resque/plugins/batch/worker_job_info.rb
|
92
|
+
- resque-batch.gemspec
|
93
|
+
- solano.yml
|
94
|
+
homepage: https://github.com/annkissam/resque-batch
|
95
|
+
licenses:
|
96
|
+
- MIT
|
97
|
+
metadata: {}
|
98
|
+
post_install_message:
|
99
|
+
rdoc_options: []
|
100
|
+
require_paths:
|
101
|
+
- lib
|
102
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
103
|
+
requirements:
|
104
|
+
- - ">="
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '0'
|
107
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
108
|
+
requirements:
|
109
|
+
- - ">="
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: '0'
|
112
|
+
requirements: []
|
113
|
+
rubyforge_project:
|
114
|
+
rubygems_version: 2.7.6
|
115
|
+
signing_key:
|
116
|
+
specification_version: 4
|
117
|
+
summary: Batch Job functionality for Resque
|
118
|
+
test_files: []
|