shoryuken-batch 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.rspec +2 -0
- data/.travis.yml +7 -0
- data/Gemfile +11 -0
- data/Guardfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +28 -0
- data/Rakefile +6 -0
- data/lib/shoryuken/batch.rb +241 -0
- data/lib/shoryuken/batch/callback.rb +74 -0
- data/lib/shoryuken/batch/extension/worker.rb +15 -0
- data/lib/shoryuken/batch/middleware.rb +63 -0
- data/lib/shoryuken/batch/status.rb +63 -0
- data/lib/shoryuken/batch/version.rb +5 -0
- data/lib/shoryuken/redis_connection.rb +58 -0
- data/shoryuken-batch.gemspec +28 -0
- metadata +130 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e2bd94a32734b3e80ce3a267f7b26b78e6e6f86b
|
4
|
+
data.tar.gz: 0624eb0df5f4bca047fc52a61f806ea64219e4bd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d8ab673d905c95f1f3051a3382ed0f9503ba2b05dfbec7f1b39993a349f35bd5461af266d415ddc46025884ef20b90409e34306cd01f15091464ab2c53602c54
|
7
|
+
data.tar.gz: 34745655d244634581fa695c001112a94ad6e87868009aca944746834c58996c7c9164d6bf1489f56d4b957e67115c93f8ceb8a59db38199c4a16e4a5900010f
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/Guardfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Breamware
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
[gem]: https://rubygems.org/gems/shoryuken-batch
|
2
|
+
|
3
|
+
# Shoryuken::Batch [Bleeding Edge]
|
4
|
+
|
5
|
+
Simple Shoryuken Batch Job implementation. A forked of sidekiq-batch and integrate with Shoryuken.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
gem 'shoryuken-batch'
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install shoryuken-batch
|
20
|
+
|
21
|
+
## Contributing
|
22
|
+
|
23
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/hbin/shoryuken-batch.
|
24
|
+
|
25
|
+
|
26
|
+
## License
|
27
|
+
|
28
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
@@ -0,0 +1,241 @@
|
|
1
|
+
require 'securerandom'
|
2
|
+
require 'shoryuken'
|
3
|
+
|
4
|
+
require 'shoryuken/redis_connection'
|
5
|
+
require 'shoryuken/batch/callback'
|
6
|
+
require 'shoryuken/batch/middleware'
|
7
|
+
require 'shoryuken/batch/status'
|
8
|
+
require 'shoryuken/batch/version'
|
9
|
+
|
10
|
+
module Shoryuken
|
11
|
+
def self.redis
|
12
|
+
raise ArgumentError, 'requires a block' unless block_given?
|
13
|
+
|
14
|
+
redis_pool.with do |conn|
|
15
|
+
retryable = true
|
16
|
+
begin
|
17
|
+
yield conn
|
18
|
+
rescue Redis::CommandError => ex
|
19
|
+
# Failover can cause the server to become a slave, need
|
20
|
+
# to disconnect and reopen the socket to get back to the master.
|
21
|
+
(conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
|
22
|
+
raise
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.redis_pool
|
28
|
+
@redis ||= Shoryuken::RedisConnection.create
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.redis=(hash)
|
32
|
+
@redis = if hash.is_a?(ConnectionPool)
|
33
|
+
hash
|
34
|
+
else
|
35
|
+
Shoryuken::RedisConnection.create(hash)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class Batch
|
40
|
+
class NoBlockGivenError < StandardError; end
|
41
|
+
|
42
|
+
BID_EXPIRE_TTL = 108_000
|
43
|
+
|
44
|
+
attr_reader :bid, :description, :callback_queue, :created_at
|
45
|
+
|
46
|
+
def initialize(existing_bid = nil)
|
47
|
+
@bid = existing_bid || SecureRandom.urlsafe_base64(10)
|
48
|
+
@existing = !(!existing_bid || existing_bid.empty?) # Basically existing_bid.present?
|
49
|
+
@initialized = false
|
50
|
+
@created_at = Time.now.utc.to_f
|
51
|
+
@bidkey = 'BID-' + @bid.to_s
|
52
|
+
@ready_to_queue = []
|
53
|
+
end
|
54
|
+
|
55
|
+
def description=(description)
|
56
|
+
@description = description
|
57
|
+
persist_bid_attr('description', description)
|
58
|
+
end
|
59
|
+
|
60
|
+
def callback_queue=(callback_queue)
|
61
|
+
@callback_queue = callback_queue
|
62
|
+
persist_bid_attr('callback_queue', callback_queue)
|
63
|
+
end
|
64
|
+
|
65
|
+
def on(event, callback, options = {})
|
66
|
+
return unless %w[success complete].include?(event.to_s)
|
67
|
+
callback_key = "#{@bidkey}-callbacks-#{event}"
|
68
|
+
|
69
|
+
Shoryuken.redis do |r|
|
70
|
+
r.multi do
|
71
|
+
r.sadd(callback_key, JSON.unparse(callback: callback,
|
72
|
+
opts: options))
|
73
|
+
r.expire(callback_key, BID_EXPIRE_TTL)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def jobs
|
79
|
+
raise NoBlockGivenError unless block_given?
|
80
|
+
|
81
|
+
bid_data, Thread.current[:bid_data] = Thread.current[:bid_data], []
|
82
|
+
|
83
|
+
begin
|
84
|
+
if !@existing && !@initialized
|
85
|
+
parent_bid = Thread.current[:bid].bid if Thread.current[:bid]
|
86
|
+
|
87
|
+
Shoryuken.redis do |r|
|
88
|
+
r.multi do
|
89
|
+
r.hset(@bidkey, 'created_at', @created_at)
|
90
|
+
r.hset(@bidkey, 'parent_bid', parent_bid.to_s) if parent_bid
|
91
|
+
r.expire(@bidkey, BID_EXPIRE_TTL)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
@initialized = true
|
96
|
+
end
|
97
|
+
|
98
|
+
@ready_to_queue = []
|
99
|
+
|
100
|
+
begin
|
101
|
+
parent = Thread.current[:bid]
|
102
|
+
Thread.current[:bid] = self
|
103
|
+
yield
|
104
|
+
ensure
|
105
|
+
Thread.current[:bid] = parent
|
106
|
+
end
|
107
|
+
|
108
|
+
return [] if @ready_to_queue.empty?
|
109
|
+
|
110
|
+
Shoryuken.redis do |r|
|
111
|
+
r.multi do
|
112
|
+
if parent_bid
|
113
|
+
r.hincrby("BID-#{parent_bid}", 'children', 1)
|
114
|
+
r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
|
115
|
+
end
|
116
|
+
|
117
|
+
r.hincrby(@bidkey, 'pending', @ready_to_queue.size)
|
118
|
+
r.hincrby(@bidkey, 'total', @ready_to_queue.size)
|
119
|
+
r.expire(@bidkey, BID_EXPIRE_TTL)
|
120
|
+
|
121
|
+
r.sadd(@bidkey + '-jids', @ready_to_queue)
|
122
|
+
r.expire(@bidkey + '-jids', BID_EXPIRE_TTL)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
@ready_to_queue
|
127
|
+
ensure
|
128
|
+
Thread.current[:bid_data] = bid_data
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def increment_job_queue(jid)
|
133
|
+
@ready_to_queue << jid
|
134
|
+
end
|
135
|
+
|
136
|
+
def parent_bid
|
137
|
+
Shoryuken.redis do |r|
|
138
|
+
r.hget(@bidkey, 'parent_bid')
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def parent
|
143
|
+
Shoryuken::Batch.new(parent_bid) if parent_bid
|
144
|
+
end
|
145
|
+
|
146
|
+
def valid?(batch = self)
|
147
|
+
valid = !Shoryuken.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }
|
148
|
+
batch.parent ? valid && valid?(batch.parent) : valid
|
149
|
+
end
|
150
|
+
|
151
|
+
private
|
152
|
+
|
153
|
+
def persist_bid_attr(attribute, value)
|
154
|
+
Shoryuken.redis do |r|
|
155
|
+
r.multi do
|
156
|
+
r.hset(@bidkey, attribute, value)
|
157
|
+
r.expire(@bidkey, BID_EXPIRE_TTL)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
class << self
|
163
|
+
def process_failed_job(bid, jid)
|
164
|
+
_, pending, failed, children, complete = Shoryuken.redis do |r|
|
165
|
+
r.multi do
|
166
|
+
r.sadd("BID-#{bid}-failed", jid)
|
167
|
+
|
168
|
+
r.hincrby("BID-#{bid}", 'pending', 0)
|
169
|
+
r.scard("BID-#{bid}-failed")
|
170
|
+
r.hincrby("BID-#{bid}", 'children', 0)
|
171
|
+
r.scard("BID-#{bid}-complete")
|
172
|
+
|
173
|
+
r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
enqueue_callbacks(:complete, bid) if pending.to_i == failed.to_i && children == complete
|
178
|
+
end
|
179
|
+
|
180
|
+
def process_successful_job(bid, jid)
|
181
|
+
failed, pending, children, complete, success, _total, parent_bid = Shoryuken.redis do |r|
|
182
|
+
r.multi do
|
183
|
+
r.scard("BID-#{bid}-failed")
|
184
|
+
r.hincrby("BID-#{bid}", 'pending', -1)
|
185
|
+
r.hincrby("BID-#{bid}", 'children', 0)
|
186
|
+
r.scard("BID-#{bid}-complete")
|
187
|
+
r.scard("BID-#{bid}-success")
|
188
|
+
r.hget("BID-#{bid}", 'total')
|
189
|
+
r.hget("BID-#{bid}", 'parent_bid')
|
190
|
+
|
191
|
+
r.srem("BID-#{bid}-failed", jid)
|
192
|
+
r.srem("BID-#{bid}-jids", jid)
|
193
|
+
r.expire("BID-#{bid}", BID_EXPIRE_TTL)
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
Shoryuken.logger.info "done: #{jid} in batch #{bid}"
|
198
|
+
|
199
|
+
enqueue_callbacks(:complete, bid) if pending.to_i == failed.to_i && children == complete
|
200
|
+
enqueue_callbacks(:success, bid) if pending.to_i.zero? && children == success
|
201
|
+
end
|
202
|
+
|
203
|
+
def enqueue_callbacks(event, bid)
|
204
|
+
batch_key = "BID-#{bid}"
|
205
|
+
callback_key = "#{batch_key}-callbacks-#{event}"
|
206
|
+
|
207
|
+
callbacks, queue, parent_bid = Shoryuken.redis do |r|
|
208
|
+
r.multi do
|
209
|
+
r.smembers(callback_key)
|
210
|
+
r.hget(batch_key, 'callback_queue')
|
211
|
+
r.hget(batch_key, 'parent_bid')
|
212
|
+
end
|
213
|
+
end
|
214
|
+
return if callbacks.empty?
|
215
|
+
|
216
|
+
parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
|
217
|
+
|
218
|
+
options ||= {}
|
219
|
+
options[:message_attributes] ||= {}
|
220
|
+
options[:message_attributes]['shoryuken_class'] = {
|
221
|
+
string_value: 'Shoryuken::Batch::Callback::Worker',
|
222
|
+
data_type: 'String'
|
223
|
+
}
|
224
|
+
|
225
|
+
callbacks.each do |jcb|
|
226
|
+
cb = JSON.parse(jcb)
|
227
|
+
|
228
|
+
options[:message_body] = {
|
229
|
+
'event' => event,
|
230
|
+
'bid' => bid,
|
231
|
+
'parent_bid' => parent_bid,
|
232
|
+
'job_class' => cb['callback'],
|
233
|
+
'arguments' => cb['opts']
|
234
|
+
}
|
235
|
+
|
236
|
+
Shoryuken::Client.queues(queue).send_message(options)
|
237
|
+
end
|
238
|
+
end
|
239
|
+
end
|
240
|
+
end
|
241
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
class Batch
|
3
|
+
module Callback
|
4
|
+
class Worker
|
5
|
+
include Shoryuken::Worker
|
6
|
+
|
7
|
+
shoryuken_options auto_delete: true, auto_visibility_timeout: true
|
8
|
+
|
9
|
+
def perform(_, body)
|
10
|
+
jcb = JSON.parse(body)
|
11
|
+
|
12
|
+
event = jcb['event']
|
13
|
+
bid = jcb['bid']
|
14
|
+
parent_bid = jcb['parent_bid']
|
15
|
+
clazz = jcb['job_class']
|
16
|
+
opts = jcb['arguments']
|
17
|
+
|
18
|
+
return unless %w[success complete].include?(event)
|
19
|
+
|
20
|
+
clazz, method = clazz.split('#') if clazz.class == String && clazz.include?('#')
|
21
|
+
method = "on_#{event}" if method.nil?
|
22
|
+
status = Shoryuken::Batch::Status.new(bid)
|
23
|
+
clazz.constantize.new.send(method, status, opts)
|
24
|
+
|
25
|
+
send(event.to_sym, bid, status, parent_bid)
|
26
|
+
end
|
27
|
+
|
28
|
+
def success(bid, _, parent_bid)
|
29
|
+
if parent_bid
|
30
|
+
_, _, success, pending, children = Shoryuken.redis do |r|
|
31
|
+
r.multi do
|
32
|
+
r.sadd("BID-#{parent_bid}-success", bid)
|
33
|
+
r.expire("BID-#{parent_bid}-success", Shoryuken::Batch::BID_EXPIRE_TTL)
|
34
|
+
r.scard("BID-#{parent_bid}-success")
|
35
|
+
r.hincrby("BID-#{parent_bid}", 'pending', 0)
|
36
|
+
r.hincrby("BID-#{parent_bid}", 'children', 0)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
Batch.enqueue_callbacks(:success, parent_bid) if pending.to_i.zero? && children == success
|
41
|
+
end
|
42
|
+
|
43
|
+
Shoryuken.redis do |r|
|
44
|
+
r.del "BID-#{bid}", "BID-#{bid}-success", "BID-#{bid}-complete", "BID-#{bid}-jids", "BID-#{bid}-failed"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def complete(bid, _, parent_bid)
|
49
|
+
if parent_bid
|
50
|
+
_, complete, pending, children, failure = Shoryuken.redis do |r|
|
51
|
+
r.multi do
|
52
|
+
r.sadd("BID-#{parent_bid}-complete", bid)
|
53
|
+
r.scard("BID-#{parent_bid}-complete")
|
54
|
+
r.hincrby("BID-#{parent_bid}", 'pending', 0)
|
55
|
+
r.hincrby("BID-#{parent_bid}", 'children', 0)
|
56
|
+
r.hlen("BID-#{parent_bid}-failed")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
Batch.enqueue_callbacks(:complete, parent_bid) if complete == children && pending == failure
|
61
|
+
end
|
62
|
+
|
63
|
+
pending, children, success = Shoryuken.redis do |r|
|
64
|
+
r.multi do
|
65
|
+
r.hincrby("BID-#{bid}", 'pending', 0)
|
66
|
+
r.hincrby("BID-#{bid}", 'children', 0)
|
67
|
+
r.scard("BID-#{bid}-success")
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require_relative 'extension/worker'
|
2
|
+
|
3
|
+
module Shoryuken
|
4
|
+
class Batch
|
5
|
+
module Middleware
|
6
|
+
class ClientMiddleware
|
7
|
+
def call(sqs_msg)
|
8
|
+
if (batch = Thread.current[:bid])
|
9
|
+
msg = JSON.parse(sqs_msg[:message_body])
|
10
|
+
batch.increment_job_queue(msg['job_id']) if (msg[:bid] = batch.bid)
|
11
|
+
sqs_msg[:message_body] = msg.to_json
|
12
|
+
end
|
13
|
+
|
14
|
+
yield
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
class ServerMiddleware
|
19
|
+
def call(_worker, _queue, sqs_msg, _body)
|
20
|
+
msg = JSON.parse(sqs_msg.body)
|
21
|
+
|
22
|
+
if (bid = msg.delete('bid'))
|
23
|
+
begin
|
24
|
+
Thread.current[:bid] = Shoryuken::Batch.new(bid)
|
25
|
+
yield
|
26
|
+
Thread.current[:bid] = nil
|
27
|
+
Batch.process_successful_job(bid, msg['job_id'])
|
28
|
+
rescue
|
29
|
+
Batch.process_failed_job(bid, msg['job_id'])
|
30
|
+
raise
|
31
|
+
ensure
|
32
|
+
Thread.current[:bid] = nil
|
33
|
+
end
|
34
|
+
else
|
35
|
+
yield
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.configure
|
41
|
+
Shoryuken.configure_client do |config|
|
42
|
+
config.client_middleware do |chain|
|
43
|
+
chain.add Shoryuken::Batch::Middleware::ClientMiddleware
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
Shoryuken.configure_server do |config|
|
48
|
+
config.client_middleware do |chain|
|
49
|
+
chain.add Shoryuken::Batch::Middleware::ClientMiddleware
|
50
|
+
end
|
51
|
+
|
52
|
+
config.server_middleware do |chain|
|
53
|
+
chain.add Shoryuken::Batch::Middleware::ServerMiddleware
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
Shoryuken::Worker.send(:include, Shoryuken::Batch::Extension::Worker)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
Shoryuken::Batch::Middleware.configure
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
class Batch
|
3
|
+
class Status
|
4
|
+
attr_reader :bid
|
5
|
+
|
6
|
+
def initialize(bid)
|
7
|
+
@bid = bid
|
8
|
+
end
|
9
|
+
|
10
|
+
def join
|
11
|
+
raise 'Not supported'
|
12
|
+
end
|
13
|
+
|
14
|
+
def description
|
15
|
+
Shoryuken.redis { |r| r.hget("BID-#{bid}", 'description') }
|
16
|
+
end
|
17
|
+
|
18
|
+
def pending
|
19
|
+
Shoryuken.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i
|
20
|
+
end
|
21
|
+
|
22
|
+
def failures
|
23
|
+
Shoryuken.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
|
24
|
+
end
|
25
|
+
|
26
|
+
def created_at
|
27
|
+
Shoryuken.redis { |r| r.hget("BID-#{bid}", 'created_at') }
|
28
|
+
end
|
29
|
+
|
30
|
+
def total
|
31
|
+
Shoryuken.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
|
32
|
+
end
|
33
|
+
|
34
|
+
def parent_bid
|
35
|
+
Shoryuken.redis { |r| r.hget("BID-#{bid}", 'parent_bid') }
|
36
|
+
end
|
37
|
+
|
38
|
+
def failure_info
|
39
|
+
Shoryuken.redis { |r| r.smembers("BID-#{bid}-failed") } || []
|
40
|
+
end
|
41
|
+
|
42
|
+
def complete?
|
43
|
+
Shoryuken.redis { |r| r.hget("BID-#{bid}", 'complete') } == 'true'
|
44
|
+
end
|
45
|
+
|
46
|
+
def child_count
|
47
|
+
Shoryuken.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
|
48
|
+
end
|
49
|
+
|
50
|
+
def data
|
51
|
+
{
|
52
|
+
total: total,
|
53
|
+
failures: failures,
|
54
|
+
pending: pending,
|
55
|
+
created_at: created_at,
|
56
|
+
complete: complete?,
|
57
|
+
failure_info: failure_info,
|
58
|
+
parent_bid: parent_bid
|
59
|
+
}
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'connection_pool'
|
4
|
+
require 'redis'
|
5
|
+
require 'uri'
|
6
|
+
|
7
|
+
module Shoryuken
|
8
|
+
class RedisConnection
|
9
|
+
class << self
|
10
|
+
def create(options = {})
|
11
|
+
options = options.symbolize_keys
|
12
|
+
|
13
|
+
options[:url] ||= determine_redis_provider
|
14
|
+
size = options[:size] || 5
|
15
|
+
pool_timeout = options[:pool_timeout] || 1
|
16
|
+
|
17
|
+
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
18
|
+
build_client(options)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def build_client(options)
|
25
|
+
client = Redis.new client_opts(options)
|
26
|
+
|
27
|
+
if (namespace = options[:namespace])
|
28
|
+
begin
|
29
|
+
require 'redis/namespace'
|
30
|
+
Redis::Namespace.new(namespace, :redis => client)
|
31
|
+
rescue LoadError
|
32
|
+
Shoryken.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
33
|
+
'Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.')
|
34
|
+
exit(-127)
|
35
|
+
end
|
36
|
+
else
|
37
|
+
client
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def client_opts(options)
|
42
|
+
opts = options.dup
|
43
|
+
opts.delete(:namespace) if opts[:namespace]
|
44
|
+
if opts[:network_timeout]
|
45
|
+
opts[:timeout] = opts[:network_timeout]
|
46
|
+
opts.delete(:network_timeout)
|
47
|
+
end
|
48
|
+
opts[:driver] ||= 'ruby'.freeze
|
49
|
+
opts[:reconnect_attempts] ||= 1
|
50
|
+
opts
|
51
|
+
end
|
52
|
+
|
53
|
+
def determine_redis_provider
|
54
|
+
ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'shoryuken/batch/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'shoryuken-batch'
|
8
|
+
spec.version = Shoryuken::Batch::VERSION
|
9
|
+
spec.authors = ['Bin Huang']
|
10
|
+
spec.email = ['huangbin88@foxmail.com']
|
11
|
+
|
12
|
+
spec.summary = 'Shoryuken Batch Jobs'
|
13
|
+
spec.description = 'Shoryuken Batch Jobs Implementation'
|
14
|
+
spec.homepage = 'https://github.com/hbin/shoryuken-batch'
|
15
|
+
spec.license = 'MIT'
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
18
|
+
spec.bindir = 'exe'
|
19
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
+
spec.require_paths = ['lib']
|
21
|
+
|
22
|
+
spec.add_dependency 'shoryuken'
|
23
|
+
|
24
|
+
spec.add_development_dependency 'bundler', "~> 1.12"
|
25
|
+
spec.add_development_dependency 'rake', "~> 10.0"
|
26
|
+
spec.add_development_dependency 'rspec', "~> 3.0"
|
27
|
+
spec.add_development_dependency 'fakeredis', "~> 0.5.0"
|
28
|
+
end
|
metadata
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: shoryuken-batch
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Bin Huang
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-11-28 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: shoryuken
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.12'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.12'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '10.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '10.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: fakeredis
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 0.5.0
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 0.5.0
|
83
|
+
description: Shoryuken Batch Jobs Implementation
|
84
|
+
email:
|
85
|
+
- huangbin88@foxmail.com
|
86
|
+
executables: []
|
87
|
+
extensions: []
|
88
|
+
extra_rdoc_files: []
|
89
|
+
files:
|
90
|
+
- ".gitignore"
|
91
|
+
- ".rspec"
|
92
|
+
- ".travis.yml"
|
93
|
+
- Gemfile
|
94
|
+
- Guardfile
|
95
|
+
- LICENSE.txt
|
96
|
+
- README.md
|
97
|
+
- Rakefile
|
98
|
+
- lib/shoryuken/batch.rb
|
99
|
+
- lib/shoryuken/batch/callback.rb
|
100
|
+
- lib/shoryuken/batch/extension/worker.rb
|
101
|
+
- lib/shoryuken/batch/middleware.rb
|
102
|
+
- lib/shoryuken/batch/status.rb
|
103
|
+
- lib/shoryuken/batch/version.rb
|
104
|
+
- lib/shoryuken/redis_connection.rb
|
105
|
+
- shoryuken-batch.gemspec
|
106
|
+
homepage: https://github.com/hbin/shoryuken-batch
|
107
|
+
licenses:
|
108
|
+
- MIT
|
109
|
+
metadata: {}
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - ">="
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
requirements: []
|
125
|
+
rubyforge_project:
|
126
|
+
rubygems_version: 2.6.11
|
127
|
+
signing_key:
|
128
|
+
specification_version: 4
|
129
|
+
summary: Shoryuken Batch Jobs
|
130
|
+
test_files: []
|