shoryuken 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +14 -15
- data/Rakefile +0 -24
- data/examples/bootstrap_queues.rb +20 -0
- data/examples/default_worker.rb +12 -1
- data/lib/shoryuken.rb +61 -37
- data/lib/shoryuken/cli.rb +32 -3
- data/lib/shoryuken/client.rb +2 -0
- data/lib/shoryuken/fetcher.rb +9 -4
- data/lib/shoryuken/manager.rb +16 -2
- data/lib/shoryuken/middleware/server/active_record.rb +13 -0
- data/lib/shoryuken/middleware/server/auto_delete.rb +2 -4
- data/lib/shoryuken/processor.rb +3 -4
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +26 -2
- data/lib/shoryuken/worker_loader.rb +17 -0
- data/shoryuken.gemspec +2 -2
- data/spec/integration/launcher_spec.rb +48 -11
- data/spec/shoryuken/client_spec.rb +16 -0
- data/spec/shoryuken/fetcher_spec.rb +17 -3
- data/spec/shoryuken/manager_spec.rb +33 -0
- data/spec/shoryuken/middleware/server/auto_delete_spec.rb +3 -3
- data/spec/shoryuken/processor_spec.rb +29 -8
- data/spec/shoryuken/worker_loader_spec.rb +27 -0
- data/spec/shoryuken/worker_spec.rb +62 -2
- data/spec/shoryuken_spec.rb +51 -0
- data/spec/spec_helper.rb +5 -1
- metadata +14 -10
- data/examples/all.rb +0 -5
- data/examples/high_priority_worker.rb +0 -9
- data/examples/low_priority_worker.rb +0 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 70a39ae890a6fe9dcff44b0c6ceeab9c77931ab2
|
4
|
+
data.tar.gz: ff223f629ed2f60cc0abb32fa17f28e15d680d2f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f0afdb29904c40200d1976f2b26973ae6694e7018b07bf133153c355e250a1d613cf1fd9c0500c2aaa612d77f4c5d631823ab186642b4a2051a76ea78feea4e6
|
7
|
+
data.tar.gz: 8b3d0dc3f255b3d921386a322e645e7875f2962c902094a48671d24a95211b02e5f0558be53e30619baf92551e9f3d7e23b60def11a2c11b84107525e339f141
|
data/README.md
CHANGED
@@ -30,13 +30,12 @@ If `high_priority` gets empty, Shoryuken will keep using the 25 processors, but
|
|
30
30
|
|
31
31
|
If `high_priority` receives a new message, Shoryuken will smoothly increase back the `high_priority` weight one by one until it reaches the weight of 6 again, which is the maximum configured for `high_priority`.
|
32
32
|
|
33
|
-
If all queues get empty, all processors will be changed to the waiting state and the queues will be checked every `delay: 25`. If any queue receives a new message, Shoryuken will start processing again.
|
33
|
+
If all queues get empty, all processors will be changed to the waiting state and the queues will be checked every `delay: 25`. If any queue receives a new message, Shoryuken will start processing again. [Check the delay option documentation for more information](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#delay).
|
34
34
|
|
35
|
-
*You can set `delay: 0` to continuously check the queues without pausing even if they are empty.*
|
36
35
|
|
37
36
|
### Fetch in batches
|
38
37
|
|
39
|
-
To be even more performance and cost efficient, Shoryuken fetches SQS messages in batches.
|
38
|
+
To be even more performance and cost efficient, Shoryuken fetches SQS messages in batches, so a single SQS request can fetch up to 10 messages.
|
40
39
|
|
41
40
|
## Installation
|
42
41
|
|
@@ -44,7 +43,7 @@ Add this line to your application's Gemfile:
|
|
44
43
|
|
45
44
|
gem 'shoryuken'
|
46
45
|
|
47
|
-
|
46
|
+
Or to get the latest updates:
|
48
47
|
|
49
48
|
gem 'shoryuken', github: 'phstc/shoryuken', branch: 'master'
|
50
49
|
|
@@ -64,7 +63,7 @@ Or install it yourself as:
|
|
64
63
|
class MyWorker
|
65
64
|
include Shoryuken::Worker
|
66
65
|
|
67
|
-
shoryuken_options queue: 'default',
|
66
|
+
shoryuken_options queue: 'default', auto_delete: true
|
68
67
|
# shoryuken_options queue: ->{ "#{ENV['environment']_default" }
|
69
68
|
|
70
69
|
# shoryuken_options body_parser: :json
|
@@ -81,16 +80,7 @@ end
|
|
81
80
|
|
82
81
|
### Sending a message
|
83
82
|
|
84
|
-
|
85
|
-
MyWorker.perform_async('Pablo')
|
86
|
-
# or
|
87
|
-
Shoryuken::Client.queues('default').send_message('Pablo')
|
88
|
-
|
89
|
-
# delaying a message
|
90
|
-
MyWorker.perform_async('Pablo', delay_seconds: 60)
|
91
|
-
# or
|
92
|
-
Shoryuken::Client.queues('default').send_message('Pablo', delay_seconds: 60)
|
93
|
-
```
|
83
|
+
[Check the Sending a message documentation](https://github.com/phstc/shoryuken/wiki/Sending-a-message)
|
94
84
|
|
95
85
|
### Midleware
|
96
86
|
|
@@ -128,6 +118,14 @@ queues:
|
|
128
118
|
- [low_priority, 1]
|
129
119
|
```
|
130
120
|
|
121
|
+
### Rails Integration
|
122
|
+
|
123
|
+
You can tell Shoryuken to load your Rails application by passing the `-R` or `--rails` flag to the "shoryuken" command.
|
124
|
+
|
125
|
+
If you load Rails, and assuming your workers are located in the `app/workers` directory, they will be auto-loaded. This means you don't need to require them explicitly with `-r`.
|
126
|
+
|
127
|
+
This feature works for Rails 4+, but needs to be confirmed for older versions.
|
128
|
+
|
131
129
|
### Start Shoryuken
|
132
130
|
|
133
131
|
```shell
|
@@ -145,6 +143,7 @@ shoryuken [options]
|
|
145
143
|
-q, --queue QUEUE[,WEIGHT]... Queues to process with optional weights
|
146
144
|
-r, --require [PATH|DIR] Location of the worker
|
147
145
|
-C, --config PATH Path to YAML config file
|
146
|
+
-R, --rails Attempts to load the containing Rails project
|
148
147
|
-L, --logfile PATH Path to writable logfile
|
149
148
|
-P, --pidfile PATH Path to pidfile
|
150
149
|
-v, --verbose Print more verbose output
|
data/Rakefile
CHANGED
@@ -21,27 +21,3 @@ task :console do
|
|
21
21
|
ARGV.clear
|
22
22
|
Pry.start
|
23
23
|
end
|
24
|
-
|
25
|
-
desc 'Push test messages to high_priority, default and low_priority'
|
26
|
-
task :push_test, :size do |t, args|
|
27
|
-
require 'yaml'
|
28
|
-
require 'shoryuken'
|
29
|
-
|
30
|
-
config = YAML.load File.read(File.join(File.expand_path('..', __FILE__), 'shoryuken.yml'))
|
31
|
-
|
32
|
-
AWS.config(config['aws'])
|
33
|
-
|
34
|
-
Shoryuken::Client.sqs.queues.create('default')
|
35
|
-
Shoryuken::Client.sqs.queues.create('high_priority')
|
36
|
-
Shoryuken::Client.sqs.queues.create('low_priority')
|
37
|
-
|
38
|
-
(args[:size] || 1).to_i.times.map do |i|
|
39
|
-
Thread.new do
|
40
|
-
puts "Pushing test ##{i}"
|
41
|
-
|
42
|
-
Shoryuken::Client.queues('high_priority').send_message("test #{i}")
|
43
|
-
Shoryuken::Client.queues('default').send_message("test #{i}")
|
44
|
-
Shoryuken::Client.queues('low_priority').send_message("test #{i}")
|
45
|
-
end
|
46
|
-
end.each &:join
|
47
|
-
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'shoryuken'
|
3
|
+
|
4
|
+
# load SQS credentials
|
5
|
+
config = YAML.load File.read(File.join(File.expand_path('..', __FILE__), 'shoryuken.yml'))
|
6
|
+
|
7
|
+
AWS.config(config['aws'])
|
8
|
+
|
9
|
+
sqs = AWS::SQS.new
|
10
|
+
|
11
|
+
# create a queue and a respective dead letter queue
|
12
|
+
# after 7 attempts SQS will move the message to the dead letter queue
|
13
|
+
|
14
|
+
dl_name = 'default_failures'
|
15
|
+
dl = sqs.queues.create(dl_name)
|
16
|
+
|
17
|
+
options = {}
|
18
|
+
options[:redrive_policy] = %Q{{"maxReceiveCount":"7", "deadLetterTargetArn":"#{dl.arn}"}"}
|
19
|
+
|
20
|
+
sqs.queues.create('default', options)
|
data/examples/default_worker.rb
CHANGED
@@ -1,9 +1,20 @@
|
|
1
1
|
class DefaultWorker
|
2
2
|
include Shoryuken::Worker
|
3
3
|
|
4
|
-
shoryuken_options queue: 'default',
|
4
|
+
shoryuken_options queue: 'default', auto_delete: true
|
5
5
|
|
6
6
|
def perform(sqs_msg, body)
|
7
7
|
puts "DefaultWorker: '#{body}'"
|
8
8
|
end
|
9
9
|
end
|
10
|
+
|
11
|
+
# multiple workers for the same queue
|
12
|
+
class DefaultWorker2
|
13
|
+
include Shoryuken::Worker
|
14
|
+
|
15
|
+
shoryuken_options queue: 'default', auto_delete: true
|
16
|
+
|
17
|
+
def perform(sqs_msg, body)
|
18
|
+
puts "DefaultWorker2: '#{body}'"
|
19
|
+
end
|
20
|
+
end
|
data/lib/shoryuken.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'yaml'
|
2
|
-
require 'aws-sdk'
|
2
|
+
require 'aws-sdk-v1'
|
3
3
|
require 'time'
|
4
4
|
|
5
5
|
require 'shoryuken/version'
|
@@ -7,6 +7,7 @@ require 'shoryuken/core_ext'
|
|
7
7
|
require 'shoryuken/util'
|
8
8
|
require 'shoryuken/client'
|
9
9
|
require 'shoryuken/worker'
|
10
|
+
require 'shoryuken/worker_loader'
|
10
11
|
require 'shoryuken/logging'
|
11
12
|
require 'shoryuken/middleware/chain'
|
12
13
|
require 'shoryuken/middleware/server/auto_delete'
|
@@ -21,52 +22,75 @@ module Shoryuken
|
|
21
22
|
timeout: 8
|
22
23
|
}
|
23
24
|
|
24
|
-
@@workers
|
25
|
-
@@queues
|
25
|
+
@@workers = {}
|
26
|
+
@@queues = []
|
27
|
+
@@worker_loader = WorkerLoader
|
26
28
|
|
27
|
-
|
28
|
-
|
29
|
-
|
29
|
+
class << self
|
30
|
+
def options
|
31
|
+
@options ||= DEFAULTS.dup
|
32
|
+
end
|
30
33
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
+
def register_worker(queue, clazz)
|
35
|
+
if worker_class = @@workers[queue]
|
36
|
+
if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
|
37
|
+
raise ArgumentError, "Could not register #{clazz} for '#{queue}', "\
|
38
|
+
"because #{worker_class} is already registered for this queue, "\
|
39
|
+
"and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
|
40
|
+
end
|
41
|
+
end
|
34
42
|
|
35
|
-
|
36
|
-
|
37
|
-
end
|
43
|
+
@@workers[queue] = clazz
|
44
|
+
end
|
38
45
|
|
39
|
-
|
40
|
-
|
41
|
-
|
46
|
+
def workers
|
47
|
+
@@workers
|
48
|
+
end
|
42
49
|
|
43
|
-
|
44
|
-
|
45
|
-
|
50
|
+
def queues
|
51
|
+
@@queues
|
52
|
+
end
|
46
53
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
# end
|
51
|
-
# end
|
52
|
-
def self.configure_server
|
53
|
-
yield self
|
54
|
-
end
|
54
|
+
def logger
|
55
|
+
Shoryuken::Logging.logger
|
56
|
+
end
|
55
57
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
58
|
+
def worker_loader=(worker_loader)
|
59
|
+
@@worker_loader = worker_loader
|
60
|
+
end
|
61
|
+
|
62
|
+
def worker_loader
|
63
|
+
@@worker_loader
|
64
|
+
end
|
65
|
+
|
66
|
+
# Shoryuken.configure_server do |config|
|
67
|
+
# config.server_middleware do |chain|
|
68
|
+
# chain.add MyServerHook
|
69
|
+
# end
|
70
|
+
# end
|
71
|
+
def configure_server
|
72
|
+
yield self
|
73
|
+
end
|
74
|
+
|
75
|
+
def server_middleware
|
76
|
+
@server_chain ||= default_server_middleware
|
77
|
+
yield @server_chain if block_given?
|
78
|
+
@server_chain
|
79
|
+
end
|
61
80
|
|
62
81
|
|
63
|
-
|
82
|
+
private
|
64
83
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
84
|
+
def default_server_middleware
|
85
|
+
Middleware::Chain.new do |m|
|
86
|
+
m.add Middleware::Server::Timing
|
87
|
+
m.add Middleware::Server::AutoDelete
|
88
|
+
if defined?(::ActiveRecord::Base)
|
89
|
+
require 'shoryuken/middleware/server/active_record'
|
90
|
+
m.add Middleware::Server::ActiveRecord
|
91
|
+
end
|
92
|
+
# TODO m.add Middleware::Server::RetryJobs
|
93
|
+
end
|
70
94
|
end
|
71
95
|
end
|
72
96
|
end
|
data/lib/shoryuken/cli.rb
CHANGED
@@ -22,7 +22,10 @@ module Shoryuken
|
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
25
|
-
setup_options(args)
|
25
|
+
setup_options(args) do |cli_options|
|
26
|
+
# this needs to happen before configuration is parsed, since it may depend on Rails env
|
27
|
+
load_rails if cli_options[:rails]
|
28
|
+
end
|
26
29
|
initialize_logger
|
27
30
|
require_workers
|
28
31
|
validate!
|
@@ -61,6 +64,25 @@ module Shoryuken
|
|
61
64
|
require 'shoryuken/manager'
|
62
65
|
end
|
63
66
|
|
67
|
+
def load_rails
|
68
|
+
# Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
|
69
|
+
|
70
|
+
require 'rails'
|
71
|
+
if ::Rails::VERSION::MAJOR < 4
|
72
|
+
require File.expand_path("config/environment.rb")
|
73
|
+
::Rails.application.eager_load!
|
74
|
+
else
|
75
|
+
# Painful contortions, see 1791 for discussion
|
76
|
+
require File.expand_path("config/application.rb")
|
77
|
+
::Rails::Application.initializer "shoryuken.eager_load" do
|
78
|
+
::Rails.application.config.eager_load = true
|
79
|
+
end
|
80
|
+
require File.expand_path("config/environment.rb")
|
81
|
+
end
|
82
|
+
|
83
|
+
logger.info "Rails environment loaded"
|
84
|
+
end
|
85
|
+
|
64
86
|
def daemonize
|
65
87
|
return unless Shoryuken.options[:daemon]
|
66
88
|
|
@@ -125,6 +147,10 @@ module Shoryuken
|
|
125
147
|
opts[:config_file] = arg
|
126
148
|
end
|
127
149
|
|
150
|
+
o.on '-R', '--rails', 'Load Rails' do |arg|
|
151
|
+
opts[:rails] = arg
|
152
|
+
end
|
153
|
+
|
128
154
|
o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
|
129
155
|
opts[:logfile] = arg
|
130
156
|
end
|
@@ -187,6 +213,9 @@ module Shoryuken
|
|
187
213
|
def setup_options(args)
|
188
214
|
options = parse_options(args)
|
189
215
|
|
216
|
+
# yield parsed options in case we need to do more setup before configuration is parsed
|
217
|
+
yield(options) if block_given?
|
218
|
+
|
190
219
|
config = options[:config_file] ? parse_config(options[:config_file]).deep_symbolize_keys : {}
|
191
220
|
|
192
221
|
Shoryuken.options.merge!(config)
|
@@ -213,8 +242,8 @@ module Shoryuken
|
|
213
242
|
def validate!
|
214
243
|
raise ArgumentError, 'No queues supplied' if Shoryuken.queues.empty?
|
215
244
|
|
216
|
-
|
217
|
-
|
245
|
+
Shoryuken.queues.each do |queue|
|
246
|
+
logger.warn "No worker supplied for '#{queue}'" unless Shoryuken.workers.include? queue
|
218
247
|
end
|
219
248
|
|
220
249
|
if Shoryuken.options[:aws][:access_key_id].nil? && Shoryuken.options[:aws][:secret_access_key].nil?
|
data/lib/shoryuken/client.rb
CHANGED
data/lib/shoryuken/fetcher.rb
CHANGED
@@ -13,17 +13,22 @@ module Shoryuken
|
|
13
13
|
# AWS limits the batch size by 10
|
14
14
|
limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
|
15
15
|
|
16
|
-
|
16
|
+
options = Shoryuken.options[:aws][:receive_message].to_h
|
17
|
+
options[:limit] = limit
|
18
|
+
options[:message_attribute_names] ||= []
|
19
|
+
options[:message_attribute_names] << 'shoryuken_class'
|
20
|
+
|
21
|
+
Shoryuken::Client.receive_message queue, options
|
17
22
|
end
|
18
23
|
|
19
24
|
def fetch(queue, available_processors)
|
20
25
|
watchdog('Fetcher#fetch died') do
|
21
26
|
started_at = Time.now
|
22
27
|
|
23
|
-
logger.
|
28
|
+
logger.debug "Looking for new messages in '#{queue}'"
|
24
29
|
|
25
30
|
begin
|
26
|
-
batch = !!Shoryuken.workers[queue].get_shoryuken_options['batch']
|
31
|
+
batch = !!(Shoryuken.workers[queue] && Shoryuken.workers[queue].get_shoryuken_options['batch'])
|
27
32
|
|
28
33
|
limit = batch ? FETCH_LIMIT : available_processors
|
29
34
|
|
@@ -38,7 +43,7 @@ module Shoryuken
|
|
38
43
|
|
39
44
|
@manager.async.rebalance_queue_weight!(queue)
|
40
45
|
else
|
41
|
-
logger.
|
46
|
+
logger.debug "No message found for '#{queue}'"
|
42
47
|
|
43
48
|
@manager.async.pause_queue!(queue)
|
44
49
|
end
|
data/lib/shoryuken/manager.rb
CHANGED
@@ -103,7 +103,7 @@ module Shoryuken
|
|
103
103
|
def pause_queue!(queue)
|
104
104
|
return if !@queues.include?(queue) || Shoryuken.options[:delay].to_f <= 0
|
105
105
|
|
106
|
-
logger.
|
106
|
+
logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty"
|
107
107
|
|
108
108
|
@queues.delete(queue)
|
109
109
|
|
@@ -139,7 +139,7 @@ module Shoryuken
|
|
139
139
|
return if stopped?
|
140
140
|
|
141
141
|
unless @queues.include? queue
|
142
|
-
logger.
|
142
|
+
logger.debug "Restarting '#{queue}'"
|
143
143
|
|
144
144
|
@queues << queue
|
145
145
|
|
@@ -169,7 +169,21 @@ module Shoryuken
|
|
169
169
|
def next_queue
|
170
170
|
return nil if @queues.empty?
|
171
171
|
|
172
|
+
# get/remove the first queue in the list
|
172
173
|
queue = @queues.shift
|
174
|
+
|
175
|
+
|
176
|
+
unless Shoryuken.workers.include? queue
|
177
|
+
# when no worker registered pause the queue to avoid endless recursion
|
178
|
+
|
179
|
+
logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because of no workers registered"
|
180
|
+
|
181
|
+
after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
|
182
|
+
|
183
|
+
return next_queue
|
184
|
+
end
|
185
|
+
|
186
|
+
# add queue back to the end of the list
|
173
187
|
@queues << queue
|
174
188
|
|
175
189
|
queue
|
@@ -2,14 +2,12 @@ module Shoryuken
|
|
2
2
|
module Middleware
|
3
3
|
module Server
|
4
4
|
class AutoDelete
|
5
|
-
|
6
5
|
def call(worker, queue, sqs_msg, body)
|
7
6
|
yield
|
8
7
|
|
9
|
-
|
10
|
-
delete = worker.class.get_shoryuken_options['delete'] || worker.class.get_shoryuken_options['auto_delete']
|
8
|
+
auto_delete = worker.class.get_shoryuken_options['delete'] || worker.class.get_shoryuken_options['auto_delete']
|
11
9
|
|
12
|
-
Shoryuken::Client.queues(queue).batch_delete(*Array(sqs_msg)) if
|
10
|
+
Shoryuken::Client.queues(queue).batch_delete(*Array(sqs_msg)) if auto_delete
|
13
11
|
end
|
14
12
|
end
|
15
13
|
end
|
data/lib/shoryuken/processor.rb
CHANGED
@@ -10,11 +10,10 @@ module Shoryuken
|
|
10
10
|
end
|
11
11
|
|
12
12
|
def process(queue, sqs_msg)
|
13
|
-
|
14
|
-
defer do
|
15
|
-
body = get_body(worker_class, sqs_msg)
|
13
|
+
worker = Shoryuken.worker_loader.call(queue, sqs_msg)
|
16
14
|
|
17
|
-
|
15
|
+
defer do
|
16
|
+
body = get_body(worker.class, sqs_msg)
|
18
17
|
|
19
18
|
Shoryuken.server_middleware.invoke(worker, queue, sqs_msg, body) do
|
20
19
|
worker.perform(sqs_msg, body)
|
data/lib/shoryuken/version.rb
CHANGED
data/lib/shoryuken/worker.rb
CHANGED
@@ -6,19 +6,43 @@ module Shoryuken
|
|
6
6
|
|
7
7
|
module ClassMethods
|
8
8
|
def perform_async(body, options = {})
|
9
|
+
options ||= {}
|
10
|
+
options[:message_attributes] ||= {}
|
11
|
+
options[:message_attributes]['shoryuken_class'] = {
|
12
|
+
string_value: self.to_s,
|
13
|
+
data_type: 'String'
|
14
|
+
}
|
15
|
+
|
9
16
|
Shoryuken::Client.send_message(get_shoryuken_options['queue'], body, options)
|
10
17
|
end
|
11
18
|
|
19
|
+
def perform_in(interval, body, options = {})
|
20
|
+
interval = interval.to_f
|
21
|
+
now = Time.now.to_f
|
22
|
+
ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
|
23
|
+
|
24
|
+
delay = (ts - now).ceil
|
25
|
+
|
26
|
+
raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
|
27
|
+
|
28
|
+
perform_async(body, options.merge(delay_seconds: delay))
|
29
|
+
end
|
30
|
+
|
31
|
+
alias_method :perform_at, :perform_in
|
32
|
+
|
12
33
|
def shoryuken_options(opts = {})
|
13
34
|
@shoryuken_options = get_shoryuken_options.merge(stringify_keys(Hash(opts)))
|
14
35
|
queue = @shoryuken_options['queue']
|
15
|
-
|
36
|
+
if queue.respond_to? :call
|
37
|
+
queue = queue.call
|
38
|
+
@shoryuken_options['queue'] = queue
|
39
|
+
end
|
16
40
|
|
17
41
|
Shoryuken.register_worker(queue, self)
|
18
42
|
end
|
19
43
|
|
20
44
|
def get_shoryuken_options # :nodoc:
|
21
|
-
@shoryuken_options || { 'queue' => 'default', 'delete' => false, 'batch' => false }
|
45
|
+
@shoryuken_options || { 'queue' => 'default', 'delete' => false, 'auto_delete' => false, 'batch' => false }
|
22
46
|
end
|
23
47
|
|
24
48
|
def stringify_keys(hash) # :nodoc:
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
class WorkerLoader
|
3
|
+
class << self
|
4
|
+
def call(queue, sqs_msg)
|
5
|
+
# missing `try?` - yes, I'm
|
6
|
+
worker_class = !sqs_msg.is_a?(Array) &&
|
7
|
+
sqs_msg.message_attributes &&
|
8
|
+
sqs_msg.message_attributes['shoryuken_class'] &&
|
9
|
+
sqs_msg.message_attributes['shoryuken_class'][:string_value]
|
10
|
+
|
11
|
+
worker_class = (worker_class.constantize rescue nil) || Shoryuken.workers[queue]
|
12
|
+
|
13
|
+
worker_class.new
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/shoryuken.gemspec
CHANGED
@@ -22,6 +22,6 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.add_development_dependency "rspec"
|
23
23
|
spec.add_development_dependency "pry-byebug"
|
24
24
|
|
25
|
-
spec.add_dependency "aws-sdk"
|
26
|
-
spec.add_dependency "celluloid"
|
25
|
+
spec.add_dependency "aws-sdk-v1"
|
26
|
+
spec.add_dependency "celluloid", "~> 0.15.2"
|
27
27
|
end
|
@@ -9,19 +9,45 @@ describe Shoryuken::Launcher do
|
|
9
9
|
Shoryuken.options[:aws][:receive_message] ||= {}
|
10
10
|
Shoryuken.options[:aws][:receive_message][:wait_time_seconds] = 5
|
11
11
|
|
12
|
+
Shoryuken.queues << 'shoryuken'
|
13
|
+
Shoryuken.queues << 'shoryuken_command'
|
14
|
+
|
15
|
+
Shoryuken.register_worker 'shoryuken', StandardWorker
|
16
|
+
Shoryuken.register_worker 'shoryuken_command', CommandWorker
|
17
|
+
|
12
18
|
subject.run
|
13
19
|
|
14
|
-
|
20
|
+
StandardWorker.received_messages = 0
|
15
21
|
end
|
16
22
|
|
17
23
|
after { subject.stop }
|
18
24
|
|
19
|
-
class
|
25
|
+
class CommandWorker
|
26
|
+
include Shoryuken::Worker
|
27
|
+
|
28
|
+
@@received_messages = 0
|
29
|
+
|
30
|
+
shoryuken_options queue: 'shoryuken_command', auto_delete: true
|
31
|
+
|
32
|
+
def perform(sqs_msg, body)
|
33
|
+
@@received_messages = Array(sqs_msg).size
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.received_messages
|
37
|
+
@@received_messages
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.received_messages=(received_messages)
|
41
|
+
@@received_messages = received_messages
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
class StandardWorker
|
20
46
|
include Shoryuken::Worker
|
21
47
|
|
22
48
|
@@received_messages = 0
|
23
49
|
|
24
|
-
shoryuken_options queue: 'shoryuken',
|
50
|
+
shoryuken_options queue: 'shoryuken', auto_delete: true
|
25
51
|
|
26
52
|
def perform(sqs_msg, body)
|
27
53
|
@@received_messages = Array(sqs_msg).size
|
@@ -36,31 +62,42 @@ describe Shoryuken::Launcher do
|
|
36
62
|
end
|
37
63
|
end
|
38
64
|
|
65
|
+
it 'consumes as a command worker' do
|
66
|
+
CommandWorker.perform_async('Yo')
|
67
|
+
|
68
|
+
10.times do
|
69
|
+
break if CommandWorker.received_messages > 0
|
70
|
+
sleep 1
|
71
|
+
end
|
72
|
+
|
73
|
+
expect(CommandWorker.received_messages).to eq 1
|
74
|
+
end
|
75
|
+
|
39
76
|
it 'consumes a message' do
|
40
|
-
|
77
|
+
StandardWorker.get_shoryuken_options['batch'] = false
|
41
78
|
|
42
79
|
Shoryuken::Client.queues('shoryuken').send_message('Yo')
|
43
80
|
|
44
81
|
10.times do
|
45
|
-
break if
|
46
|
-
sleep
|
82
|
+
break if StandardWorker.received_messages > 0
|
83
|
+
sleep 1
|
47
84
|
end
|
48
85
|
|
49
|
-
expect(
|
86
|
+
expect(StandardWorker.received_messages).to eq 1
|
50
87
|
end
|
51
88
|
|
52
89
|
it 'consumes a batch' do
|
53
|
-
|
90
|
+
StandardWorker.get_shoryuken_options['batch'] = true
|
54
91
|
|
55
92
|
Shoryuken::Client.queues('shoryuken').batch_send *(['Yo'] * 10)
|
56
93
|
|
57
94
|
10.times do
|
58
|
-
break if
|
59
|
-
sleep
|
95
|
+
break if StandardWorker.received_messages > 0
|
96
|
+
sleep 1
|
60
97
|
end
|
61
98
|
|
62
99
|
# the fetch result is uncertain, should be greater than 1, but hard to tell the exact size
|
63
|
-
expect(
|
100
|
+
expect(StandardWorker.received_messages).to be > 1
|
64
101
|
end
|
65
102
|
end
|
66
103
|
end
|
@@ -33,6 +33,22 @@ describe Shoryuken::Client do
|
|
33
33
|
|
34
34
|
described_class.send_message(queue, 'test2', delay_seconds: 60)
|
35
35
|
end
|
36
|
+
|
37
|
+
it 'parsers as JSON by default' do
|
38
|
+
msg = { field: 'test', other_field: 'other' }
|
39
|
+
|
40
|
+
expect(sqs_queue).to receive(:send_message).with(JSON.dump(msg), {})
|
41
|
+
|
42
|
+
described_class.send_message(queue, msg)
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'parsers as JSON by default and keep the options' do
|
46
|
+
msg = { field: 'test', other_field: 'other' }
|
47
|
+
|
48
|
+
expect(sqs_queue).to receive(:send_message).with(JSON.dump(msg), { delay_seconds: 60 })
|
49
|
+
|
50
|
+
described_class.send_message(queue, msg, delay_seconds: 60)
|
51
|
+
end
|
36
52
|
end
|
37
53
|
|
38
54
|
describe '.visibility_timeout' do
|
@@ -18,7 +18,7 @@ describe Shoryuken::Fetcher do
|
|
18
18
|
|
19
19
|
describe '#fetch' do
|
20
20
|
it 'calls pause when no message' do
|
21
|
-
allow(sqs_queue).to receive(:receive_message).with(limit: 1).and_return([])
|
21
|
+
allow(sqs_queue).to receive(:receive_message).with(limit: 1, message_attribute_names: ['shoryuken_class']).and_return([])
|
22
22
|
|
23
23
|
expect(manager).to receive(:pause_queue!).with(queue)
|
24
24
|
expect(manager).to receive(:dispatch)
|
@@ -27,7 +27,7 @@ describe Shoryuken::Fetcher do
|
|
27
27
|
end
|
28
28
|
|
29
29
|
it 'assigns messages' do
|
30
|
-
allow(sqs_queue).to receive(:receive_message).with(limit: 5).and_return(sqs_msg)
|
30
|
+
allow(sqs_queue).to receive(:receive_message).with(limit: 5, message_attribute_names: ['shoryuken_class']).and_return(sqs_msg)
|
31
31
|
|
32
32
|
expect(manager).to receive(:rebalance_queue_weight!).with(queue)
|
33
33
|
expect(manager).to receive(:assign).with(queue, sqs_msg)
|
@@ -39,7 +39,7 @@ describe Shoryuken::Fetcher do
|
|
39
39
|
it 'assigns messages in batch' do
|
40
40
|
TestWorker.get_shoryuken_options['batch'] = true
|
41
41
|
|
42
|
-
allow(sqs_queue).to receive(:receive_message).with(limit: described_class::FETCH_LIMIT).and_return(sqs_msg)
|
42
|
+
allow(sqs_queue).to receive(:receive_message).with(limit: described_class::FETCH_LIMIT, message_attribute_names: ['shoryuken_class']).and_return(sqs_msg)
|
43
43
|
|
44
44
|
expect(manager).to receive(:rebalance_queue_weight!).with(queue)
|
45
45
|
expect(manager).to receive(:assign).with(queue, [sqs_msg])
|
@@ -47,5 +47,19 @@ describe Shoryuken::Fetcher do
|
|
47
47
|
|
48
48
|
subject.fetch(queue, 5)
|
49
49
|
end
|
50
|
+
|
51
|
+
context 'when worker not found' do
|
52
|
+
let(:queue) { 'notfound' }
|
53
|
+
|
54
|
+
it 'ignores batch' do
|
55
|
+
allow(sqs_queue).to receive(:receive_message).with(limit: 5, message_attribute_names: ['shoryuken_class']).and_return(sqs_msg)
|
56
|
+
|
57
|
+
expect(manager).to receive(:rebalance_queue_weight!).with(queue)
|
58
|
+
expect(manager).to receive(:assign).with(queue, sqs_msg)
|
59
|
+
expect(manager).to receive(:dispatch)
|
60
|
+
|
61
|
+
subject.fetch(queue, 5)
|
62
|
+
end
|
63
|
+
end
|
50
64
|
end
|
51
65
|
end
|
@@ -71,4 +71,37 @@ describe Shoryuken::Manager do
|
|
71
71
|
expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
|
72
72
|
end
|
73
73
|
end
|
74
|
+
|
75
|
+
describe '#next_queue' do
|
76
|
+
it 'returns queues' do
|
77
|
+
queue1 = 'shoryuken'
|
78
|
+
queue2 = 'uppercut'
|
79
|
+
|
80
|
+
Shoryuken.queues.clear
|
81
|
+
|
82
|
+
Shoryuken.register_worker queue1, TestWorker
|
83
|
+
Shoryuken.register_worker queue2, TestWorker
|
84
|
+
|
85
|
+
Shoryuken.queues << queue1
|
86
|
+
Shoryuken.queues << queue2
|
87
|
+
|
88
|
+
expect(subject.send :next_queue).to eq queue1
|
89
|
+
expect(subject.send :next_queue).to eq queue2
|
90
|
+
end
|
91
|
+
|
92
|
+
it 'skips when no worker' do
|
93
|
+
queue1 = 'shoryuken'
|
94
|
+
queue2 = 'uppercut'
|
95
|
+
|
96
|
+
Shoryuken.queues.clear
|
97
|
+
|
98
|
+
Shoryuken.register_worker queue2, TestWorker
|
99
|
+
|
100
|
+
Shoryuken.queues << queue1
|
101
|
+
Shoryuken.queues << queue2
|
102
|
+
|
103
|
+
expect(subject.send :next_queue).to eq queue2
|
104
|
+
expect(subject.send :next_queue).to eq queue2
|
105
|
+
end
|
106
|
+
end
|
74
107
|
end
|
@@ -10,7 +10,7 @@ describe Shoryuken::Middleware::Server::AutoDelete do
|
|
10
10
|
end
|
11
11
|
|
12
12
|
it 'deletes a message' do
|
13
|
-
TestWorker.get_shoryuken_options['
|
13
|
+
TestWorker.get_shoryuken_options['auto_delete'] = true
|
14
14
|
|
15
15
|
expect(sqs_queue).to receive(:batch_delete).with(sqs_msg)
|
16
16
|
|
@@ -18,7 +18,7 @@ describe Shoryuken::Middleware::Server::AutoDelete do
|
|
18
18
|
end
|
19
19
|
|
20
20
|
it 'deletes a batch' do
|
21
|
-
TestWorker.get_shoryuken_options['
|
21
|
+
TestWorker.get_shoryuken_options['auto_delete'] = true
|
22
22
|
|
23
23
|
sqs_msg2 = double 'SQS msg', body: 'test'
|
24
24
|
sqs_msg3 = double 'SQS msg', body: 'test'
|
@@ -31,7 +31,7 @@ describe Shoryuken::Middleware::Server::AutoDelete do
|
|
31
31
|
end
|
32
32
|
|
33
33
|
it 'does not delete a message' do
|
34
|
-
TestWorker.get_shoryuken_options['
|
34
|
+
TestWorker.get_shoryuken_options['auto_delete'] = false
|
35
35
|
|
36
36
|
expect(sqs_queue).to_not receive(:batch_delete)
|
37
37
|
|
@@ -6,7 +6,7 @@ describe Shoryuken::Processor do
|
|
6
6
|
let(:manager) { double Shoryuken::Manager, processor_done: nil }
|
7
7
|
let(:sqs_queue) { double AWS::SQS::Queue, visibility_timeout: 30 }
|
8
8
|
let(:queue) { 'default' }
|
9
|
-
let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test' }
|
9
|
+
let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test', message_attributes: {} }
|
10
10
|
|
11
11
|
subject { described_class.new(manager) }
|
12
12
|
|
@@ -16,7 +16,7 @@ describe Shoryuken::Processor do
|
|
16
16
|
end
|
17
17
|
|
18
18
|
describe '#process' do
|
19
|
-
it '
|
19
|
+
it 'parsers the body into JSON' do
|
20
20
|
TestWorker.get_shoryuken_options['body_parser'] = :json
|
21
21
|
|
22
22
|
body = { 'test' => 'hi' }
|
@@ -28,7 +28,7 @@ describe Shoryuken::Processor do
|
|
28
28
|
subject.process(queue, sqs_msg)
|
29
29
|
end
|
30
30
|
|
31
|
-
it '
|
31
|
+
it 'parsers the body calling the proc' do
|
32
32
|
TestWorker.get_shoryuken_options['body_parser'] = Proc.new { |sqs_msg| "*#{sqs_msg.body}*" }
|
33
33
|
|
34
34
|
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, '*test*')
|
@@ -38,7 +38,7 @@ describe Shoryuken::Processor do
|
|
38
38
|
subject.process(queue, sqs_msg)
|
39
39
|
end
|
40
40
|
|
41
|
-
it '
|
41
|
+
it 'parsers the body as text' do
|
42
42
|
TestWorker.get_shoryuken_options['body_parser'] = :text
|
43
43
|
|
44
44
|
body = 'test'
|
@@ -50,7 +50,7 @@ describe Shoryuken::Processor do
|
|
50
50
|
subject.process(queue, sqs_msg)
|
51
51
|
end
|
52
52
|
|
53
|
-
it '
|
53
|
+
it 'parsers calling `.parse`' do
|
54
54
|
TestWorker.get_shoryuken_options['body_parser'] = JSON
|
55
55
|
|
56
56
|
body = { 'test' => 'hi' }
|
@@ -77,7 +77,7 @@ describe Shoryuken::Processor do
|
|
77
77
|
end
|
78
78
|
|
79
79
|
context 'when `object_type: nil`' do
|
80
|
-
it '
|
80
|
+
it 'parsers the body as text' do
|
81
81
|
TestWorker.get_shoryuken_options['body_parser'] = nil
|
82
82
|
|
83
83
|
body = 'test'
|
@@ -126,7 +126,7 @@ describe Shoryuken::Processor do
|
|
126
126
|
end
|
127
127
|
|
128
128
|
it 'performs with delete' do
|
129
|
-
TestWorker.get_shoryuken_options['
|
129
|
+
TestWorker.get_shoryuken_options['auto_delete'] = true
|
130
130
|
|
131
131
|
expect(manager).to receive(:processor_done).with(queue, subject)
|
132
132
|
|
@@ -138,7 +138,7 @@ describe Shoryuken::Processor do
|
|
138
138
|
end
|
139
139
|
|
140
140
|
it 'performs without delete' do
|
141
|
-
TestWorker.get_shoryuken_options['
|
141
|
+
TestWorker.get_shoryuken_options['auto_delete'] = false
|
142
142
|
|
143
143
|
expect(manager).to receive(:processor_done).with(queue, subject)
|
144
144
|
|
@@ -148,5 +148,26 @@ describe Shoryuken::Processor do
|
|
148
148
|
|
149
149
|
subject.process(queue, sqs_msg)
|
150
150
|
end
|
151
|
+
|
152
|
+
context 'when shoryuken_class header' do
|
153
|
+
let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test', message_attributes: {
|
154
|
+
'shoryuken_class' => {
|
155
|
+
string_value: TestWorker.to_s,
|
156
|
+
data_type: 'String'
|
157
|
+
}
|
158
|
+
} }
|
159
|
+
|
160
|
+
it 'performs without delete' do
|
161
|
+
Shoryuken.workers.clear # unregister TestWorker
|
162
|
+
|
163
|
+
expect(manager).to receive(:processor_done).with(queue, subject)
|
164
|
+
|
165
|
+
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
|
166
|
+
|
167
|
+
expect(sqs_queue).to_not receive(:batch_delete)
|
168
|
+
|
169
|
+
subject.process(queue, sqs_msg)
|
170
|
+
end
|
171
|
+
end
|
151
172
|
end
|
152
173
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Shoryuken::WorkerLoader do
|
4
|
+
let(:queue) { 'default' }
|
5
|
+
let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test', message_attributes: { } }
|
6
|
+
|
7
|
+
describe '.call' do
|
8
|
+
it 'returns the worker using `Shoryuken.workers`' do
|
9
|
+
expect(described_class.call(queue, sqs_msg)).to be_an_instance_of TestWorker
|
10
|
+
end
|
11
|
+
|
12
|
+
context 'when `message_attributes`' do
|
13
|
+
let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', body: 'test', message_attributes: {
|
14
|
+
'shoryuken_class' => {
|
15
|
+
string_value: TestWorker.to_s,
|
16
|
+
data_type: 'String'
|
17
|
+
}
|
18
|
+
} }
|
19
|
+
|
20
|
+
it 'returns the worker using `message_attributes`' do
|
21
|
+
Shoryuken.workers.clear
|
22
|
+
|
23
|
+
expect(described_class.call(queue, sqs_msg)).to be_an_instance_of TestWorker
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -8,15 +8,74 @@ describe 'Shoryuken::Worker' do
|
|
8
8
|
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
9
9
|
end
|
10
10
|
|
11
|
+
describe '.perform_in' do
|
12
|
+
it 'delays a message' do
|
13
|
+
expect(sqs_queue).to receive(:send_message).with('message', {
|
14
|
+
message_attributes: {
|
15
|
+
'shoryuken_class' => {
|
16
|
+
string_value: TestWorker.to_s,
|
17
|
+
data_type: 'String'
|
18
|
+
}
|
19
|
+
},
|
20
|
+
delay_seconds: 60
|
21
|
+
})
|
22
|
+
|
23
|
+
TestWorker.perform_in(60, 'message')
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'raises an exception' do
|
27
|
+
expect {
|
28
|
+
TestWorker.perform_in(901, 'message')
|
29
|
+
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe '.perform_at' do
|
34
|
+
it 'delays a message' do
|
35
|
+
expect(sqs_queue).to receive(:send_message).with('message', {
|
36
|
+
message_attributes: {
|
37
|
+
'shoryuken_class' => {
|
38
|
+
string_value: TestWorker.to_s,
|
39
|
+
data_type: 'String'
|
40
|
+
}
|
41
|
+
},
|
42
|
+
delay_seconds: 60
|
43
|
+
})
|
44
|
+
|
45
|
+
TestWorker.perform_in(Time.now + 60, 'message')
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'raises an exception' do
|
49
|
+
expect {
|
50
|
+
TestWorker.perform_in(Time.now + 901, 'message')
|
51
|
+
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
11
55
|
describe '.perform_async' do
|
12
56
|
it 'enqueues a message' do
|
13
|
-
expect(sqs_queue).to receive(:send_message).with('message', {
|
57
|
+
expect(sqs_queue).to receive(:send_message).with('message', {
|
58
|
+
message_attributes: {
|
59
|
+
'shoryuken_class' => {
|
60
|
+
string_value: TestWorker.to_s,
|
61
|
+
data_type: 'String'
|
62
|
+
}
|
63
|
+
}
|
64
|
+
})
|
14
65
|
|
15
66
|
TestWorker.perform_async('message')
|
16
67
|
end
|
17
68
|
|
18
69
|
it 'enqueues a message with options' do
|
19
|
-
expect(sqs_queue).to receive(:send_message).with('delayed message',
|
70
|
+
expect(sqs_queue).to receive(:send_message).with('delayed message', {
|
71
|
+
delay_seconds: 60,
|
72
|
+
message_attributes: {
|
73
|
+
'shoryuken_class' => {
|
74
|
+
string_value: TestWorker.to_s,
|
75
|
+
data_type: 'String'
|
76
|
+
}
|
77
|
+
}
|
78
|
+
})
|
20
79
|
|
21
80
|
TestWorker.perform_async('delayed message', delay_seconds: 60)
|
22
81
|
end
|
@@ -37,6 +96,7 @@ describe 'Shoryuken::Worker' do
|
|
37
96
|
end
|
38
97
|
|
39
98
|
expect(Shoryuken.workers['production_default']).to eq NewTestWorker
|
99
|
+
expect(NewTestWorker.get_shoryuken_options['queue']).to eq 'production_default'
|
40
100
|
end
|
41
101
|
end
|
42
102
|
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Shoryuken do
|
4
|
+
describe '.register_worker' do
|
5
|
+
it 'registers a worker' do
|
6
|
+
described_class.workers.clear
|
7
|
+
described_class.register_worker('default', TestWorker)
|
8
|
+
expect(described_class.workers).to eq('default' => TestWorker)
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'registers a batchable worker' do
|
12
|
+
described_class.workers.clear
|
13
|
+
TestWorker.get_shoryuken_options['batch'] = true
|
14
|
+
described_class.register_worker('default', TestWorker)
|
15
|
+
expect(described_class.workers).to eq('default' => TestWorker)
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'allows multiple workers' do
|
19
|
+
described_class.workers.clear
|
20
|
+
described_class.register_worker('default', TestWorker)
|
21
|
+
expect(described_class.workers).to eq('default' => TestWorker)
|
22
|
+
|
23
|
+
class Test2Worker
|
24
|
+
include Shoryuken::Worker
|
25
|
+
|
26
|
+
shoryuken_options queue: 'default'
|
27
|
+
|
28
|
+
def perform(sqs_msg, body); end
|
29
|
+
end
|
30
|
+
|
31
|
+
expect(described_class.workers).to eq('default' => Test2Worker)
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'raises an exception when mixing batchable with non batchable' do
|
35
|
+
described_class.workers.clear
|
36
|
+
TestWorker.get_shoryuken_options['batch'] = true
|
37
|
+
described_class.register_worker('default', TestWorker)
|
38
|
+
|
39
|
+
expect {
|
40
|
+
class BatchableWorker
|
41
|
+
include Shoryuken::Worker
|
42
|
+
|
43
|
+
shoryuken_options queue: 'default', batch: true
|
44
|
+
|
45
|
+
def perform(sqs_msg, body); end
|
46
|
+
end
|
47
|
+
}.to raise_error("Could not register BatchableWorker for 'default', because TestWorker is already registered for this queue, " \
|
48
|
+
"and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -4,6 +4,7 @@ Bundler.setup
|
|
4
4
|
require 'pry-byebug'
|
5
5
|
require 'celluloid'
|
6
6
|
require 'shoryuken'
|
7
|
+
require 'json'
|
7
8
|
|
8
9
|
options_file = File.join(File.expand_path('../..', __FILE__), 'shoryuken.yml')
|
9
10
|
|
@@ -40,7 +41,7 @@ RSpec.configure do |config|
|
|
40
41
|
Shoryuken.options.merge!($options)
|
41
42
|
|
42
43
|
Shoryuken.queues.clear
|
43
|
-
|
44
|
+
|
44
45
|
Shoryuken.options[:concurrency] = 1
|
45
46
|
Shoryuken.options[:delay] = 1
|
46
47
|
Shoryuken.options[:timeout] = 1
|
@@ -49,5 +50,8 @@ RSpec.configure do |config|
|
|
49
50
|
|
50
51
|
TestWorker.get_shoryuken_options.clear
|
51
52
|
TestWorker.get_shoryuken_options['queue'] = 'default'
|
53
|
+
|
54
|
+
Shoryuken.workers.clear
|
55
|
+
Shoryuken.register_worker('default', TestWorker)
|
52
56
|
end
|
53
57
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-11-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -67,7 +67,7 @@ dependencies:
|
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
-
name: aws-sdk
|
70
|
+
name: aws-sdk-v1
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - '>='
|
@@ -84,16 +84,16 @@ dependencies:
|
|
84
84
|
name: celluloid
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - ~>
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
89
|
+
version: 0.15.2
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- -
|
94
|
+
- - ~>
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
96
|
+
version: 0.15.2
|
97
97
|
description: Shoryuken is a super efficient AWS SQS thread based message processor
|
98
98
|
email:
|
99
99
|
- pablo@pablocantero.com
|
@@ -110,10 +110,8 @@ files:
|
|
110
110
|
- README.md
|
111
111
|
- Rakefile
|
112
112
|
- bin/shoryuken
|
113
|
-
- examples/
|
113
|
+
- examples/bootstrap_queues.rb
|
114
114
|
- examples/default_worker.rb
|
115
|
-
- examples/high_priority_worker.rb
|
116
|
-
- examples/low_priority_worker.rb
|
117
115
|
- lib/shoryuken.rb
|
118
116
|
- lib/shoryuken/cli.rb
|
119
117
|
- lib/shoryuken/client.rb
|
@@ -123,12 +121,14 @@ files:
|
|
123
121
|
- lib/shoryuken/logging.rb
|
124
122
|
- lib/shoryuken/manager.rb
|
125
123
|
- lib/shoryuken/middleware/chain.rb
|
124
|
+
- lib/shoryuken/middleware/server/active_record.rb
|
126
125
|
- lib/shoryuken/middleware/server/auto_delete.rb
|
127
126
|
- lib/shoryuken/middleware/server/timing.rb
|
128
127
|
- lib/shoryuken/processor.rb
|
129
128
|
- lib/shoryuken/util.rb
|
130
129
|
- lib/shoryuken/version.rb
|
131
130
|
- lib/shoryuken/worker.rb
|
131
|
+
- lib/shoryuken/worker_loader.rb
|
132
132
|
- shoryuken.gemspec
|
133
133
|
- shoryuken.jpg
|
134
134
|
- spec/integration/launcher_spec.rb
|
@@ -141,7 +141,9 @@ files:
|
|
141
141
|
- spec/shoryuken/middleware/server/timing_spec.rb
|
142
142
|
- spec/shoryuken/processor_spec.rb
|
143
143
|
- spec/shoryuken/util_spec.rb
|
144
|
+
- spec/shoryuken/worker_loader_spec.rb
|
144
145
|
- spec/shoryuken/worker_spec.rb
|
146
|
+
- spec/shoryuken_spec.rb
|
145
147
|
- spec/spec_helper.rb
|
146
148
|
homepage: https://github.com/phstc/shoryuken
|
147
149
|
licenses:
|
@@ -178,5 +180,7 @@ test_files:
|
|
178
180
|
- spec/shoryuken/middleware/server/timing_spec.rb
|
179
181
|
- spec/shoryuken/processor_spec.rb
|
180
182
|
- spec/shoryuken/util_spec.rb
|
183
|
+
- spec/shoryuken/worker_loader_spec.rb
|
181
184
|
- spec/shoryuken/worker_spec.rb
|
185
|
+
- spec/shoryuken_spec.rb
|
182
186
|
- spec/spec_helper.rb
|
data/examples/all.rb
DELETED
@@ -1,11 +0,0 @@
|
|
1
|
-
class LowPriorityWorker
|
2
|
-
include Shoryuken::Worker
|
3
|
-
|
4
|
-
shoryuken_options queue: 'low_priority', delete: true, batch: true
|
5
|
-
|
6
|
-
def perform(sqs_msgs, bodies)
|
7
|
-
bodies.each_with_index do |body, index|
|
8
|
-
puts "LowPriorityWorker (#{index}): '#{body}'"
|
9
|
-
end
|
10
|
-
end
|
11
|
-
end
|