shoryuken 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +16 -0
- data/README.md +34 -13
- data/Rakefile +2 -0
- data/bin/shoryuken +1 -2
- data/examples/shoryuken_worker.rb +1 -1
- data/examples/sidekiq_worker.rb +1 -1
- data/examples/uppercut_worker.rb +1 -1
- data/lib/shoryuken/cli.rb +55 -23
- data/lib/shoryuken/client.rb +14 -13
- data/lib/shoryuken/fetcher.rb +32 -11
- data/lib/shoryuken/manager.rb +6 -18
- data/lib/shoryuken/middleware/server/delete.rb +17 -0
- data/lib/shoryuken/middleware/server/logging.rb +13 -4
- data/lib/shoryuken/processor.rb +0 -8
- data/lib/shoryuken/util.rb +7 -0
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +1 -2
- data/lib/shoryuken.rb +15 -12
- data/spec/integration/launcher_spec.rb +58 -0
- data/spec/shoryuken/client_spec.rb +15 -5
- data/spec/shoryuken/fetcher_spec.rb +25 -2
- data/spec/shoryuken/manager_spec.rb +1 -0
- data/spec/shoryuken/{chain_spec.rb → middleware/chain_spec.rb} +0 -0
- data/spec/shoryuken/middleware/server/delete_spec.rb +52 -0
- data/spec/shoryuken/processor_spec.rb +6 -5
- data/spec/shoryuken/util_spec.rb +17 -0
- data/spec/spec_helper.rb +3 -1
- metadata +12 -7
- data/lib/shoryuken/middleware/server/auto_delete.rb +0 -14
- data/spec/shoryuken/integration/launcher_spec.rb +0 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7391a170e4f76601238417e61b88861951bc6c73
|
4
|
+
data.tar.gz: 8983d19ef10bea4723533a12b23b74ca55562baf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 87a30b0002f3eaf8a62fd9f76e938fb57a5fcecc829c6a3cf8fdb7c4d1ab3388001fc460b92d59d3ae2762b441f447141bc92c136dbf6fb8730820c101df2dc0
|
7
|
+
data.tar.gz: 79f4d02f0dac764f23d07f8d27711c70b0cb97543cd410c2436300ff3e633a625de290caedf3746366b77d6c9b737c0c0138d000e2d610dab715267a40e74bb3
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -4,6 +4,8 @@
|
|
4
4
|
|
5
5
|
Shoryuken _sho-ryu-ken_ is a super efficient [AWS SQS](https://aws.amazon.com/sqs/) thread based message processor.
|
6
6
|
|
7
|
+
[](https://travis-ci.org/phstc/shoryuken)
|
8
|
+
|
7
9
|
## Key features
|
8
10
|
|
9
11
|
### Load balancing
|
@@ -34,10 +36,6 @@ If all queues get empty, all processors will be changed to the waiting state and
|
|
34
36
|
|
35
37
|
To be even more performance and cost efficient, Shoryuken fetches SQS messages in batches.
|
36
38
|
|
37
|
-
## Resque compatible?
|
38
|
-
|
39
|
-
Shoryuken isn't Resque compatible, it passes the [original SQS message](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SQS/ReceivedMessage.html) to the workers.
|
40
|
-
|
41
39
|
## Installation
|
42
40
|
|
43
41
|
Add this line to your application's Gemfile:
|
@@ -60,8 +58,8 @@ Or install it yourself as:
|
|
60
58
|
class HelloWorker
|
61
59
|
include Shoryuken::Worker
|
62
60
|
|
63
|
-
shoryuken_options queue: 'hello',
|
64
|
-
# shoryuken_options queue: ->{ "#{ENV['environment']_hello" },
|
61
|
+
shoryuken_options queue: 'hello', delete: true
|
62
|
+
# shoryuken_options queue: ->{ "#{ENV['environment']_hello" }, delete: true
|
65
63
|
|
66
64
|
def perform(sqs_msg)
|
67
65
|
puts "Hello #{sqs_msg.body}"
|
@@ -81,16 +79,16 @@ Sample configuration file `shoryuken.yml`.
|
|
81
79
|
|
82
80
|
```yaml
|
83
81
|
aws:
|
84
|
-
access_key_id: ...
|
85
|
-
secret_access_key: ...
|
86
|
-
region: us-east-1
|
87
|
-
receive_message:
|
82
|
+
access_key_id: ... # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
|
83
|
+
secret_access_key: ... # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
|
84
|
+
region: us-east-1 # or <%= ENV['AWS_REGION'] %>
|
85
|
+
receive_message: # See http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SQS/Queue.html#receive_message-instance_method
|
86
|
+
# wait_time_seconds: N # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
|
88
87
|
attributes:
|
89
88
|
- receive_count
|
90
89
|
- sent_at
|
91
|
-
concurrency: 25,
|
92
|
-
delay: 25,
|
93
|
-
timeout: 8
|
90
|
+
concurrency: 25, # The number of allocated threads to process messages. Default 25
|
91
|
+
delay: 25, # The delay in seconds to pause a queue when it's empty. Default 0
|
94
92
|
queues:
|
95
93
|
- [shoryuken, 6]
|
96
94
|
- [uppercut, 2]
|
@@ -103,6 +101,25 @@ queues:
|
|
103
101
|
bundle exec shoryuken -r worker.rb -C shoryuken.yml
|
104
102
|
```
|
105
103
|
|
104
|
+
Other options:
|
105
|
+
|
106
|
+
```bash
|
107
|
+
shoryuken --help
|
108
|
+
|
109
|
+
shoryuken [options]
|
110
|
+
-c, --concurrency INT Processor threads to use
|
111
|
+
-d, --daemon Daemonize process
|
112
|
+
-q, --queue QUEUE[,WEIGHT]... Queues to process with optional weights
|
113
|
+
-r, --require [PATH|DIR] Location of the worker
|
114
|
+
-C, --config PATH Path to YAML config file
|
115
|
+
-L, --logfile PATH Path to writable logfile
|
116
|
+
-P, --pidfile PATH Path to pidfile
|
117
|
+
-v, --verbose Print more verbose output
|
118
|
+
-V, --version Print version and exit
|
119
|
+
-h, --help Show help
|
120
|
+
...
|
121
|
+
```
|
122
|
+
|
106
123
|
### Middleware
|
107
124
|
|
108
125
|
```ruby
|
@@ -122,6 +139,10 @@ Shoryuken.configure_server do |config|
|
|
122
139
|
end
|
123
140
|
```
|
124
141
|
|
142
|
+
## More Information
|
143
|
+
|
144
|
+
Please check the [Shoryuken Wiki](https://github.com/phstc/shoryuken/wiki).
|
145
|
+
|
125
146
|
## Credits
|
126
147
|
|
127
148
|
[Mike Perham](https://github.com/mperham), creator of [Sidekiq](https://github.com/mperham/sidekiq), and [everybody who contributed to it](https://github.com/mperham/sidekiq/graphs/contributors). Shoryuken wouldn't exist as it is without those contributions.
|
data/Rakefile
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'bundler/gem_tasks'
|
2
2
|
$stdout.sync = true
|
3
3
|
|
4
|
+
desc 'Open Shoryuken pry console'
|
4
5
|
task :console do
|
5
6
|
require 'pry'
|
6
7
|
require 'shoryuken'
|
@@ -21,6 +22,7 @@ task :console do
|
|
21
22
|
Pry.start
|
22
23
|
end
|
23
24
|
|
25
|
+
desc 'Push test messages to shoryuken, uppercut and sidekiq'
|
24
26
|
task :push_test, :size do |t, args|
|
25
27
|
require 'yaml'
|
26
28
|
require 'shoryuken'
|
data/bin/shoryuken
CHANGED
data/examples/sidekiq_worker.rb
CHANGED
data/examples/uppercut_worker.rb
CHANGED
data/lib/shoryuken/cli.rb
CHANGED
@@ -4,6 +4,8 @@ require 'singleton'
|
|
4
4
|
require 'optparse'
|
5
5
|
require 'erb'
|
6
6
|
|
7
|
+
require 'shoryuken'
|
8
|
+
|
7
9
|
module Shoryuken
|
8
10
|
class CLI
|
9
11
|
include Util
|
@@ -24,10 +26,11 @@ module Shoryuken
|
|
24
26
|
initialize_logger
|
25
27
|
validate!
|
26
28
|
daemonize
|
27
|
-
initialize_aws
|
28
|
-
require_workers
|
29
29
|
write_pid
|
30
|
+
load_celluloid
|
31
|
+
require_workers
|
30
32
|
|
33
|
+
require 'shoryuken/launcher'
|
31
34
|
@launcher = Shoryuken::Launcher.new
|
32
35
|
|
33
36
|
begin
|
@@ -45,6 +48,18 @@ module Shoryuken
|
|
45
48
|
|
46
49
|
private
|
47
50
|
|
51
|
+
def load_celluloid
|
52
|
+
raise "Celluloid cannot be required until here, or it will break Shoryuken's daemonization" if defined?(::Celluloid) && Shoryuken.options[:daemon]
|
53
|
+
|
54
|
+
# Celluloid can't be loaded until after we've daemonized
|
55
|
+
# because it spins up threads and creates locks which get
|
56
|
+
# into a very bad state if forked.
|
57
|
+
require 'celluloid/autostart'
|
58
|
+
Celluloid.logger = (Shoryuken.options[:verbose] ? Shoryuken.logger : nil)
|
59
|
+
|
60
|
+
require 'shoryuken/manager'
|
61
|
+
end
|
62
|
+
|
48
63
|
def daemonize
|
49
64
|
return unless Shoryuken.options[:daemon]
|
50
65
|
|
@@ -88,7 +103,7 @@ module Shoryuken
|
|
88
103
|
opts = {}
|
89
104
|
|
90
105
|
@parser = OptionParser.new do |o|
|
91
|
-
o.on '-c', '--concurrency INT',
|
106
|
+
o.on '-c', '--concurrency INT', 'Processor threads to use' do |arg|
|
92
107
|
opts[:concurrency] = Integer(arg)
|
93
108
|
end
|
94
109
|
|
@@ -97,23 +112,23 @@ module Shoryuken
|
|
97
112
|
end
|
98
113
|
|
99
114
|
o.on '-q', '--queue QUEUE[,WEIGHT]...', 'Queues to process with optional weights' do |arg|
|
100
|
-
|
101
|
-
|
115
|
+
queue, weight = arg.split(',')
|
116
|
+
parse_queue queue, weight
|
102
117
|
end
|
103
118
|
|
104
119
|
o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
|
105
120
|
opts[:require] = arg
|
106
121
|
end
|
107
122
|
|
108
|
-
o.on '-C', '--config PATH', '
|
123
|
+
o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
|
109
124
|
opts[:config_file] = arg
|
110
125
|
end
|
111
126
|
|
112
|
-
o.on '-L', '--logfile PATH', '
|
127
|
+
o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
|
113
128
|
opts[:logfile] = arg
|
114
129
|
end
|
115
130
|
|
116
|
-
o.on '-P', '--pidfile PATH',
|
131
|
+
o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
|
117
132
|
opts[:pidfile] = arg
|
118
133
|
end
|
119
134
|
|
@@ -156,15 +171,11 @@ module Shoryuken
|
|
156
171
|
end
|
157
172
|
end
|
158
173
|
|
159
|
-
|
160
|
-
|
174
|
+
ready = launcher.manager.instance_variable_get(:@ready).size
|
175
|
+
busy = launcher.manager.instance_variable_get(:@busy).size
|
176
|
+
queues = launcher.manager.instance_variable_get(:@queues)
|
161
177
|
|
162
|
-
|
163
|
-
weights[queue] = weights[queue].to_i + 1
|
164
|
-
weights
|
165
|
-
end.each do |queue, weight|
|
166
|
-
Shoryuken.logger.info "Current queue '#{queue}' weight: #{weight}"
|
167
|
-
end
|
178
|
+
Shoryuken.logger.info "Ready: #{ready}, Busy: #{busy}, Active Queues: #{unparse_queues(queues)}"
|
168
179
|
else
|
169
180
|
Shoryuken.logger.info "Received #{sig}, will shutdown down"
|
170
181
|
|
@@ -184,13 +195,12 @@ module Shoryuken
|
|
184
195
|
parse_queues
|
185
196
|
end
|
186
197
|
|
187
|
-
def parse_config(
|
188
|
-
|
189
|
-
|
190
|
-
|
198
|
+
def parse_config(config_file)
|
199
|
+
if File.exist?(config_file)
|
200
|
+
YAML.load(ERB.new(IO.read(config_file)).result)
|
201
|
+
else
|
202
|
+
raise ArgumentError, "Config file #{config_file} does not exist"
|
191
203
|
end
|
192
|
-
|
193
|
-
opts
|
194
204
|
end
|
195
205
|
|
196
206
|
def initialize_logger
|
@@ -200,10 +210,32 @@ module Shoryuken
|
|
200
210
|
end
|
201
211
|
|
202
212
|
def validate!
|
213
|
+
if Shoryuken.options[:aws][:access_key_id].nil? && Shoryuken.options[:aws][:secret_access_key].nil?
|
214
|
+
if ENV['AWS_ACCESS_KEY_ID'].nil? && ENV['AWS_SECRET_ACCESS_KEY'].nil?
|
215
|
+
raise ArgumentError, 'No AWS credentials supplied'
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
|
220
|
+
initialize_aws
|
221
|
+
|
222
|
+
Shoryuken.queues.uniq.each do |queue|
|
223
|
+
# validate all queues and AWS credentials consequently
|
224
|
+
begin
|
225
|
+
Shoryuken::Client.queues queue
|
226
|
+
rescue AWS::SQS::Errors::NonExistentQueue => e
|
227
|
+
raise ArgumentError, "Queue '#{queue}' does not exist"
|
228
|
+
rescue => e
|
229
|
+
raise
|
230
|
+
end
|
231
|
+
end
|
232
|
+
|
203
233
|
raise ArgumentError, 'No queues supplied' if Shoryuken.queues.empty?
|
204
234
|
end
|
205
235
|
|
206
236
|
def initialize_aws
|
237
|
+
# aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
|
238
|
+
# when not explicit supplied
|
207
239
|
AWS.config Shoryuken.options[:aws] if Shoryuken.options[:aws]
|
208
240
|
end
|
209
241
|
|
@@ -212,7 +244,7 @@ module Shoryuken
|
|
212
244
|
end
|
213
245
|
|
214
246
|
def parse_queues
|
215
|
-
Shoryuken.options[:queues].each { |queue_and_weight| parse_queue *queue_and_weight }
|
247
|
+
Shoryuken.options[:queues].to_a.each { |queue_and_weight| parse_queue *queue_and_weight }
|
216
248
|
end
|
217
249
|
|
218
250
|
def parse_queue(queue, weight = nil)
|
data/lib/shoryuken/client.rb
CHANGED
@@ -1,23 +1,24 @@
|
|
1
|
-
|
2
1
|
module Shoryuken
|
3
2
|
class Client
|
4
3
|
@@queues = {}
|
4
|
+
@@visibility_timeouts = {}
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
6
|
+
class << self
|
7
|
+
def queues(queue)
|
8
|
+
@@queues[queue.to_s] ||= sqs.queues.named(queue)
|
9
|
+
end
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
11
|
+
def visibility_timeout(queue)
|
12
|
+
@@visibility_timeouts[queue.to_s] ||= queues(queue).visibility_timeout
|
13
|
+
end
|
13
14
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
end
|
15
|
+
def receive_message(queue, options = {})
|
16
|
+
queues(queue).receive_message(Hash(options))
|
17
|
+
end
|
18
18
|
|
19
|
-
|
20
|
-
|
19
|
+
def sqs
|
20
|
+
@sqs ||= AWS::SQS.new
|
21
|
+
end
|
21
22
|
end
|
22
23
|
end
|
23
24
|
end
|
data/lib/shoryuken/fetcher.rb
CHANGED
@@ -3,13 +3,15 @@ module Shoryuken
|
|
3
3
|
include Celluloid
|
4
4
|
include Util
|
5
5
|
|
6
|
+
FETCH_LIMIT = 10
|
7
|
+
|
6
8
|
def initialize(manager)
|
7
9
|
@manager = manager
|
8
10
|
end
|
9
11
|
|
10
12
|
def receive_message(queue, limit)
|
11
13
|
# AWS limits the batch size by 10
|
12
|
-
limit = limit >
|
14
|
+
limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
|
13
15
|
|
14
16
|
Shoryuken::Client.receive_message queue, Shoryuken.options[:aws][:receive_message].to_h.merge(limit: limit)
|
15
17
|
end
|
@@ -18,32 +20,51 @@ module Shoryuken
|
|
18
20
|
watchdog('Fetcher#fetch died') do
|
19
21
|
started_at = Time.now
|
20
22
|
|
21
|
-
logger.info "Looking for new messages
|
23
|
+
logger.info "Looking for new messages '#{queue}'"
|
22
24
|
|
23
25
|
begin
|
24
|
-
|
25
|
-
|
26
|
+
batch = !!Shoryuken.workers[queue].get_shoryuken_options['batch']
|
27
|
+
|
28
|
+
limit = batch ? FETCH_LIMIT : available_processors
|
29
|
+
|
30
|
+
if (sqs_msgs = Array(receive_message(queue, limit))).any?
|
31
|
+
logger.info "Found #{sqs_msgs.size} messages for '#{queue}'"
|
26
32
|
|
27
|
-
|
28
|
-
@manager.async.
|
29
|
-
|
33
|
+
if batch
|
34
|
+
@manager.async.assign(queue, patch_sqs_msgs!(sqs_msgs))
|
35
|
+
else
|
36
|
+
sqs_msgs.each { |sqs_msg| @manager.async.assign(queue, sqs_msg) }
|
30
37
|
end
|
38
|
+
|
39
|
+
@manager.async.rebalance_queue_weight!(queue)
|
31
40
|
else
|
32
|
-
logger.info "No message found for
|
41
|
+
logger.info "No message found for '#{queue}'"
|
33
42
|
|
34
43
|
@manager.async.pause_queue!(queue)
|
35
44
|
end
|
36
45
|
|
37
46
|
@manager.async.dispatch
|
38
47
|
|
39
|
-
logger.debug "Fetcher
|
48
|
+
logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
|
40
49
|
rescue => ex
|
41
|
-
logger.error
|
42
|
-
logger.error
|
50
|
+
logger.error "Error fetching message: #{ex}"
|
51
|
+
logger.error ex.backtrace.first
|
43
52
|
|
44
53
|
@manager.async.dispatch
|
45
54
|
end
|
46
55
|
end
|
56
|
+
|
57
|
+
end
|
58
|
+
private
|
59
|
+
|
60
|
+
def patch_sqs_msgs!(sqs_msgs)
|
61
|
+
sqs_msgs.instance_eval do
|
62
|
+
def id
|
63
|
+
"batch-with-#{size}-messages"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
sqs_msgs
|
47
68
|
end
|
48
69
|
end
|
49
70
|
end
|
data/lib/shoryuken/manager.rb
CHANGED
@@ -1,9 +1,3 @@
|
|
1
|
-
require 'yaml'
|
2
|
-
require 'aws-sdk'
|
3
|
-
require 'celluloid'
|
4
|
-
|
5
|
-
require 'shoryuken/version'
|
6
|
-
require 'shoryuken/manager'
|
7
1
|
require 'shoryuken/processor'
|
8
2
|
require 'shoryuken/fetcher'
|
9
3
|
|
@@ -57,7 +51,7 @@ module Shoryuken
|
|
57
51
|
|
58
52
|
def processor_done(queue, processor)
|
59
53
|
watchdog('Manager#processor_done died') do
|
60
|
-
logger.info "Process done for
|
54
|
+
logger.info "Process done for '#{queue}'"
|
61
55
|
|
62
56
|
@busy.delete processor
|
63
57
|
|
@@ -99,11 +93,7 @@ module Shoryuken
|
|
99
93
|
def rebalance_queue_weight!(queue)
|
100
94
|
watchdog('Manager#rebalance_queue_weight! died') do
|
101
95
|
if (original = original_queue_weight(queue)) > (current = current_queue_weight(queue))
|
102
|
-
|
103
|
-
logger.info "Increasing queue '#{queue}' weight to #{current + 1}, original: #{original}"
|
104
|
-
else
|
105
|
-
logger.info "Queue '#{queue}' is back to its normal weight #{original}"
|
106
|
-
end
|
96
|
+
logger.info "Increasing '#{queue}' weight to #{current + 1}, max: #{original}"
|
107
97
|
|
108
98
|
@queues << queue
|
109
99
|
end
|
@@ -111,9 +101,9 @@ module Shoryuken
|
|
111
101
|
end
|
112
102
|
|
113
103
|
def pause_queue!(queue)
|
114
|
-
return
|
104
|
+
return if !@queues.include?(queue) || Shoryuken.options[:delay].to_f <= 0
|
115
105
|
|
116
|
-
logger.info "Pausing
|
106
|
+
logger.info "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty"
|
117
107
|
|
118
108
|
@queues.delete(queue)
|
119
109
|
|
@@ -124,9 +114,7 @@ module Shoryuken
|
|
124
114
|
def dispatch
|
125
115
|
return if stopped?
|
126
116
|
|
127
|
-
logger.debug { "Ready size: #{@
|
128
|
-
logger.debug { "Busy size: #{@busy.size}" }
|
129
|
-
logger.debug { "Queues: #{@queues.inspect}" }
|
117
|
+
logger.debug { "Ready: #{@ready.size}, Busy: #{@busy.size}, Active Queues: #{unparse_queues(@queues)}" }
|
130
118
|
|
131
119
|
if @ready.empty?
|
132
120
|
logger.debug { 'Pausing fetcher, because all processors are busy' }
|
@@ -151,7 +139,7 @@ module Shoryuken
|
|
151
139
|
return if stopped?
|
152
140
|
|
153
141
|
unless @queues.include? queue
|
154
|
-
logger.info "Restarting
|
142
|
+
logger.info "Restarting '#{queue}'"
|
155
143
|
|
156
144
|
@queues << queue
|
157
145
|
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
module Middleware
|
3
|
+
module Server
|
4
|
+
class Delete
|
5
|
+
def call(worker, queue, sqs_msg)
|
6
|
+
yield
|
7
|
+
|
8
|
+
# auto_delete is deprecated
|
9
|
+
delete = worker.class.get_shoryuken_options['delete'] || worker.class.get_shoryuken_options['auto_delete']
|
10
|
+
|
11
|
+
Array(sqs_msg).each(&:delete) if delete
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
@@ -5,15 +5,24 @@ module Shoryuken
|
|
5
5
|
include Util
|
6
6
|
|
7
7
|
def call(worker, queue, sqs_msg)
|
8
|
-
Shoryuken::Logging.with_context("#{worker.class.to_s}
|
8
|
+
Shoryuken::Logging.with_context("#{worker.class.to_s}/#{queue}/#{sqs_msg.id}") do
|
9
9
|
begin
|
10
10
|
started_at = Time.now
|
11
|
+
|
11
12
|
logger.info { "started at #{started_at}" }
|
13
|
+
|
12
14
|
yield
|
13
|
-
|
14
|
-
|
15
|
+
|
16
|
+
total_time = elapsed(started_at)
|
17
|
+
|
18
|
+
if (total_time / 1000.0) > (timeout = Shoryuken::Client.visibility_timeout(queue))
|
19
|
+
logger.warn "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms"
|
20
|
+
end
|
21
|
+
|
22
|
+
logger.info { "completed in: #{total_time} ms" }
|
23
|
+
rescue => e
|
15
24
|
logger.info { "failed in: #{elapsed(started_at)} ms" }
|
16
|
-
raise
|
25
|
+
raise e
|
17
26
|
end
|
18
27
|
end
|
19
28
|
end
|
data/lib/shoryuken/processor.rb
CHANGED
@@ -24,13 +24,5 @@ module Shoryuken
|
|
24
24
|
|
25
25
|
@manager.async.processor_done(queue, current_actor)
|
26
26
|
end
|
27
|
-
|
28
|
-
def self.default_middleware
|
29
|
-
Middleware::Chain.new do |m|
|
30
|
-
m.add Middleware::Server::Logging
|
31
|
-
m.add Middleware::Server::AutoDelete
|
32
|
-
# m.add Middleware::Server::RetryJobs
|
33
|
-
end
|
34
|
-
end
|
35
27
|
end
|
36
28
|
end
|
data/lib/shoryuken/util.rb
CHANGED
@@ -15,5 +15,12 @@ module Shoryuken
|
|
15
15
|
def elapsed(started_at)
|
16
16
|
(Time.now - started_at) * 1000
|
17
17
|
end
|
18
|
+
|
19
|
+
def unparse_queues(queues)
|
20
|
+
queues.inject({}) do |queue_and_weights, name|
|
21
|
+
queue_and_weights[name] = queue_and_weights[name].to_i + 1
|
22
|
+
queue_and_weights
|
23
|
+
end.to_a
|
24
|
+
end
|
18
25
|
end
|
19
26
|
end
|
data/lib/shoryuken/version.rb
CHANGED
data/lib/shoryuken/worker.rb
CHANGED
@@ -10,12 +10,11 @@ module Shoryuken
|
|
10
10
|
queue = @shoryuken_options['queue']
|
11
11
|
queue = queue.call if queue.respond_to? :call
|
12
12
|
|
13
|
-
|
14
13
|
Shoryuken.register_worker(queue, self)
|
15
14
|
end
|
16
15
|
|
17
16
|
def get_shoryuken_options # :nodoc:
|
18
|
-
@shoryuken_options || { 'queue' => 'default' }
|
17
|
+
@shoryuken_options || { 'queue' => 'default', 'delete' => false, 'batch' => false }
|
19
18
|
end
|
20
19
|
|
21
20
|
def stringify_keys(hash) # :nodoc:
|
data/lib/shoryuken.rb
CHANGED
@@ -1,35 +1,27 @@
|
|
1
1
|
require 'yaml'
|
2
2
|
require 'aws-sdk'
|
3
|
-
require 'celluloid'
|
4
3
|
require 'time'
|
5
4
|
|
6
5
|
require 'shoryuken/version'
|
7
6
|
require 'shoryuken/core_ext'
|
8
7
|
require 'shoryuken/util'
|
9
|
-
require 'shoryuken/manager'
|
10
|
-
require 'shoryuken/processor'
|
11
|
-
require 'shoryuken/fetcher'
|
12
8
|
require 'shoryuken/client'
|
13
9
|
require 'shoryuken/worker'
|
14
|
-
require 'shoryuken/launcher'
|
15
10
|
require 'shoryuken/logging'
|
16
11
|
require 'shoryuken/middleware/chain'
|
17
|
-
require 'shoryuken/middleware/server/
|
12
|
+
require 'shoryuken/middleware/server/delete'
|
18
13
|
require 'shoryuken/middleware/server/logging'
|
19
14
|
|
20
15
|
module Shoryuken
|
21
16
|
DEFAULTS = {
|
22
17
|
concurrency: 25,
|
23
18
|
queues: [],
|
24
|
-
|
25
|
-
delay:
|
19
|
+
aws: {},
|
20
|
+
delay: 0,
|
26
21
|
timeout: 8
|
27
22
|
}
|
28
23
|
|
29
|
-
# { 'my_queue1' => Worker1
|
30
|
-
# 'my_queue2' => Worker2 }
|
31
24
|
@@workers = {}
|
32
|
-
|
33
25
|
@@queues = []
|
34
26
|
|
35
27
|
def self.options
|
@@ -62,8 +54,19 @@ module Shoryuken
|
|
62
54
|
end
|
63
55
|
|
64
56
|
def self.server_middleware
|
65
|
-
@server_chain ||=
|
57
|
+
@server_chain ||= default_server_middleware
|
66
58
|
yield @server_chain if block_given?
|
67
59
|
@server_chain
|
68
60
|
end
|
61
|
+
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def self.default_server_middleware
|
66
|
+
Middleware::Chain.new do |m|
|
67
|
+
m.add Middleware::Server::Logging
|
68
|
+
m.add Middleware::Server::Delete
|
69
|
+
# TODO m.add Middleware::Server::RetryJobs
|
70
|
+
end
|
71
|
+
end
|
69
72
|
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'shoryuken/manager'
|
3
|
+
require 'shoryuken/launcher'
|
4
|
+
|
5
|
+
describe Shoryuken::Launcher do
|
6
|
+
describe 'Consuming messages', slow: :true do
|
7
|
+
before do
|
8
|
+
Shoryuken.options[:aws] ||= {}
|
9
|
+
Shoryuken.options[:aws][:receive_message] ||= {}
|
10
|
+
Shoryuken.options[:aws][:receive_message][:wait_time_seconds] = 5
|
11
|
+
|
12
|
+
subject.run
|
13
|
+
|
14
|
+
$received_messages = 0
|
15
|
+
end
|
16
|
+
|
17
|
+
after do
|
18
|
+
subject.stop
|
19
|
+
end
|
20
|
+
|
21
|
+
class ShoryukenWorker
|
22
|
+
include Shoryuken::Worker
|
23
|
+
|
24
|
+
shoryuken_options queue: 'shoryuken', delete: true
|
25
|
+
|
26
|
+
def perform(sqs_msg)
|
27
|
+
$received_messages = Array(sqs_msg).size
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'consumes a message' do
|
32
|
+
ShoryukenWorker.get_shoryuken_options['batch'] = false
|
33
|
+
|
34
|
+
Shoryuken::Client.queues('shoryuken').send_message('Yo')
|
35
|
+
|
36
|
+
10.times do
|
37
|
+
break if $received_messages > 0
|
38
|
+
sleep 0.2
|
39
|
+
end
|
40
|
+
|
41
|
+
expect($received_messages).to eq 1
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'consumes a batch' do
|
45
|
+
ShoryukenWorker.get_shoryuken_options['batch'] = true
|
46
|
+
|
47
|
+
Shoryuken::Client.queues('shoryuken').batch_send *(['Yo'] * 10)
|
48
|
+
|
49
|
+
10.times do
|
50
|
+
break if $received_messages > 0
|
51
|
+
sleep 0.2
|
52
|
+
end
|
53
|
+
|
54
|
+
# the fetch result is uncertain, should be greater than 1, but hard to tell the exact size
|
55
|
+
expect($received_messages).to be > 1
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -3,20 +3,30 @@ require 'spec_helper'
|
|
3
3
|
describe Shoryuken::Client do
|
4
4
|
let(:sqs) { double 'SQS' }
|
5
5
|
let(:queue_collection) { double 'Queues Collection' }
|
6
|
-
let(:
|
7
|
-
let(:
|
6
|
+
let(:sqs_queue) { double 'SQS Queue' }
|
7
|
+
let(:queue) { 'shoryuken' }
|
8
8
|
|
9
9
|
before do
|
10
10
|
allow(described_class).to receive(:sqs).and_return(sqs)
|
11
11
|
allow(sqs).to receive(:queues).and_return(queue_collection)
|
12
|
+
allow(queue_collection).to receive(:named).and_return(sqs_queue)
|
12
13
|
end
|
13
14
|
|
14
15
|
describe '.queues' do
|
15
16
|
it 'memoizes queues' do
|
16
|
-
expect(queue_collection).to receive(:named).once.with(
|
17
|
+
expect(queue_collection).to receive(:named).once.with(queue).and_return(sqs_queue)
|
17
18
|
|
18
|
-
expect(Shoryuken::Client.queues(
|
19
|
-
expect(Shoryuken::Client.queues(
|
19
|
+
expect(Shoryuken::Client.queues(queue)).to eq sqs_queue
|
20
|
+
expect(Shoryuken::Client.queues(queue)).to eq sqs_queue
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
describe '.visibility_timeout' do
|
25
|
+
it 'memoizes visibility_timeout' do
|
26
|
+
expect(sqs_queue).to receive(:visibility_timeout).once.and_return(30)
|
27
|
+
|
28
|
+
expect(Shoryuken::Client.visibility_timeout(queue)).to eq 30
|
29
|
+
expect(Shoryuken::Client.visibility_timeout(queue)).to eq 30
|
20
30
|
end
|
21
31
|
end
|
22
32
|
end
|
@@ -1,9 +1,11 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
+
require 'shoryuken/manager'
|
3
|
+
require 'shoryuken/fetcher'
|
2
4
|
|
3
5
|
describe Shoryuken::Fetcher do
|
4
6
|
let(:manager) { double Shoryuken::Manager }
|
5
7
|
let(:sqs_queue) { double 'sqs_queue' }
|
6
|
-
let(:queue) { '
|
8
|
+
let(:queue) { 'shoryuken2' }
|
7
9
|
let(:sqs_msg) { double 'SQS msg'}
|
8
10
|
|
9
11
|
subject { described_class.new(manager) }
|
@@ -13,8 +15,17 @@ describe Shoryuken::Fetcher do
|
|
13
15
|
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
14
16
|
end
|
15
17
|
|
18
|
+
|
19
|
+
class Shoryuken2Worker
|
20
|
+
include Shoryuken::Worker
|
21
|
+
|
22
|
+
shoryuken_options queue: 'shoryuken2'
|
23
|
+
|
24
|
+
def perform(sqs_msg); end
|
25
|
+
end
|
26
|
+
|
16
27
|
describe '#fetch' do
|
17
|
-
it 'calls
|
28
|
+
it 'calls pause when no message' do
|
18
29
|
allow(sqs_queue).to receive(:receive_message).with(limit: 1).and_return([])
|
19
30
|
|
20
31
|
expect(manager).to receive(:pause_queue!).with(queue)
|
@@ -32,5 +43,17 @@ describe Shoryuken::Fetcher do
|
|
32
43
|
|
33
44
|
subject.fetch(queue, 5)
|
34
45
|
end
|
46
|
+
|
47
|
+
it 'assigns messages in batch' do
|
48
|
+
Shoryuken2Worker.get_shoryuken_options['batch'] = true
|
49
|
+
|
50
|
+
allow(sqs_queue).to receive(:receive_message).with(limit: described_class::FETCH_LIMIT).and_return(sqs_msg)
|
51
|
+
|
52
|
+
expect(manager).to receive(:rebalance_queue_weight!).with(queue)
|
53
|
+
expect(manager).to receive(:assign).with(queue, [sqs_msg])
|
54
|
+
expect(manager).to receive(:dispatch)
|
55
|
+
|
56
|
+
subject.fetch(queue, 5)
|
57
|
+
end
|
35
58
|
end
|
36
59
|
end
|
File without changes
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Shoryuken::Middleware::Server::Delete do
|
4
|
+
let(:sqs_msg) { double 'SQS msg' }
|
5
|
+
|
6
|
+
before do
|
7
|
+
DeleteWorker.get_shoryuken_options['delete'] = true
|
8
|
+
end
|
9
|
+
|
10
|
+
class DeleteWorker
|
11
|
+
include Shoryuken::Worker
|
12
|
+
|
13
|
+
shoryuken_options queue: 'delete', delete: true
|
14
|
+
|
15
|
+
def perform; end
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'deletes a message' do
|
19
|
+
expect(sqs_msg).to receive(:delete)
|
20
|
+
|
21
|
+
subject.call(DeleteWorker.new, 'delete', sqs_msg) {}
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'deletes a batch' do
|
25
|
+
sqs_msg2 = double
|
26
|
+
sqs_msg3 = double
|
27
|
+
|
28
|
+
expect(sqs_msg).to receive(:delete)
|
29
|
+
expect(sqs_msg2).to receive(:delete)
|
30
|
+
expect(sqs_msg3).to receive(:delete)
|
31
|
+
|
32
|
+
subject.call(DeleteWorker.new, 'delete', [sqs_msg, sqs_msg2, sqs_msg3]) {}
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'does not delete a message' do
|
36
|
+
DeleteWorker.get_shoryuken_options['delete'] = false
|
37
|
+
|
38
|
+
expect(sqs_msg).to_not receive(:delete)
|
39
|
+
|
40
|
+
subject.call(DeleteWorker.new, 'delete', sqs_msg) {}
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'when exception' do
|
44
|
+
it 'does not delete a message' do
|
45
|
+
expect(sqs_msg).to_not receive(:delete)
|
46
|
+
|
47
|
+
expect {
|
48
|
+
subject.call(DeleteWorker.new, 'delete', sqs_msg) { raise }
|
49
|
+
}.to raise_error
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -1,8 +1,9 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
+
require 'shoryuken/processor'
|
2
3
|
|
3
4
|
describe Shoryuken::Processor do
|
4
5
|
let(:manager) { double Shoryuken::Manager }
|
5
|
-
let(:sqs_queue) { double 'Queue' }
|
6
|
+
let(:sqs_queue) { double 'Queue', visibility_timeout: 30 }
|
6
7
|
let(:queue) { 'yo' }
|
7
8
|
let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e' }
|
8
9
|
|
@@ -66,8 +67,8 @@ describe Shoryuken::Processor do
|
|
66
67
|
end
|
67
68
|
end
|
68
69
|
|
69
|
-
it 'performs with
|
70
|
-
YoWorker.get_shoryuken_options['
|
70
|
+
it 'performs with delete' do
|
71
|
+
YoWorker.get_shoryuken_options['delete'] = true
|
71
72
|
|
72
73
|
expect(manager).to receive(:processor_done).with(queue, subject)
|
73
74
|
|
@@ -78,8 +79,8 @@ describe Shoryuken::Processor do
|
|
78
79
|
subject.process(queue, sqs_msg)
|
79
80
|
end
|
80
81
|
|
81
|
-
it 'performs without
|
82
|
-
YoWorker.get_shoryuken_options['
|
82
|
+
it 'performs without delete' do
|
83
|
+
YoWorker.get_shoryuken_options['delete'] = false
|
83
84
|
|
84
85
|
expect(manager).to receive(:processor_done).with(queue, subject)
|
85
86
|
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'Shoryuken::Util' do
|
4
|
+
subject do
|
5
|
+
Class.new do
|
6
|
+
extend Shoryuken::Util
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
describe '#unparse_queues' do
|
11
|
+
it 'returns queues and weights' do
|
12
|
+
queues = %w[queue1 queue1 queue2 queue3 queue4 queue4 queue4]
|
13
|
+
|
14
|
+
expect(subject.unparse_queues(queues)).to eq([['queue1', 2], ['queue2', 1], ['queue3', 1], ['queue4', 3]])
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -3,6 +3,7 @@ Bundler.setup
|
|
3
3
|
|
4
4
|
require 'pry-byebug'
|
5
5
|
require 'shoryuken'
|
6
|
+
require 'celluloid'
|
6
7
|
|
7
8
|
options_file = File.join(File.expand_path('../..', __FILE__), 'shoryuken.yml')
|
8
9
|
|
@@ -24,7 +25,8 @@ RSpec.configure do |config|
|
|
24
25
|
# remove doubles, preventing:
|
25
26
|
# Double "Queue" was originally created in one example but has leaked into another example and can no longer be used.
|
26
27
|
# rspec-mocks' doubles are designed to only last for one example, and you need to create a new one in each example you wish to use it for.
|
27
|
-
Shoryuken::Client.
|
28
|
+
Shoryuken::Client.class_variable_set :@@queues, {}
|
29
|
+
Shoryuken::Client.class_variable_set :@@visibility_timeouts, {}
|
28
30
|
|
29
31
|
Shoryuken.options.clear
|
30
32
|
Shoryuken.options.merge!($options)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-10-
|
11
|
+
date: 2014-10-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -118,6 +118,7 @@ extra_rdoc_files: []
|
|
118
118
|
files:
|
119
119
|
- .gitignore
|
120
120
|
- .rspec
|
121
|
+
- .travis.yml
|
121
122
|
- Gemfile
|
122
123
|
- LICENSE.txt
|
123
124
|
- README.md
|
@@ -136,7 +137,7 @@ files:
|
|
136
137
|
- lib/shoryuken/logging.rb
|
137
138
|
- lib/shoryuken/manager.rb
|
138
139
|
- lib/shoryuken/middleware/chain.rb
|
139
|
-
- lib/shoryuken/middleware/server/
|
140
|
+
- lib/shoryuken/middleware/server/delete.rb
|
140
141
|
- lib/shoryuken/middleware/server/logging.rb
|
141
142
|
- lib/shoryuken/processor.rb
|
142
143
|
- lib/shoryuken/util.rb
|
@@ -144,13 +145,15 @@ files:
|
|
144
145
|
- lib/shoryuken/worker.rb
|
145
146
|
- shoryuken.gemspec
|
146
147
|
- shoryuken.jpg
|
147
|
-
- spec/
|
148
|
+
- spec/integration/launcher_spec.rb
|
148
149
|
- spec/shoryuken/client_spec.rb
|
149
150
|
- spec/shoryuken/core_ext_spec.rb
|
150
151
|
- spec/shoryuken/fetcher_spec.rb
|
151
|
-
- spec/shoryuken/integration/launcher_spec.rb
|
152
152
|
- spec/shoryuken/manager_spec.rb
|
153
|
+
- spec/shoryuken/middleware/chain_spec.rb
|
154
|
+
- spec/shoryuken/middleware/server/delete_spec.rb
|
153
155
|
- spec/shoryuken/processor_spec.rb
|
156
|
+
- spec/shoryuken/util_spec.rb
|
154
157
|
- spec/shoryuken/worker_spec.rb
|
155
158
|
- spec/spec_helper.rb
|
156
159
|
homepage: https://github.com/phstc/shoryuken
|
@@ -178,12 +181,14 @@ signing_key:
|
|
178
181
|
specification_version: 4
|
179
182
|
summary: Shoryuken is a super efficient AWS SQS thread based message processor
|
180
183
|
test_files:
|
181
|
-
- spec/
|
184
|
+
- spec/integration/launcher_spec.rb
|
182
185
|
- spec/shoryuken/client_spec.rb
|
183
186
|
- spec/shoryuken/core_ext_spec.rb
|
184
187
|
- spec/shoryuken/fetcher_spec.rb
|
185
|
-
- spec/shoryuken/integration/launcher_spec.rb
|
186
188
|
- spec/shoryuken/manager_spec.rb
|
189
|
+
- spec/shoryuken/middleware/chain_spec.rb
|
190
|
+
- spec/shoryuken/middleware/server/delete_spec.rb
|
187
191
|
- spec/shoryuken/processor_spec.rb
|
192
|
+
- spec/shoryuken/util_spec.rb
|
188
193
|
- spec/shoryuken/worker_spec.rb
|
189
194
|
- spec/spec_helper.rb
|
@@ -1,36 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Shoryuken::Launcher do
|
4
|
-
describe 'Consuming messages', slow: :true do
|
5
|
-
before do
|
6
|
-
subject.run
|
7
|
-
|
8
|
-
$received_messages = 0
|
9
|
-
end
|
10
|
-
|
11
|
-
after do
|
12
|
-
subject.stop
|
13
|
-
end
|
14
|
-
|
15
|
-
class ShoryukenWorker
|
16
|
-
include Shoryuken::Worker
|
17
|
-
|
18
|
-
shoryuken_options queue: 'shoryuken', auto_delete: true
|
19
|
-
|
20
|
-
def perform(sqs_msg)
|
21
|
-
$received_messages += 1
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
it 'consumes a message' do
|
26
|
-
Shoryuken::Client.queues('shoryuken').send_message('Yo')
|
27
|
-
|
28
|
-
10.times do
|
29
|
-
break if $received_messages > 0
|
30
|
-
sleep 0.5
|
31
|
-
end
|
32
|
-
|
33
|
-
expect($received_messages).to eq 1
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|