shoryuken 3.1.12 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/CHANGELOG.md +14 -3
- data/bin/shoryuken +1 -0
- data/lib/shoryuken.rb +6 -0
- data/lib/shoryuken/body_parser.rb +27 -0
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +2 -5
- data/lib/shoryuken/middleware/server/timing.rb +12 -14
- data/lib/shoryuken/options.rb +9 -0
- data/lib/shoryuken/processor.rb +5 -21
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +66 -28
- data/lib/shoryuken/worker/default_executor.rb +33 -0
- data/lib/shoryuken/worker/inline_executor.rb +28 -0
- data/spec/shoryuken/body_parser_spec.rb +89 -0
- data/spec/shoryuken/processor_spec.rb +15 -97
- data/spec/shoryuken/worker/default_executor_spec.rb +100 -0
- data/spec/shoryuken/worker/inline_executor_spec.rb +23 -0
- data/spec/shoryuken/worker_spec.rb +18 -91
- metadata +11 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f74e5fbb599fe5bbe41bed5fd542c0c1fdcc2d35
|
4
|
+
data.tar.gz: 6099d42682b5e2a1797d7cdbdc7bf27b344b692c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f076e8f427f39ca92be4f27be819350d90270ce9e695d1173a6944b82d16680d11afd78a0075dd549e0a2c6031e6d5708c306cb245c4d7dfcc69d8a5e7151bc7
|
7
|
+
data.tar.gz: 2fd34c84e6260df3c35427ffb27d3d067fc1ea11469609d61e8eedd465e4516d75e052079b5195ede7c7917ad3c97bb39c5440bc5436ae3ab2581f3fb72bad31
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
## [v3.2.0] - 2018-01-03
|
2
|
+
|
3
|
+
- Preserve parent worker class options
|
4
|
+
- [#451](https://github.com/phstc/shoryuken/pull/451)
|
5
|
+
|
6
|
+
- Add -t (shutdown timeout) option to CL
|
7
|
+
- [#449](https://github.com/phstc/shoryuken/pull/449)
|
8
|
+
|
9
|
+
- Support inline (Active Job like) for standard workers
|
10
|
+
- [#448](https://github.com/phstc/shoryuken/pull/448)
|
11
|
+
|
1
12
|
## [v3.1.12] - 2017-09-25
|
2
13
|
|
3
14
|
- Reduce fetch log verbosity
|
@@ -15,12 +26,12 @@
|
|
15
26
|
|
16
27
|
## [v3.1.9] - 2017-08-24
|
17
28
|
|
18
|
-
- Add support for dispatch fire event
|
19
|
-
- [#426](https://github.com/phstc/shoryuken/pull/426)
|
20
|
-
|
21
29
|
- Add support for adding a middleware to the front of chain
|
22
30
|
- [#427](https://github.com/phstc/shoryuken/pull/427)
|
23
31
|
|
32
|
+
- Add support for dispatch fire event
|
33
|
+
- [#426](https://github.com/phstc/shoryuken/pull/426)
|
34
|
+
|
24
35
|
## [v3.1.8] - 2017-08-17
|
25
36
|
|
26
37
|
- Make Polling strategy backward compatibility
|
data/bin/shoryuken
CHANGED
@@ -21,6 +21,7 @@ module Shoryuken
|
|
21
21
|
method_option :daemon, aliases: '-d', type: :boolean, desc: 'Daemonize process'
|
22
22
|
method_option :queues, aliases: '-q', type: :array, desc: 'Queues to process with optional weights'
|
23
23
|
method_option :require, aliases: '-r', type: :string, desc: 'Dir or path of the workers'
|
24
|
+
method_option :timeout, aliases: '-t', type: :numeric, desc: 'Hard shutdown timeout'
|
24
25
|
method_option :config, aliases: '-C', type: :string, desc: 'Path to config file'
|
25
26
|
method_option :config_file, type: :string, desc: 'Path to config file (backwards compatibility)'
|
26
27
|
method_option :rails, aliases: '-R', type: :boolean, desc: 'Load Rails'
|
data/lib/shoryuken.rb
CHANGED
@@ -19,6 +19,8 @@ require 'shoryuken/queue'
|
|
19
19
|
require 'shoryuken/message'
|
20
20
|
require 'shoryuken/client'
|
21
21
|
require 'shoryuken/worker'
|
22
|
+
require 'shoryuken/worker/default_executor'
|
23
|
+
require 'shoryuken/worker/inline_executor'
|
22
24
|
require 'shoryuken/worker_registry'
|
23
25
|
require 'shoryuken/default_worker_registry'
|
24
26
|
require 'shoryuken/middleware/chain'
|
@@ -32,9 +34,11 @@ require 'shoryuken/polling/strict_priority'
|
|
32
34
|
require 'shoryuken/manager'
|
33
35
|
require 'shoryuken/launcher'
|
34
36
|
require 'shoryuken/processor'
|
37
|
+
require 'shoryuken/body_parser'
|
35
38
|
require 'shoryuken/fetcher'
|
36
39
|
require 'shoryuken/options'
|
37
40
|
|
41
|
+
|
38
42
|
module Shoryuken
|
39
43
|
extend SingleForwardable
|
40
44
|
|
@@ -47,6 +51,8 @@ module Shoryuken
|
|
47
51
|
:ungrouped_queues,
|
48
52
|
:worker_registry,
|
49
53
|
:worker_registry=,
|
54
|
+
:worker_executor,
|
55
|
+
:worker_executor=,
|
50
56
|
:polling_strategy,
|
51
57
|
:start_callback,
|
52
58
|
:start_callback=,
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
class BodyParser
|
3
|
+
class << self
|
4
|
+
def parse(worker_class, sqs_msg)
|
5
|
+
body_parser = worker_class.get_shoryuken_options['body_parser']
|
6
|
+
|
7
|
+
case body_parser
|
8
|
+
when :json
|
9
|
+
JSON.parse(sqs_msg.body)
|
10
|
+
when Proc
|
11
|
+
body_parser.call(sqs_msg)
|
12
|
+
when :text, nil
|
13
|
+
sqs_msg.body
|
14
|
+
else
|
15
|
+
if body_parser.respond_to?(:parse)
|
16
|
+
# JSON.parse
|
17
|
+
body_parser.parse(sqs_msg.body)
|
18
|
+
elsif body_parser.respond_to?(:load)
|
19
|
+
# see https://github.com/phstc/shoryuken/pull/91
|
20
|
+
# JSON.load
|
21
|
+
body_parser.load(sqs_msg.body)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -31,16 +31,13 @@ module Shoryuken
|
|
31
31
|
Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
|
32
32
|
begin
|
33
33
|
logger.debug do
|
34
|
-
"Extending message #{
|
35
|
-
"visibility timeout by #{queue_visibility_timeout}s."
|
34
|
+
"Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
|
36
35
|
end
|
37
36
|
|
38
37
|
sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
|
39
38
|
rescue => ex
|
40
39
|
logger.error do
|
41
|
-
|
42
|
-
"#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
|
43
|
-
"visibility timeout. Error: #{ex.message}"
|
40
|
+
"Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
|
44
41
|
end
|
45
42
|
end
|
46
43
|
end
|
@@ -5,25 +5,23 @@ module Shoryuken
|
|
5
5
|
include Util
|
6
6
|
|
7
7
|
def call(worker, queue, sqs_msg, body)
|
8
|
-
|
9
|
-
|
10
|
-
started_at = Time.now
|
8
|
+
begin
|
9
|
+
started_at = Time.now
|
11
10
|
|
12
|
-
|
11
|
+
logger.info { "started at #{started_at}" }
|
13
12
|
|
14
|
-
|
13
|
+
yield
|
15
14
|
|
16
|
-
|
15
|
+
total_time = elapsed(started_at)
|
17
16
|
|
18
|
-
|
19
|
-
|
20
|
-
end
|
21
|
-
|
22
|
-
logger.info { "completed in: #{total_time} ms" }
|
23
|
-
rescue
|
24
|
-
logger.info { "failed in: #{elapsed(started_at)} ms" }
|
25
|
-
raise
|
17
|
+
if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
|
18
|
+
logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
|
26
19
|
end
|
20
|
+
|
21
|
+
logger.info { "completed in: #{total_time} ms" }
|
22
|
+
rescue
|
23
|
+
logger.info { "failed in: #{elapsed(started_at)} ms" }
|
24
|
+
raise
|
27
25
|
end
|
28
26
|
end
|
29
27
|
end
|
data/lib/shoryuken/options.rb
CHANGED
@@ -21,6 +21,7 @@ module Shoryuken
|
|
21
21
|
@@sqs_client_receive_message_opts = {}
|
22
22
|
@@start_callback = nil
|
23
23
|
@@stop_callback = nil
|
24
|
+
@@worker_executor = Worker::DefaultExecutor
|
24
25
|
|
25
26
|
class << self
|
26
27
|
def active_job?
|
@@ -56,6 +57,14 @@ module Shoryuken
|
|
56
57
|
@@worker_registry = worker_registry
|
57
58
|
end
|
58
59
|
|
60
|
+
def worker_executor
|
61
|
+
@@worker_executor
|
62
|
+
end
|
63
|
+
|
64
|
+
def worker_executor=(worker_executor)
|
65
|
+
@@worker_executor = worker_executor
|
66
|
+
end
|
67
|
+
|
59
68
|
def polling_strategy(group)
|
60
69
|
strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
|
61
70
|
|
data/lib/shoryuken/processor.rb
CHANGED
@@ -16,8 +16,10 @@ module Shoryuken
|
|
16
16
|
def process
|
17
17
|
return logger.error { "No worker found for #{queue}" } unless worker
|
18
18
|
|
19
|
-
worker.class
|
20
|
-
worker.
|
19
|
+
Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
|
20
|
+
worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
|
21
|
+
worker.perform(sqs_msg, body)
|
22
|
+
end
|
21
23
|
end
|
22
24
|
rescue Exception => ex
|
23
25
|
logger.error { "Processor failed: #{ex.message}" }
|
@@ -41,25 +43,7 @@ module Shoryuken
|
|
41
43
|
end
|
42
44
|
|
43
45
|
def parse_body(sqs_msg)
|
44
|
-
|
45
|
-
|
46
|
-
case body_parser
|
47
|
-
when :json
|
48
|
-
JSON.parse(sqs_msg.body)
|
49
|
-
when Proc
|
50
|
-
body_parser.call(sqs_msg)
|
51
|
-
when :text, nil
|
52
|
-
sqs_msg.body
|
53
|
-
else
|
54
|
-
if body_parser.respond_to?(:parse)
|
55
|
-
# JSON.parse
|
56
|
-
body_parser.parse(sqs_msg.body)
|
57
|
-
elsif body_parser.respond_to?(:load)
|
58
|
-
# see https://github.com/phstc/shoryuken/pull/91
|
59
|
-
# JSON.load
|
60
|
-
body_parser.load(sqs_msg.body)
|
61
|
-
end
|
62
|
-
end
|
46
|
+
BodyParser.parse(worker_class, sqs_msg)
|
63
47
|
end
|
64
48
|
end
|
65
49
|
end
|
data/lib/shoryuken/version.rb
CHANGED
data/lib/shoryuken/worker.rb
CHANGED
@@ -2,33 +2,16 @@ module Shoryuken
|
|
2
2
|
module Worker
|
3
3
|
def self.included(base)
|
4
4
|
base.extend(ClassMethods)
|
5
|
+
base.shoryuken_class_attribute :shoryuken_options_hash
|
5
6
|
end
|
6
7
|
|
7
8
|
module ClassMethods
|
8
9
|
def perform_async(body, options = {})
|
9
|
-
|
10
|
-
options[:message_attributes]['shoryuken_class'] = {
|
11
|
-
string_value: self.to_s,
|
12
|
-
data_type: 'String'
|
13
|
-
}
|
14
|
-
|
15
|
-
options[:message_body] = body
|
16
|
-
|
17
|
-
queue = options.delete(:queue) || get_shoryuken_options['queue']
|
18
|
-
|
19
|
-
Shoryuken::Client.queues(queue).send_message(options)
|
10
|
+
Shoryuken.worker_executor.perform_async(self, body, options)
|
20
11
|
end
|
21
12
|
|
22
13
|
def perform_in(interval, body, options = {})
|
23
|
-
interval
|
24
|
-
now = Time.now.to_f
|
25
|
-
ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
|
26
|
-
|
27
|
-
delay = (ts - now).ceil
|
28
|
-
|
29
|
-
raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
|
30
|
-
|
31
|
-
perform_async(body, options.merge(delay_seconds: delay))
|
14
|
+
Shoryuken.worker_executor.perform_in(self, interval, body, options)
|
32
15
|
end
|
33
16
|
|
34
17
|
alias_method :perform_at, :perform_in
|
@@ -40,7 +23,7 @@ module Shoryuken
|
|
40
23
|
end
|
41
24
|
|
42
25
|
def shoryuken_options(opts = {})
|
43
|
-
|
26
|
+
self.shoryuken_options_hash = get_shoryuken_options.merge(stringify_keys(opts || {}))
|
44
27
|
normalize_worker_queue!
|
45
28
|
end
|
46
29
|
|
@@ -57,7 +40,7 @@ module Shoryuken
|
|
57
40
|
end
|
58
41
|
|
59
42
|
def get_shoryuken_options # :nodoc:
|
60
|
-
|
43
|
+
shoryuken_options_hash || Shoryuken.default_worker_options
|
61
44
|
end
|
62
45
|
|
63
46
|
def stringify_keys(hash) # :nodoc:
|
@@ -66,23 +49,78 @@ module Shoryuken
|
|
66
49
|
new_hash
|
67
50
|
end
|
68
51
|
|
52
|
+
def shoryuken_class_attribute(*attrs) # :nodoc:
|
53
|
+
attrs.each do |name|
|
54
|
+
singleton_class.instance_eval do
|
55
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
56
|
+
end
|
57
|
+
define_singleton_method(name) { nil }
|
58
|
+
|
59
|
+
ivar = "@#{name}"
|
60
|
+
|
61
|
+
singleton_class.instance_eval do
|
62
|
+
m = "#{name}="
|
63
|
+
undef_method(m) if method_defined?(m) || private_method_defined?(m)
|
64
|
+
end
|
65
|
+
|
66
|
+
define_singleton_method("#{name}=") do |val|
|
67
|
+
singleton_class.class_eval do
|
68
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
69
|
+
define_method(name) { val }
|
70
|
+
end
|
71
|
+
|
72
|
+
# singleton? backwards compatibility for ruby < 2.1
|
73
|
+
singleton_klass = respond_to?(:singleton?) ? singleton? : self != ancestors.first
|
74
|
+
|
75
|
+
if singleton_klass
|
76
|
+
class_eval do
|
77
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
78
|
+
define_method(name) do
|
79
|
+
if instance_variable_defined? ivar
|
80
|
+
instance_variable_get ivar
|
81
|
+
else
|
82
|
+
singleton_class.send name
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
val
|
88
|
+
end
|
89
|
+
|
90
|
+
# instance reader
|
91
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
92
|
+
define_method(name) do
|
93
|
+
if instance_variable_defined?(ivar)
|
94
|
+
instance_variable_get ivar
|
95
|
+
else
|
96
|
+
self.class.public_send name
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
# instance writer
|
101
|
+
m = "#{name}="
|
102
|
+
undef_method(m) if method_defined?(m) || private_method_defined?(m)
|
103
|
+
attr_writer name
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
69
107
|
private
|
70
108
|
|
71
109
|
def normalize_worker_queue!
|
72
|
-
queue =
|
110
|
+
queue = shoryuken_options_hash['queue']
|
73
111
|
if queue.respond_to?(:call)
|
74
112
|
queue = queue.call
|
75
|
-
|
113
|
+
shoryuken_options_hash['queue'] = queue
|
76
114
|
end
|
77
115
|
|
78
|
-
case
|
116
|
+
case shoryuken_options_hash['queue']
|
79
117
|
when Array
|
80
|
-
|
118
|
+
shoryuken_options_hash['queue'].map!(&:to_s)
|
81
119
|
when Symbol
|
82
|
-
|
120
|
+
shoryuken_options_hash['queue'] = shoryuken_options_hash['queue'].to_s
|
83
121
|
end
|
84
122
|
|
85
|
-
[
|
123
|
+
[shoryuken_options_hash['queue']].flatten.compact.each(&method(:register_worker))
|
86
124
|
end
|
87
125
|
|
88
126
|
def register_worker(queue)
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
module Worker
|
3
|
+
class DefaultExecutor
|
4
|
+
class << self
|
5
|
+
def perform_async(worker_class, body, options = {})
|
6
|
+
options[:message_attributes] ||= {}
|
7
|
+
options[:message_attributes]['shoryuken_class'] = {
|
8
|
+
string_value: worker_class.to_s,
|
9
|
+
data_type: 'String'
|
10
|
+
}
|
11
|
+
|
12
|
+
options[:message_body] = body
|
13
|
+
|
14
|
+
queue = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
|
15
|
+
|
16
|
+
Shoryuken::Client.queues(queue).send_message(options)
|
17
|
+
end
|
18
|
+
|
19
|
+
def perform_in(worker_class, interval, body, options = {})
|
20
|
+
interval = interval.to_f
|
21
|
+
now = Time.now.to_f
|
22
|
+
ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
|
23
|
+
|
24
|
+
delay = (ts - now).ceil
|
25
|
+
|
26
|
+
raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
|
27
|
+
|
28
|
+
worker_class.perform_async(body, options.merge(delay_seconds: delay))
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
module Worker
|
3
|
+
class InlineExecutor
|
4
|
+
class << self
|
5
|
+
def perform_async(worker_class, body, options = {})
|
6
|
+
body = JSON.dump(body) if body.is_a?(Hash)
|
7
|
+
|
8
|
+
sqs_msg = OpenStruct.new(
|
9
|
+
body: body,
|
10
|
+
attributes: nil,
|
11
|
+
md5_of_body: nil,
|
12
|
+
md5_of_message_attributes: nil,
|
13
|
+
message_attributes: nil,
|
14
|
+
message_id: nil,
|
15
|
+
receipt_handle: nil,
|
16
|
+
delete: nil
|
17
|
+
)
|
18
|
+
|
19
|
+
worker_class.new.perform(sqs_msg, BodyParser.parse(worker_class, sqs_msg))
|
20
|
+
end
|
21
|
+
|
22
|
+
def perform_in(worker_class, _interval, body, options = {})
|
23
|
+
perform_async(worker_class, body, options)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'shoryuken/body_parser'
|
3
|
+
|
4
|
+
RSpec.describe Shoryuken::BodyParser do
|
5
|
+
let(:sqs_msg) { double }
|
6
|
+
|
7
|
+
describe '#parser' do
|
8
|
+
it 'parses the body into JSON' do
|
9
|
+
TestWorker.get_shoryuken_options['body_parser'] = :json
|
10
|
+
|
11
|
+
body = { 'test' => 'hi' }
|
12
|
+
|
13
|
+
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
14
|
+
|
15
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'parses the body calling the proc' do
|
19
|
+
TestWorker.get_shoryuken_options['body_parser'] = proc { |sqs_msg| "*#{sqs_msg.body}*" }
|
20
|
+
|
21
|
+
allow(sqs_msg).to receive(:body).and_return('test')
|
22
|
+
|
23
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq('*test*')
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'parses the body as text' do
|
27
|
+
TestWorker.get_shoryuken_options['body_parser'] = :text
|
28
|
+
|
29
|
+
body = 'test'
|
30
|
+
|
31
|
+
allow(sqs_msg).to receive(:body).and_return(body)
|
32
|
+
|
33
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq('test')
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'parses calling `.load`' do
|
37
|
+
TestWorker.get_shoryuken_options['body_parser'] = Class.new do
|
38
|
+
def self.load(*args)
|
39
|
+
JSON.load(*args)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
body = { 'test' => 'hi' }
|
44
|
+
|
45
|
+
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
46
|
+
|
47
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'parses calling `.parse`' do
|
51
|
+
TestWorker.get_shoryuken_options['body_parser'] = Class.new do
|
52
|
+
def self.parse(*args)
|
53
|
+
JSON.parse(*args)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
body = { 'test' => 'hi' }
|
58
|
+
|
59
|
+
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
60
|
+
|
61
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
|
62
|
+
end
|
63
|
+
|
64
|
+
context 'when parse errors' do
|
65
|
+
before do
|
66
|
+
TestWorker.get_shoryuken_options['body_parser'] = :json
|
67
|
+
|
68
|
+
allow(sqs_msg).to receive(:body).and_return('invalid JSON')
|
69
|
+
end
|
70
|
+
|
71
|
+
specify do
|
72
|
+
expect { described_class.parse(TestWorker, sqs_msg) }.
|
73
|
+
to raise_error(JSON::ParserError, /unexpected token at 'invalid JSON'/)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
context 'when `object_type: nil`' do
|
78
|
+
it 'parses the body as text' do
|
79
|
+
TestWorker.get_shoryuken_options['body_parser'] = nil
|
80
|
+
|
81
|
+
body = 'test'
|
82
|
+
|
83
|
+
allow(sqs_msg).to receive(:body).and_return(body)
|
84
|
+
|
85
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -1,6 +1,4 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
-
require 'shoryuken/processor'
|
3
|
-
require 'shoryuken/manager'
|
4
2
|
|
5
3
|
RSpec.describe Shoryuken::Processor do
|
6
4
|
let(:manager) { double Shoryuken::Manager }
|
@@ -8,12 +6,14 @@ RSpec.describe Shoryuken::Processor do
|
|
8
6
|
let(:queue) { 'default' }
|
9
7
|
|
10
8
|
let(:sqs_msg) do
|
11
|
-
double
|
9
|
+
double(
|
10
|
+
Shoryuken::Message,
|
12
11
|
queue_url: queue,
|
13
12
|
body: 'test',
|
14
13
|
message_attributes: {},
|
15
14
|
message_id: SecureRandom.uuid,
|
16
15
|
receipt_handle: SecureRandom.uuid
|
16
|
+
)
|
17
17
|
end
|
18
18
|
|
19
19
|
before do
|
@@ -25,101 +25,15 @@ RSpec.describe Shoryuken::Processor do
|
|
25
25
|
subject { described_class.new(queue, sqs_msg) }
|
26
26
|
|
27
27
|
describe '#process' do
|
28
|
-
it '
|
29
|
-
|
28
|
+
it 'sets log context' do
|
29
|
+
expect(Shoryuken::Logging).to receive(:with_context).with("TestWorker/#{queue}/#{sqs_msg.message_id}")
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
34
|
-
|
35
|
-
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
31
|
+
allow_any_instance_of(TestWorker).to receive(:perform)
|
32
|
+
allow(sqs_msg).to receive(:body)
|
36
33
|
|
37
34
|
subject.process
|
38
35
|
end
|
39
36
|
|
40
|
-
it 'parses the body calling the proc' do
|
41
|
-
TestWorker.get_shoryuken_options['body_parser'] = proc { |sqs_msg| "*#{sqs_msg.body}*" }
|
42
|
-
|
43
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, '*test*')
|
44
|
-
|
45
|
-
allow(sqs_msg).to receive(:body).and_return('test')
|
46
|
-
|
47
|
-
subject.process
|
48
|
-
end
|
49
|
-
|
50
|
-
it 'parses the body as text' do
|
51
|
-
TestWorker.get_shoryuken_options['body_parser'] = :text
|
52
|
-
|
53
|
-
body = 'test'
|
54
|
-
|
55
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
56
|
-
|
57
|
-
allow(sqs_msg).to receive(:body).and_return(body)
|
58
|
-
|
59
|
-
subject.process
|
60
|
-
end
|
61
|
-
|
62
|
-
it 'parses calling `.load`' do
|
63
|
-
TestWorker.get_shoryuken_options['body_parser'] = Class.new do
|
64
|
-
def self.load(*args)
|
65
|
-
JSON.load(*args)
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
body = { 'test' => 'hi' }
|
70
|
-
|
71
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
72
|
-
|
73
|
-
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
74
|
-
|
75
|
-
subject.process
|
76
|
-
end
|
77
|
-
|
78
|
-
it 'parses calling `.parse`' do
|
79
|
-
TestWorker.get_shoryuken_options['body_parser'] = Class.new do
|
80
|
-
def self.parse(*args)
|
81
|
-
JSON.parse(*args)
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
body = { 'test' => 'hi' }
|
86
|
-
|
87
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
88
|
-
|
89
|
-
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
90
|
-
|
91
|
-
subject.process
|
92
|
-
end
|
93
|
-
|
94
|
-
context 'when parse errors' do
|
95
|
-
before do
|
96
|
-
TestWorker.get_shoryuken_options['body_parser'] = :json
|
97
|
-
|
98
|
-
allow(sqs_msg).to receive(:body).and_return('invalid JSON')
|
99
|
-
end
|
100
|
-
|
101
|
-
specify do
|
102
|
-
expect(subject.logger).to receive(:error).twice
|
103
|
-
|
104
|
-
expect { subject.process }.
|
105
|
-
to raise_error(JSON::ParserError, /unexpected token at 'invalid JSON'/)
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
context 'when `object_type: nil`' do
|
110
|
-
it 'parses the body as text' do
|
111
|
-
TestWorker.get_shoryuken_options['body_parser'] = nil
|
112
|
-
|
113
|
-
body = 'test'
|
114
|
-
|
115
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
116
|
-
|
117
|
-
allow(sqs_msg).to receive(:body).and_return(body)
|
118
|
-
|
119
|
-
subject.process
|
120
|
-
end
|
121
|
-
end
|
122
|
-
|
123
37
|
context 'when custom middleware' do
|
124
38
|
let(:queue) { 'worker_called_middleware' }
|
125
39
|
|
@@ -220,15 +134,19 @@ RSpec.describe Shoryuken::Processor do
|
|
220
134
|
|
221
135
|
context 'when shoryuken_class header' do
|
222
136
|
let(:sqs_msg) do
|
223
|
-
double
|
137
|
+
double(
|
138
|
+
Shoryuken::Message,
|
224
139
|
queue_url: queue,
|
225
140
|
body: 'test',
|
226
141
|
message_attributes: {
|
227
142
|
'shoryuken_class' => {
|
228
143
|
string_value: TestWorker.to_s,
|
229
|
-
data_type: 'String'
|
230
|
-
|
231
|
-
|
144
|
+
data_type: 'String'
|
145
|
+
}
|
146
|
+
},
|
147
|
+
message_id: SecureRandom.uuid,
|
148
|
+
receipt_handle: SecureRandom.uuid
|
149
|
+
)
|
232
150
|
end
|
233
151
|
|
234
152
|
it 'performs without delete' do
|
@@ -0,0 +1,100 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Shoryuken::Worker::DefaultExecutor do
|
4
|
+
let(:sqs_queue) { double 'SQS Queue' }
|
5
|
+
let(:queue) { 'default' }
|
6
|
+
|
7
|
+
before do
|
8
|
+
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '.perform_in' do
|
12
|
+
it 'delays a message' do
|
13
|
+
expect(sqs_queue).to receive(:send_message).with(
|
14
|
+
message_attributes: {
|
15
|
+
'shoryuken_class' => {
|
16
|
+
string_value: TestWorker.to_s,
|
17
|
+
data_type: 'String'
|
18
|
+
}
|
19
|
+
},
|
20
|
+
message_body: 'message',
|
21
|
+
delay_seconds: 60)
|
22
|
+
|
23
|
+
TestWorker.perform_in(60, 'message')
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'raises an exception' do
|
27
|
+
expect {
|
28
|
+
TestWorker.perform_in(901, 'message')
|
29
|
+
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe '.perform_at' do
|
34
|
+
it 'delays a message' do
|
35
|
+
expect(sqs_queue).to receive(:send_message).with(
|
36
|
+
message_attributes: {
|
37
|
+
'shoryuken_class' => {
|
38
|
+
string_value: TestWorker.to_s,
|
39
|
+
data_type: 'String'
|
40
|
+
}
|
41
|
+
},
|
42
|
+
message_body: 'message',
|
43
|
+
delay_seconds: 60)
|
44
|
+
|
45
|
+
TestWorker.perform_in(Time.now + 60, 'message')
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'raises an exception' do
|
49
|
+
expect {
|
50
|
+
TestWorker.perform_in(Time.now + 901, 'message')
|
51
|
+
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe '.perform_async' do
|
56
|
+
it 'enqueues a message' do
|
57
|
+
expect(sqs_queue).to receive(:send_message).with(
|
58
|
+
message_attributes: {
|
59
|
+
'shoryuken_class' => {
|
60
|
+
string_value: TestWorker.to_s,
|
61
|
+
data_type: 'String'
|
62
|
+
}
|
63
|
+
},
|
64
|
+
message_body: 'message')
|
65
|
+
|
66
|
+
TestWorker.perform_async('message')
|
67
|
+
end
|
68
|
+
|
69
|
+
it 'enqueues a message with options' do
|
70
|
+
expect(sqs_queue).to receive(:send_message).with(
|
71
|
+
delay_seconds: 60,
|
72
|
+
message_attributes: {
|
73
|
+
'shoryuken_class' => {
|
74
|
+
string_value: TestWorker.to_s,
|
75
|
+
data_type: 'String'
|
76
|
+
}
|
77
|
+
},
|
78
|
+
message_body: 'delayed message')
|
79
|
+
|
80
|
+
TestWorker.perform_async('delayed message', delay_seconds: 60)
|
81
|
+
end
|
82
|
+
|
83
|
+
it 'accepts an `queue` option' do
|
84
|
+
new_queue = 'some_different_queue'
|
85
|
+
|
86
|
+
expect(Shoryuken::Client).to receive(:queues).with(new_queue).and_return(sqs_queue)
|
87
|
+
|
88
|
+
expect(sqs_queue).to receive(:send_message).with(
|
89
|
+
message_attributes: {
|
90
|
+
'shoryuken_class' => {
|
91
|
+
string_value: TestWorker.to_s,
|
92
|
+
data_type: 'String'
|
93
|
+
}
|
94
|
+
},
|
95
|
+
message_body: 'delayed message')
|
96
|
+
|
97
|
+
TestWorker.perform_async('delayed message', queue: new_queue)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Shoryuken::Worker::InlineExecutor do
|
4
|
+
before do
|
5
|
+
Shoryuken.worker_executor = described_class
|
6
|
+
end
|
7
|
+
|
8
|
+
describe '.perform_async' do
|
9
|
+
specify do
|
10
|
+
expect_any_instance_of(TestWorker).to receive(:perform)
|
11
|
+
|
12
|
+
TestWorker.perform_async('test')
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
describe '.perform_in' do
|
17
|
+
specify do
|
18
|
+
expect_any_instance_of(TestWorker).to receive(:perform)
|
19
|
+
|
20
|
+
TestWorker.perform_in(60, 'test')
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
RSpec.describe
|
3
|
+
RSpec.describe Shoryuken::Worker do
|
4
4
|
let(:sqs_queue) { double 'SQS Queue' }
|
5
5
|
let(:queue) { 'default' }
|
6
6
|
|
@@ -8,96 +8,6 @@ RSpec.describe 'Shoryuken::Worker' do
|
|
8
8
|
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
9
9
|
end
|
10
10
|
|
11
|
-
describe '.perform_in' do
|
12
|
-
it 'delays a message' do
|
13
|
-
expect(sqs_queue).to receive(:send_message).with(
|
14
|
-
message_attributes: {
|
15
|
-
'shoryuken_class' => {
|
16
|
-
string_value: TestWorker.to_s,
|
17
|
-
data_type: 'String'
|
18
|
-
}
|
19
|
-
},
|
20
|
-
message_body: 'message',
|
21
|
-
delay_seconds: 60)
|
22
|
-
|
23
|
-
TestWorker.perform_in(60, 'message')
|
24
|
-
end
|
25
|
-
|
26
|
-
it 'raises an exception' do
|
27
|
-
expect {
|
28
|
-
TestWorker.perform_in(901, 'message')
|
29
|
-
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
describe '.perform_at' do
|
34
|
-
it 'delays a message' do
|
35
|
-
expect(sqs_queue).to receive(:send_message).with(
|
36
|
-
message_attributes: {
|
37
|
-
'shoryuken_class' => {
|
38
|
-
string_value: TestWorker.to_s,
|
39
|
-
data_type: 'String'
|
40
|
-
}
|
41
|
-
},
|
42
|
-
message_body: 'message',
|
43
|
-
delay_seconds: 60)
|
44
|
-
|
45
|
-
TestWorker.perform_in(Time.now + 60, 'message')
|
46
|
-
end
|
47
|
-
|
48
|
-
it 'raises an exception' do
|
49
|
-
expect {
|
50
|
-
TestWorker.perform_in(Time.now + 901, 'message')
|
51
|
-
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe '.perform_async' do
|
56
|
-
it 'enqueues a message' do
|
57
|
-
expect(sqs_queue).to receive(:send_message).with(
|
58
|
-
message_attributes: {
|
59
|
-
'shoryuken_class' => {
|
60
|
-
string_value: TestWorker.to_s,
|
61
|
-
data_type: 'String'
|
62
|
-
}
|
63
|
-
},
|
64
|
-
message_body: 'message')
|
65
|
-
|
66
|
-
TestWorker.perform_async('message')
|
67
|
-
end
|
68
|
-
|
69
|
-
it 'enqueues a message with options' do
|
70
|
-
expect(sqs_queue).to receive(:send_message).with(
|
71
|
-
delay_seconds: 60,
|
72
|
-
message_attributes: {
|
73
|
-
'shoryuken_class' => {
|
74
|
-
string_value: TestWorker.to_s,
|
75
|
-
data_type: 'String'
|
76
|
-
}
|
77
|
-
},
|
78
|
-
message_body: 'delayed message')
|
79
|
-
|
80
|
-
TestWorker.perform_async('delayed message', delay_seconds: 60)
|
81
|
-
end
|
82
|
-
|
83
|
-
it 'accepts an `queue` option' do
|
84
|
-
new_queue = 'some_different_queue'
|
85
|
-
|
86
|
-
expect(Shoryuken::Client).to receive(:queues).with(new_queue).and_return(sqs_queue)
|
87
|
-
|
88
|
-
expect(sqs_queue).to receive(:send_message).with(
|
89
|
-
message_attributes: {
|
90
|
-
'shoryuken_class' => {
|
91
|
-
string_value: TestWorker.to_s,
|
92
|
-
data_type: 'String'
|
93
|
-
}
|
94
|
-
},
|
95
|
-
message_body: 'delayed message')
|
96
|
-
|
97
|
-
TestWorker.perform_async('delayed message', queue: new_queue)
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
11
|
describe '.shoryuken_options' do
|
102
12
|
it 'registers a worker' do
|
103
13
|
expect(Shoryuken.worker_registry.workers('default')).to eq([TestWorker])
|
@@ -178,6 +88,23 @@ RSpec.describe 'Shoryuken::Worker' do
|
|
178
88
|
expect(Shoryuken.worker_registry.workers('symbol_queue2')).to eq([WorkerMultipleSymbolQueues])
|
179
89
|
expect(Shoryuken.worker_registry.workers('symbol_queue3')).to eq([WorkerMultipleSymbolQueues])
|
180
90
|
end
|
91
|
+
|
92
|
+
it 'preserves parent class options' do
|
93
|
+
class ParentWorker
|
94
|
+
include Shoryuken::Worker
|
95
|
+
|
96
|
+
shoryuken_options queue: "myqueue", auto_delete: false
|
97
|
+
end
|
98
|
+
|
99
|
+
class ChildWorker < ParentWorker
|
100
|
+
shoryuken_options auto_delete: true
|
101
|
+
end
|
102
|
+
|
103
|
+
expect(ParentWorker.get_shoryuken_options['queue']).to eq("myqueue")
|
104
|
+
expect(ChildWorker.get_shoryuken_options['queue']).to eq("myqueue")
|
105
|
+
expect(ParentWorker.get_shoryuken_options['auto_delete']).to eq(false)
|
106
|
+
expect(ChildWorker.get_shoryuken_options['auto_delete']).to eq(true)
|
107
|
+
end
|
181
108
|
end
|
182
109
|
|
183
110
|
describe '.server_middleware' do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -147,6 +147,7 @@ files:
|
|
147
147
|
- examples/bootstrap_queues.rb
|
148
148
|
- examples/default_worker.rb
|
149
149
|
- lib/shoryuken.rb
|
150
|
+
- lib/shoryuken/body_parser.rb
|
150
151
|
- lib/shoryuken/client.rb
|
151
152
|
- lib/shoryuken/core_ext.rb
|
152
153
|
- lib/shoryuken/default_worker_registry.rb
|
@@ -173,11 +174,14 @@ files:
|
|
173
174
|
- lib/shoryuken/util.rb
|
174
175
|
- lib/shoryuken/version.rb
|
175
176
|
- lib/shoryuken/worker.rb
|
177
|
+
- lib/shoryuken/worker/default_executor.rb
|
178
|
+
- lib/shoryuken/worker/inline_executor.rb
|
176
179
|
- lib/shoryuken/worker_registry.rb
|
177
180
|
- shoryuken.gemspec
|
178
181
|
- shoryuken.jpg
|
179
182
|
- spec/integration/launcher_spec.rb
|
180
183
|
- spec/shoryuken.yml
|
184
|
+
- spec/shoryuken/body_parser_spec.rb
|
181
185
|
- spec/shoryuken/client_spec.rb
|
182
186
|
- spec/shoryuken/core_ext_spec.rb
|
183
187
|
- spec/shoryuken/default_worker_registry_spec.rb
|
@@ -196,6 +200,8 @@ files:
|
|
196
200
|
- spec/shoryuken/queue_spec.rb
|
197
201
|
- spec/shoryuken/runner_spec.rb
|
198
202
|
- spec/shoryuken/util_spec.rb
|
203
|
+
- spec/shoryuken/worker/default_executor_spec.rb
|
204
|
+
- spec/shoryuken/worker/inline_executor_spec.rb
|
199
205
|
- spec/shoryuken/worker_spec.rb
|
200
206
|
- spec/shoryuken_spec.rb
|
201
207
|
- spec/spec_helper.rb
|
@@ -228,6 +234,7 @@ summary: Shoryuken is a super efficient AWS SQS thread based message processor
|
|
228
234
|
test_files:
|
229
235
|
- spec/integration/launcher_spec.rb
|
230
236
|
- spec/shoryuken.yml
|
237
|
+
- spec/shoryuken/body_parser_spec.rb
|
231
238
|
- spec/shoryuken/client_spec.rb
|
232
239
|
- spec/shoryuken/core_ext_spec.rb
|
233
240
|
- spec/shoryuken/default_worker_registry_spec.rb
|
@@ -246,6 +253,8 @@ test_files:
|
|
246
253
|
- spec/shoryuken/queue_spec.rb
|
247
254
|
- spec/shoryuken/runner_spec.rb
|
248
255
|
- spec/shoryuken/util_spec.rb
|
256
|
+
- spec/shoryuken/worker/default_executor_spec.rb
|
257
|
+
- spec/shoryuken/worker/inline_executor_spec.rb
|
249
258
|
- spec/shoryuken/worker_spec.rb
|
250
259
|
- spec/shoryuken_spec.rb
|
251
260
|
- spec/spec_helper.rb
|