shoryuken 3.1.12 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/CHANGELOG.md +14 -3
- data/bin/shoryuken +1 -0
- data/lib/shoryuken.rb +6 -0
- data/lib/shoryuken/body_parser.rb +27 -0
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +2 -5
- data/lib/shoryuken/middleware/server/timing.rb +12 -14
- data/lib/shoryuken/options.rb +9 -0
- data/lib/shoryuken/processor.rb +5 -21
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +66 -28
- data/lib/shoryuken/worker/default_executor.rb +33 -0
- data/lib/shoryuken/worker/inline_executor.rb +28 -0
- data/spec/shoryuken/body_parser_spec.rb +89 -0
- data/spec/shoryuken/processor_spec.rb +15 -97
- data/spec/shoryuken/worker/default_executor_spec.rb +100 -0
- data/spec/shoryuken/worker/inline_executor_spec.rb +23 -0
- data/spec/shoryuken/worker_spec.rb +18 -91
- metadata +11 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f74e5fbb599fe5bbe41bed5fd542c0c1fdcc2d35
|
4
|
+
data.tar.gz: 6099d42682b5e2a1797d7cdbdc7bf27b344b692c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f076e8f427f39ca92be4f27be819350d90270ce9e695d1173a6944b82d16680d11afd78a0075dd549e0a2c6031e6d5708c306cb245c4d7dfcc69d8a5e7151bc7
|
7
|
+
data.tar.gz: 2fd34c84e6260df3c35427ffb27d3d067fc1ea11469609d61e8eedd465e4516d75e052079b5195ede7c7917ad3c97bb39c5440bc5436ae3ab2581f3fb72bad31
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
## [v3.2.0] - 2018-01-03
|
2
|
+
|
3
|
+
- Preserve parent worker class options
|
4
|
+
- [#451](https://github.com/phstc/shoryuken/pull/451)
|
5
|
+
|
6
|
+
- Add -t (shutdown timeout) option to CL
|
7
|
+
- [#449](https://github.com/phstc/shoryuken/pull/449)
|
8
|
+
|
9
|
+
- Support inline (Active Job like) for standard workers
|
10
|
+
- [#448](https://github.com/phstc/shoryuken/pull/448)
|
11
|
+
|
1
12
|
## [v3.1.12] - 2017-09-25
|
2
13
|
|
3
14
|
- Reduce fetch log verbosity
|
@@ -15,12 +26,12 @@
|
|
15
26
|
|
16
27
|
## [v3.1.9] - 2017-08-24
|
17
28
|
|
18
|
-
- Add support for dispatch fire event
|
19
|
-
- [#426](https://github.com/phstc/shoryuken/pull/426)
|
20
|
-
|
21
29
|
- Add support for adding a middleware to the front of chain
|
22
30
|
- [#427](https://github.com/phstc/shoryuken/pull/427)
|
23
31
|
|
32
|
+
- Add support for dispatch fire event
|
33
|
+
- [#426](https://github.com/phstc/shoryuken/pull/426)
|
34
|
+
|
24
35
|
## [v3.1.8] - 2017-08-17
|
25
36
|
|
26
37
|
- Make Polling strategy backward compatibility
|
data/bin/shoryuken
CHANGED
@@ -21,6 +21,7 @@ module Shoryuken
|
|
21
21
|
method_option :daemon, aliases: '-d', type: :boolean, desc: 'Daemonize process'
|
22
22
|
method_option :queues, aliases: '-q', type: :array, desc: 'Queues to process with optional weights'
|
23
23
|
method_option :require, aliases: '-r', type: :string, desc: 'Dir or path of the workers'
|
24
|
+
method_option :timeout, aliases: '-t', type: :numeric, desc: 'Hard shutdown timeout'
|
24
25
|
method_option :config, aliases: '-C', type: :string, desc: 'Path to config file'
|
25
26
|
method_option :config_file, type: :string, desc: 'Path to config file (backwards compatibility)'
|
26
27
|
method_option :rails, aliases: '-R', type: :boolean, desc: 'Load Rails'
|
data/lib/shoryuken.rb
CHANGED
@@ -19,6 +19,8 @@ require 'shoryuken/queue'
|
|
19
19
|
require 'shoryuken/message'
|
20
20
|
require 'shoryuken/client'
|
21
21
|
require 'shoryuken/worker'
|
22
|
+
require 'shoryuken/worker/default_executor'
|
23
|
+
require 'shoryuken/worker/inline_executor'
|
22
24
|
require 'shoryuken/worker_registry'
|
23
25
|
require 'shoryuken/default_worker_registry'
|
24
26
|
require 'shoryuken/middleware/chain'
|
@@ -32,9 +34,11 @@ require 'shoryuken/polling/strict_priority'
|
|
32
34
|
require 'shoryuken/manager'
|
33
35
|
require 'shoryuken/launcher'
|
34
36
|
require 'shoryuken/processor'
|
37
|
+
require 'shoryuken/body_parser'
|
35
38
|
require 'shoryuken/fetcher'
|
36
39
|
require 'shoryuken/options'
|
37
40
|
|
41
|
+
|
38
42
|
module Shoryuken
|
39
43
|
extend SingleForwardable
|
40
44
|
|
@@ -47,6 +51,8 @@ module Shoryuken
|
|
47
51
|
:ungrouped_queues,
|
48
52
|
:worker_registry,
|
49
53
|
:worker_registry=,
|
54
|
+
:worker_executor,
|
55
|
+
:worker_executor=,
|
50
56
|
:polling_strategy,
|
51
57
|
:start_callback,
|
52
58
|
:start_callback=,
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
class BodyParser
|
3
|
+
class << self
|
4
|
+
def parse(worker_class, sqs_msg)
|
5
|
+
body_parser = worker_class.get_shoryuken_options['body_parser']
|
6
|
+
|
7
|
+
case body_parser
|
8
|
+
when :json
|
9
|
+
JSON.parse(sqs_msg.body)
|
10
|
+
when Proc
|
11
|
+
body_parser.call(sqs_msg)
|
12
|
+
when :text, nil
|
13
|
+
sqs_msg.body
|
14
|
+
else
|
15
|
+
if body_parser.respond_to?(:parse)
|
16
|
+
# JSON.parse
|
17
|
+
body_parser.parse(sqs_msg.body)
|
18
|
+
elsif body_parser.respond_to?(:load)
|
19
|
+
# see https://github.com/phstc/shoryuken/pull/91
|
20
|
+
# JSON.load
|
21
|
+
body_parser.load(sqs_msg.body)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -31,16 +31,13 @@ module Shoryuken
|
|
31
31
|
Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
|
32
32
|
begin
|
33
33
|
logger.debug do
|
34
|
-
"Extending message #{
|
35
|
-
"visibility timeout by #{queue_visibility_timeout}s."
|
34
|
+
"Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
|
36
35
|
end
|
37
36
|
|
38
37
|
sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
|
39
38
|
rescue => ex
|
40
39
|
logger.error do
|
41
|
-
|
42
|
-
"#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
|
43
|
-
"visibility timeout. Error: #{ex.message}"
|
40
|
+
"Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
|
44
41
|
end
|
45
42
|
end
|
46
43
|
end
|
@@ -5,25 +5,23 @@ module Shoryuken
|
|
5
5
|
include Util
|
6
6
|
|
7
7
|
def call(worker, queue, sqs_msg, body)
|
8
|
-
|
9
|
-
|
10
|
-
started_at = Time.now
|
8
|
+
begin
|
9
|
+
started_at = Time.now
|
11
10
|
|
12
|
-
|
11
|
+
logger.info { "started at #{started_at}" }
|
13
12
|
|
14
|
-
|
13
|
+
yield
|
15
14
|
|
16
|
-
|
15
|
+
total_time = elapsed(started_at)
|
17
16
|
|
18
|
-
|
19
|
-
|
20
|
-
end
|
21
|
-
|
22
|
-
logger.info { "completed in: #{total_time} ms" }
|
23
|
-
rescue
|
24
|
-
logger.info { "failed in: #{elapsed(started_at)} ms" }
|
25
|
-
raise
|
17
|
+
if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
|
18
|
+
logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
|
26
19
|
end
|
20
|
+
|
21
|
+
logger.info { "completed in: #{total_time} ms" }
|
22
|
+
rescue
|
23
|
+
logger.info { "failed in: #{elapsed(started_at)} ms" }
|
24
|
+
raise
|
27
25
|
end
|
28
26
|
end
|
29
27
|
end
|
data/lib/shoryuken/options.rb
CHANGED
@@ -21,6 +21,7 @@ module Shoryuken
|
|
21
21
|
@@sqs_client_receive_message_opts = {}
|
22
22
|
@@start_callback = nil
|
23
23
|
@@stop_callback = nil
|
24
|
+
@@worker_executor = Worker::DefaultExecutor
|
24
25
|
|
25
26
|
class << self
|
26
27
|
def active_job?
|
@@ -56,6 +57,14 @@ module Shoryuken
|
|
56
57
|
@@worker_registry = worker_registry
|
57
58
|
end
|
58
59
|
|
60
|
+
def worker_executor
|
61
|
+
@@worker_executor
|
62
|
+
end
|
63
|
+
|
64
|
+
def worker_executor=(worker_executor)
|
65
|
+
@@worker_executor = worker_executor
|
66
|
+
end
|
67
|
+
|
59
68
|
def polling_strategy(group)
|
60
69
|
strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
|
61
70
|
|
data/lib/shoryuken/processor.rb
CHANGED
@@ -16,8 +16,10 @@ module Shoryuken
|
|
16
16
|
def process
|
17
17
|
return logger.error { "No worker found for #{queue}" } unless worker
|
18
18
|
|
19
|
-
worker.class
|
20
|
-
worker.
|
19
|
+
Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
|
20
|
+
worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
|
21
|
+
worker.perform(sqs_msg, body)
|
22
|
+
end
|
21
23
|
end
|
22
24
|
rescue Exception => ex
|
23
25
|
logger.error { "Processor failed: #{ex.message}" }
|
@@ -41,25 +43,7 @@ module Shoryuken
|
|
41
43
|
end
|
42
44
|
|
43
45
|
def parse_body(sqs_msg)
|
44
|
-
|
45
|
-
|
46
|
-
case body_parser
|
47
|
-
when :json
|
48
|
-
JSON.parse(sqs_msg.body)
|
49
|
-
when Proc
|
50
|
-
body_parser.call(sqs_msg)
|
51
|
-
when :text, nil
|
52
|
-
sqs_msg.body
|
53
|
-
else
|
54
|
-
if body_parser.respond_to?(:parse)
|
55
|
-
# JSON.parse
|
56
|
-
body_parser.parse(sqs_msg.body)
|
57
|
-
elsif body_parser.respond_to?(:load)
|
58
|
-
# see https://github.com/phstc/shoryuken/pull/91
|
59
|
-
# JSON.load
|
60
|
-
body_parser.load(sqs_msg.body)
|
61
|
-
end
|
62
|
-
end
|
46
|
+
BodyParser.parse(worker_class, sqs_msg)
|
63
47
|
end
|
64
48
|
end
|
65
49
|
end
|
data/lib/shoryuken/version.rb
CHANGED
data/lib/shoryuken/worker.rb
CHANGED
@@ -2,33 +2,16 @@ module Shoryuken
|
|
2
2
|
module Worker
|
3
3
|
def self.included(base)
|
4
4
|
base.extend(ClassMethods)
|
5
|
+
base.shoryuken_class_attribute :shoryuken_options_hash
|
5
6
|
end
|
6
7
|
|
7
8
|
module ClassMethods
|
8
9
|
def perform_async(body, options = {})
|
9
|
-
|
10
|
-
options[:message_attributes]['shoryuken_class'] = {
|
11
|
-
string_value: self.to_s,
|
12
|
-
data_type: 'String'
|
13
|
-
}
|
14
|
-
|
15
|
-
options[:message_body] = body
|
16
|
-
|
17
|
-
queue = options.delete(:queue) || get_shoryuken_options['queue']
|
18
|
-
|
19
|
-
Shoryuken::Client.queues(queue).send_message(options)
|
10
|
+
Shoryuken.worker_executor.perform_async(self, body, options)
|
20
11
|
end
|
21
12
|
|
22
13
|
def perform_in(interval, body, options = {})
|
23
|
-
interval
|
24
|
-
now = Time.now.to_f
|
25
|
-
ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
|
26
|
-
|
27
|
-
delay = (ts - now).ceil
|
28
|
-
|
29
|
-
raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
|
30
|
-
|
31
|
-
perform_async(body, options.merge(delay_seconds: delay))
|
14
|
+
Shoryuken.worker_executor.perform_in(self, interval, body, options)
|
32
15
|
end
|
33
16
|
|
34
17
|
alias_method :perform_at, :perform_in
|
@@ -40,7 +23,7 @@ module Shoryuken
|
|
40
23
|
end
|
41
24
|
|
42
25
|
def shoryuken_options(opts = {})
|
43
|
-
|
26
|
+
self.shoryuken_options_hash = get_shoryuken_options.merge(stringify_keys(opts || {}))
|
44
27
|
normalize_worker_queue!
|
45
28
|
end
|
46
29
|
|
@@ -57,7 +40,7 @@ module Shoryuken
|
|
57
40
|
end
|
58
41
|
|
59
42
|
def get_shoryuken_options # :nodoc:
|
60
|
-
|
43
|
+
shoryuken_options_hash || Shoryuken.default_worker_options
|
61
44
|
end
|
62
45
|
|
63
46
|
def stringify_keys(hash) # :nodoc:
|
@@ -66,23 +49,78 @@ module Shoryuken
|
|
66
49
|
new_hash
|
67
50
|
end
|
68
51
|
|
52
|
+
def shoryuken_class_attribute(*attrs) # :nodoc:
|
53
|
+
attrs.each do |name|
|
54
|
+
singleton_class.instance_eval do
|
55
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
56
|
+
end
|
57
|
+
define_singleton_method(name) { nil }
|
58
|
+
|
59
|
+
ivar = "@#{name}"
|
60
|
+
|
61
|
+
singleton_class.instance_eval do
|
62
|
+
m = "#{name}="
|
63
|
+
undef_method(m) if method_defined?(m) || private_method_defined?(m)
|
64
|
+
end
|
65
|
+
|
66
|
+
define_singleton_method("#{name}=") do |val|
|
67
|
+
singleton_class.class_eval do
|
68
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
69
|
+
define_method(name) { val }
|
70
|
+
end
|
71
|
+
|
72
|
+
# singleton? backwards compatibility for ruby < 2.1
|
73
|
+
singleton_klass = respond_to?(:singleton?) ? singleton? : self != ancestors.first
|
74
|
+
|
75
|
+
if singleton_klass
|
76
|
+
class_eval do
|
77
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
78
|
+
define_method(name) do
|
79
|
+
if instance_variable_defined? ivar
|
80
|
+
instance_variable_get ivar
|
81
|
+
else
|
82
|
+
singleton_class.send name
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
val
|
88
|
+
end
|
89
|
+
|
90
|
+
# instance reader
|
91
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
92
|
+
define_method(name) do
|
93
|
+
if instance_variable_defined?(ivar)
|
94
|
+
instance_variable_get ivar
|
95
|
+
else
|
96
|
+
self.class.public_send name
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
# instance writer
|
101
|
+
m = "#{name}="
|
102
|
+
undef_method(m) if method_defined?(m) || private_method_defined?(m)
|
103
|
+
attr_writer name
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
69
107
|
private
|
70
108
|
|
71
109
|
def normalize_worker_queue!
|
72
|
-
queue =
|
110
|
+
queue = shoryuken_options_hash['queue']
|
73
111
|
if queue.respond_to?(:call)
|
74
112
|
queue = queue.call
|
75
|
-
|
113
|
+
shoryuken_options_hash['queue'] = queue
|
76
114
|
end
|
77
115
|
|
78
|
-
case
|
116
|
+
case shoryuken_options_hash['queue']
|
79
117
|
when Array
|
80
|
-
|
118
|
+
shoryuken_options_hash['queue'].map!(&:to_s)
|
81
119
|
when Symbol
|
82
|
-
|
120
|
+
shoryuken_options_hash['queue'] = shoryuken_options_hash['queue'].to_s
|
83
121
|
end
|
84
122
|
|
85
|
-
[
|
123
|
+
[shoryuken_options_hash['queue']].flatten.compact.each(&method(:register_worker))
|
86
124
|
end
|
87
125
|
|
88
126
|
def register_worker(queue)
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
module Worker
|
3
|
+
class DefaultExecutor
|
4
|
+
class << self
|
5
|
+
def perform_async(worker_class, body, options = {})
|
6
|
+
options[:message_attributes] ||= {}
|
7
|
+
options[:message_attributes]['shoryuken_class'] = {
|
8
|
+
string_value: worker_class.to_s,
|
9
|
+
data_type: 'String'
|
10
|
+
}
|
11
|
+
|
12
|
+
options[:message_body] = body
|
13
|
+
|
14
|
+
queue = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
|
15
|
+
|
16
|
+
Shoryuken::Client.queues(queue).send_message(options)
|
17
|
+
end
|
18
|
+
|
19
|
+
def perform_in(worker_class, interval, body, options = {})
|
20
|
+
interval = interval.to_f
|
21
|
+
now = Time.now.to_f
|
22
|
+
ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
|
23
|
+
|
24
|
+
delay = (ts - now).ceil
|
25
|
+
|
26
|
+
raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
|
27
|
+
|
28
|
+
worker_class.perform_async(body, options.merge(delay_seconds: delay))
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
module Worker
|
3
|
+
class InlineExecutor
|
4
|
+
class << self
|
5
|
+
def perform_async(worker_class, body, options = {})
|
6
|
+
body = JSON.dump(body) if body.is_a?(Hash)
|
7
|
+
|
8
|
+
sqs_msg = OpenStruct.new(
|
9
|
+
body: body,
|
10
|
+
attributes: nil,
|
11
|
+
md5_of_body: nil,
|
12
|
+
md5_of_message_attributes: nil,
|
13
|
+
message_attributes: nil,
|
14
|
+
message_id: nil,
|
15
|
+
receipt_handle: nil,
|
16
|
+
delete: nil
|
17
|
+
)
|
18
|
+
|
19
|
+
worker_class.new.perform(sqs_msg, BodyParser.parse(worker_class, sqs_msg))
|
20
|
+
end
|
21
|
+
|
22
|
+
def perform_in(worker_class, _interval, body, options = {})
|
23
|
+
perform_async(worker_class, body, options)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'shoryuken/body_parser'
|
3
|
+
|
4
|
+
RSpec.describe Shoryuken::BodyParser do
|
5
|
+
let(:sqs_msg) { double }
|
6
|
+
|
7
|
+
describe '#parser' do
|
8
|
+
it 'parses the body into JSON' do
|
9
|
+
TestWorker.get_shoryuken_options['body_parser'] = :json
|
10
|
+
|
11
|
+
body = { 'test' => 'hi' }
|
12
|
+
|
13
|
+
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
14
|
+
|
15
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'parses the body calling the proc' do
|
19
|
+
TestWorker.get_shoryuken_options['body_parser'] = proc { |sqs_msg| "*#{sqs_msg.body}*" }
|
20
|
+
|
21
|
+
allow(sqs_msg).to receive(:body).and_return('test')
|
22
|
+
|
23
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq('*test*')
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'parses the body as text' do
|
27
|
+
TestWorker.get_shoryuken_options['body_parser'] = :text
|
28
|
+
|
29
|
+
body = 'test'
|
30
|
+
|
31
|
+
allow(sqs_msg).to receive(:body).and_return(body)
|
32
|
+
|
33
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq('test')
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'parses calling `.load`' do
|
37
|
+
TestWorker.get_shoryuken_options['body_parser'] = Class.new do
|
38
|
+
def self.load(*args)
|
39
|
+
JSON.load(*args)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
body = { 'test' => 'hi' }
|
44
|
+
|
45
|
+
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
46
|
+
|
47
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'parses calling `.parse`' do
|
51
|
+
TestWorker.get_shoryuken_options['body_parser'] = Class.new do
|
52
|
+
def self.parse(*args)
|
53
|
+
JSON.parse(*args)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
body = { 'test' => 'hi' }
|
58
|
+
|
59
|
+
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
60
|
+
|
61
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
|
62
|
+
end
|
63
|
+
|
64
|
+
context 'when parse errors' do
|
65
|
+
before do
|
66
|
+
TestWorker.get_shoryuken_options['body_parser'] = :json
|
67
|
+
|
68
|
+
allow(sqs_msg).to receive(:body).and_return('invalid JSON')
|
69
|
+
end
|
70
|
+
|
71
|
+
specify do
|
72
|
+
expect { described_class.parse(TestWorker, sqs_msg) }.
|
73
|
+
to raise_error(JSON::ParserError, /unexpected token at 'invalid JSON'/)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
context 'when `object_type: nil`' do
|
78
|
+
it 'parses the body as text' do
|
79
|
+
TestWorker.get_shoryuken_options['body_parser'] = nil
|
80
|
+
|
81
|
+
body = 'test'
|
82
|
+
|
83
|
+
allow(sqs_msg).to receive(:body).and_return(body)
|
84
|
+
|
85
|
+
expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -1,6 +1,4 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
-
require 'shoryuken/processor'
|
3
|
-
require 'shoryuken/manager'
|
4
2
|
|
5
3
|
RSpec.describe Shoryuken::Processor do
|
6
4
|
let(:manager) { double Shoryuken::Manager }
|
@@ -8,12 +6,14 @@ RSpec.describe Shoryuken::Processor do
|
|
8
6
|
let(:queue) { 'default' }
|
9
7
|
|
10
8
|
let(:sqs_msg) do
|
11
|
-
double
|
9
|
+
double(
|
10
|
+
Shoryuken::Message,
|
12
11
|
queue_url: queue,
|
13
12
|
body: 'test',
|
14
13
|
message_attributes: {},
|
15
14
|
message_id: SecureRandom.uuid,
|
16
15
|
receipt_handle: SecureRandom.uuid
|
16
|
+
)
|
17
17
|
end
|
18
18
|
|
19
19
|
before do
|
@@ -25,101 +25,15 @@ RSpec.describe Shoryuken::Processor do
|
|
25
25
|
subject { described_class.new(queue, sqs_msg) }
|
26
26
|
|
27
27
|
describe '#process' do
|
28
|
-
it '
|
29
|
-
|
28
|
+
it 'sets log context' do
|
29
|
+
expect(Shoryuken::Logging).to receive(:with_context).with("TestWorker/#{queue}/#{sqs_msg.message_id}")
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
34
|
-
|
35
|
-
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
31
|
+
allow_any_instance_of(TestWorker).to receive(:perform)
|
32
|
+
allow(sqs_msg).to receive(:body)
|
36
33
|
|
37
34
|
subject.process
|
38
35
|
end
|
39
36
|
|
40
|
-
it 'parses the body calling the proc' do
|
41
|
-
TestWorker.get_shoryuken_options['body_parser'] = proc { |sqs_msg| "*#{sqs_msg.body}*" }
|
42
|
-
|
43
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, '*test*')
|
44
|
-
|
45
|
-
allow(sqs_msg).to receive(:body).and_return('test')
|
46
|
-
|
47
|
-
subject.process
|
48
|
-
end
|
49
|
-
|
50
|
-
it 'parses the body as text' do
|
51
|
-
TestWorker.get_shoryuken_options['body_parser'] = :text
|
52
|
-
|
53
|
-
body = 'test'
|
54
|
-
|
55
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
56
|
-
|
57
|
-
allow(sqs_msg).to receive(:body).and_return(body)
|
58
|
-
|
59
|
-
subject.process
|
60
|
-
end
|
61
|
-
|
62
|
-
it 'parses calling `.load`' do
|
63
|
-
TestWorker.get_shoryuken_options['body_parser'] = Class.new do
|
64
|
-
def self.load(*args)
|
65
|
-
JSON.load(*args)
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
body = { 'test' => 'hi' }
|
70
|
-
|
71
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
72
|
-
|
73
|
-
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
74
|
-
|
75
|
-
subject.process
|
76
|
-
end
|
77
|
-
|
78
|
-
it 'parses calling `.parse`' do
|
79
|
-
TestWorker.get_shoryuken_options['body_parser'] = Class.new do
|
80
|
-
def self.parse(*args)
|
81
|
-
JSON.parse(*args)
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
body = { 'test' => 'hi' }
|
86
|
-
|
87
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
88
|
-
|
89
|
-
allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
|
90
|
-
|
91
|
-
subject.process
|
92
|
-
end
|
93
|
-
|
94
|
-
context 'when parse errors' do
|
95
|
-
before do
|
96
|
-
TestWorker.get_shoryuken_options['body_parser'] = :json
|
97
|
-
|
98
|
-
allow(sqs_msg).to receive(:body).and_return('invalid JSON')
|
99
|
-
end
|
100
|
-
|
101
|
-
specify do
|
102
|
-
expect(subject.logger).to receive(:error).twice
|
103
|
-
|
104
|
-
expect { subject.process }.
|
105
|
-
to raise_error(JSON::ParserError, /unexpected token at 'invalid JSON'/)
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
context 'when `object_type: nil`' do
|
110
|
-
it 'parses the body as text' do
|
111
|
-
TestWorker.get_shoryuken_options['body_parser'] = nil
|
112
|
-
|
113
|
-
body = 'test'
|
114
|
-
|
115
|
-
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
|
116
|
-
|
117
|
-
allow(sqs_msg).to receive(:body).and_return(body)
|
118
|
-
|
119
|
-
subject.process
|
120
|
-
end
|
121
|
-
end
|
122
|
-
|
123
37
|
context 'when custom middleware' do
|
124
38
|
let(:queue) { 'worker_called_middleware' }
|
125
39
|
|
@@ -220,15 +134,19 @@ RSpec.describe Shoryuken::Processor do
|
|
220
134
|
|
221
135
|
context 'when shoryuken_class header' do
|
222
136
|
let(:sqs_msg) do
|
223
|
-
double
|
137
|
+
double(
|
138
|
+
Shoryuken::Message,
|
224
139
|
queue_url: queue,
|
225
140
|
body: 'test',
|
226
141
|
message_attributes: {
|
227
142
|
'shoryuken_class' => {
|
228
143
|
string_value: TestWorker.to_s,
|
229
|
-
data_type: 'String'
|
230
|
-
|
231
|
-
|
144
|
+
data_type: 'String'
|
145
|
+
}
|
146
|
+
},
|
147
|
+
message_id: SecureRandom.uuid,
|
148
|
+
receipt_handle: SecureRandom.uuid
|
149
|
+
)
|
232
150
|
end
|
233
151
|
|
234
152
|
it 'performs without delete' do
|
@@ -0,0 +1,100 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Shoryuken::Worker::DefaultExecutor do
|
4
|
+
let(:sqs_queue) { double 'SQS Queue' }
|
5
|
+
let(:queue) { 'default' }
|
6
|
+
|
7
|
+
before do
|
8
|
+
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '.perform_in' do
|
12
|
+
it 'delays a message' do
|
13
|
+
expect(sqs_queue).to receive(:send_message).with(
|
14
|
+
message_attributes: {
|
15
|
+
'shoryuken_class' => {
|
16
|
+
string_value: TestWorker.to_s,
|
17
|
+
data_type: 'String'
|
18
|
+
}
|
19
|
+
},
|
20
|
+
message_body: 'message',
|
21
|
+
delay_seconds: 60)
|
22
|
+
|
23
|
+
TestWorker.perform_in(60, 'message')
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'raises an exception' do
|
27
|
+
expect {
|
28
|
+
TestWorker.perform_in(901, 'message')
|
29
|
+
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe '.perform_at' do
|
34
|
+
it 'delays a message' do
|
35
|
+
expect(sqs_queue).to receive(:send_message).with(
|
36
|
+
message_attributes: {
|
37
|
+
'shoryuken_class' => {
|
38
|
+
string_value: TestWorker.to_s,
|
39
|
+
data_type: 'String'
|
40
|
+
}
|
41
|
+
},
|
42
|
+
message_body: 'message',
|
43
|
+
delay_seconds: 60)
|
44
|
+
|
45
|
+
TestWorker.perform_in(Time.now + 60, 'message')
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'raises an exception' do
|
49
|
+
expect {
|
50
|
+
TestWorker.perform_in(Time.now + 901, 'message')
|
51
|
+
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe '.perform_async' do
|
56
|
+
it 'enqueues a message' do
|
57
|
+
expect(sqs_queue).to receive(:send_message).with(
|
58
|
+
message_attributes: {
|
59
|
+
'shoryuken_class' => {
|
60
|
+
string_value: TestWorker.to_s,
|
61
|
+
data_type: 'String'
|
62
|
+
}
|
63
|
+
},
|
64
|
+
message_body: 'message')
|
65
|
+
|
66
|
+
TestWorker.perform_async('message')
|
67
|
+
end
|
68
|
+
|
69
|
+
it 'enqueues a message with options' do
|
70
|
+
expect(sqs_queue).to receive(:send_message).with(
|
71
|
+
delay_seconds: 60,
|
72
|
+
message_attributes: {
|
73
|
+
'shoryuken_class' => {
|
74
|
+
string_value: TestWorker.to_s,
|
75
|
+
data_type: 'String'
|
76
|
+
}
|
77
|
+
},
|
78
|
+
message_body: 'delayed message')
|
79
|
+
|
80
|
+
TestWorker.perform_async('delayed message', delay_seconds: 60)
|
81
|
+
end
|
82
|
+
|
83
|
+
it 'accepts an `queue` option' do
|
84
|
+
new_queue = 'some_different_queue'
|
85
|
+
|
86
|
+
expect(Shoryuken::Client).to receive(:queues).with(new_queue).and_return(sqs_queue)
|
87
|
+
|
88
|
+
expect(sqs_queue).to receive(:send_message).with(
|
89
|
+
message_attributes: {
|
90
|
+
'shoryuken_class' => {
|
91
|
+
string_value: TestWorker.to_s,
|
92
|
+
data_type: 'String'
|
93
|
+
}
|
94
|
+
},
|
95
|
+
message_body: 'delayed message')
|
96
|
+
|
97
|
+
TestWorker.perform_async('delayed message', queue: new_queue)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Shoryuken::Worker::InlineExecutor do
|
4
|
+
before do
|
5
|
+
Shoryuken.worker_executor = described_class
|
6
|
+
end
|
7
|
+
|
8
|
+
describe '.perform_async' do
|
9
|
+
specify do
|
10
|
+
expect_any_instance_of(TestWorker).to receive(:perform)
|
11
|
+
|
12
|
+
TestWorker.perform_async('test')
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
describe '.perform_in' do
|
17
|
+
specify do
|
18
|
+
expect_any_instance_of(TestWorker).to receive(:perform)
|
19
|
+
|
20
|
+
TestWorker.perform_in(60, 'test')
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
RSpec.describe
|
3
|
+
RSpec.describe Shoryuken::Worker do
|
4
4
|
let(:sqs_queue) { double 'SQS Queue' }
|
5
5
|
let(:queue) { 'default' }
|
6
6
|
|
@@ -8,96 +8,6 @@ RSpec.describe 'Shoryuken::Worker' do
|
|
8
8
|
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
9
9
|
end
|
10
10
|
|
11
|
-
describe '.perform_in' do
|
12
|
-
it 'delays a message' do
|
13
|
-
expect(sqs_queue).to receive(:send_message).with(
|
14
|
-
message_attributes: {
|
15
|
-
'shoryuken_class' => {
|
16
|
-
string_value: TestWorker.to_s,
|
17
|
-
data_type: 'String'
|
18
|
-
}
|
19
|
-
},
|
20
|
-
message_body: 'message',
|
21
|
-
delay_seconds: 60)
|
22
|
-
|
23
|
-
TestWorker.perform_in(60, 'message')
|
24
|
-
end
|
25
|
-
|
26
|
-
it 'raises an exception' do
|
27
|
-
expect {
|
28
|
-
TestWorker.perform_in(901, 'message')
|
29
|
-
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
describe '.perform_at' do
|
34
|
-
it 'delays a message' do
|
35
|
-
expect(sqs_queue).to receive(:send_message).with(
|
36
|
-
message_attributes: {
|
37
|
-
'shoryuken_class' => {
|
38
|
-
string_value: TestWorker.to_s,
|
39
|
-
data_type: 'String'
|
40
|
-
}
|
41
|
-
},
|
42
|
-
message_body: 'message',
|
43
|
-
delay_seconds: 60)
|
44
|
-
|
45
|
-
TestWorker.perform_in(Time.now + 60, 'message')
|
46
|
-
end
|
47
|
-
|
48
|
-
it 'raises an exception' do
|
49
|
-
expect {
|
50
|
-
TestWorker.perform_in(Time.now + 901, 'message')
|
51
|
-
}.to raise_error 'The maximum allowed delay is 15 minutes'
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe '.perform_async' do
|
56
|
-
it 'enqueues a message' do
|
57
|
-
expect(sqs_queue).to receive(:send_message).with(
|
58
|
-
message_attributes: {
|
59
|
-
'shoryuken_class' => {
|
60
|
-
string_value: TestWorker.to_s,
|
61
|
-
data_type: 'String'
|
62
|
-
}
|
63
|
-
},
|
64
|
-
message_body: 'message')
|
65
|
-
|
66
|
-
TestWorker.perform_async('message')
|
67
|
-
end
|
68
|
-
|
69
|
-
it 'enqueues a message with options' do
|
70
|
-
expect(sqs_queue).to receive(:send_message).with(
|
71
|
-
delay_seconds: 60,
|
72
|
-
message_attributes: {
|
73
|
-
'shoryuken_class' => {
|
74
|
-
string_value: TestWorker.to_s,
|
75
|
-
data_type: 'String'
|
76
|
-
}
|
77
|
-
},
|
78
|
-
message_body: 'delayed message')
|
79
|
-
|
80
|
-
TestWorker.perform_async('delayed message', delay_seconds: 60)
|
81
|
-
end
|
82
|
-
|
83
|
-
it 'accepts an `queue` option' do
|
84
|
-
new_queue = 'some_different_queue'
|
85
|
-
|
86
|
-
expect(Shoryuken::Client).to receive(:queues).with(new_queue).and_return(sqs_queue)
|
87
|
-
|
88
|
-
expect(sqs_queue).to receive(:send_message).with(
|
89
|
-
message_attributes: {
|
90
|
-
'shoryuken_class' => {
|
91
|
-
string_value: TestWorker.to_s,
|
92
|
-
data_type: 'String'
|
93
|
-
}
|
94
|
-
},
|
95
|
-
message_body: 'delayed message')
|
96
|
-
|
97
|
-
TestWorker.perform_async('delayed message', queue: new_queue)
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
11
|
describe '.shoryuken_options' do
|
102
12
|
it 'registers a worker' do
|
103
13
|
expect(Shoryuken.worker_registry.workers('default')).to eq([TestWorker])
|
@@ -178,6 +88,23 @@ RSpec.describe 'Shoryuken::Worker' do
|
|
178
88
|
expect(Shoryuken.worker_registry.workers('symbol_queue2')).to eq([WorkerMultipleSymbolQueues])
|
179
89
|
expect(Shoryuken.worker_registry.workers('symbol_queue3')).to eq([WorkerMultipleSymbolQueues])
|
180
90
|
end
|
91
|
+
|
92
|
+
it 'preserves parent class options' do
|
93
|
+
class ParentWorker
|
94
|
+
include Shoryuken::Worker
|
95
|
+
|
96
|
+
shoryuken_options queue: "myqueue", auto_delete: false
|
97
|
+
end
|
98
|
+
|
99
|
+
class ChildWorker < ParentWorker
|
100
|
+
shoryuken_options auto_delete: true
|
101
|
+
end
|
102
|
+
|
103
|
+
expect(ParentWorker.get_shoryuken_options['queue']).to eq("myqueue")
|
104
|
+
expect(ChildWorker.get_shoryuken_options['queue']).to eq("myqueue")
|
105
|
+
expect(ParentWorker.get_shoryuken_options['auto_delete']).to eq(false)
|
106
|
+
expect(ChildWorker.get_shoryuken_options['auto_delete']).to eq(true)
|
107
|
+
end
|
181
108
|
end
|
182
109
|
|
183
110
|
describe '.server_middleware' do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -147,6 +147,7 @@ files:
|
|
147
147
|
- examples/bootstrap_queues.rb
|
148
148
|
- examples/default_worker.rb
|
149
149
|
- lib/shoryuken.rb
|
150
|
+
- lib/shoryuken/body_parser.rb
|
150
151
|
- lib/shoryuken/client.rb
|
151
152
|
- lib/shoryuken/core_ext.rb
|
152
153
|
- lib/shoryuken/default_worker_registry.rb
|
@@ -173,11 +174,14 @@ files:
|
|
173
174
|
- lib/shoryuken/util.rb
|
174
175
|
- lib/shoryuken/version.rb
|
175
176
|
- lib/shoryuken/worker.rb
|
177
|
+
- lib/shoryuken/worker/default_executor.rb
|
178
|
+
- lib/shoryuken/worker/inline_executor.rb
|
176
179
|
- lib/shoryuken/worker_registry.rb
|
177
180
|
- shoryuken.gemspec
|
178
181
|
- shoryuken.jpg
|
179
182
|
- spec/integration/launcher_spec.rb
|
180
183
|
- spec/shoryuken.yml
|
184
|
+
- spec/shoryuken/body_parser_spec.rb
|
181
185
|
- spec/shoryuken/client_spec.rb
|
182
186
|
- spec/shoryuken/core_ext_spec.rb
|
183
187
|
- spec/shoryuken/default_worker_registry_spec.rb
|
@@ -196,6 +200,8 @@ files:
|
|
196
200
|
- spec/shoryuken/queue_spec.rb
|
197
201
|
- spec/shoryuken/runner_spec.rb
|
198
202
|
- spec/shoryuken/util_spec.rb
|
203
|
+
- spec/shoryuken/worker/default_executor_spec.rb
|
204
|
+
- spec/shoryuken/worker/inline_executor_spec.rb
|
199
205
|
- spec/shoryuken/worker_spec.rb
|
200
206
|
- spec/shoryuken_spec.rb
|
201
207
|
- spec/spec_helper.rb
|
@@ -228,6 +234,7 @@ summary: Shoryuken is a super efficient AWS SQS thread based message processor
|
|
228
234
|
test_files:
|
229
235
|
- spec/integration/launcher_spec.rb
|
230
236
|
- spec/shoryuken.yml
|
237
|
+
- spec/shoryuken/body_parser_spec.rb
|
231
238
|
- spec/shoryuken/client_spec.rb
|
232
239
|
- spec/shoryuken/core_ext_spec.rb
|
233
240
|
- spec/shoryuken/default_worker_registry_spec.rb
|
@@ -246,6 +253,8 @@ test_files:
|
|
246
253
|
- spec/shoryuken/queue_spec.rb
|
247
254
|
- spec/shoryuken/runner_spec.rb
|
248
255
|
- spec/shoryuken/util_spec.rb
|
256
|
+
- spec/shoryuken/worker/default_executor_spec.rb
|
257
|
+
- spec/shoryuken/worker/inline_executor_spec.rb
|
249
258
|
- spec/shoryuken/worker_spec.rb
|
250
259
|
- spec/shoryuken_spec.rb
|
251
260
|
- spec/spec_helper.rb
|