shoryuken 2.0.11 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +20 -0
- data/.rubocop.yml +8 -2
- data/.travis.yml +7 -5
- data/CHANGELOG.md +92 -10
- data/Gemfile +1 -0
- data/README.md +20 -57
- data/Rakefile +0 -1
- data/bin/cli/base.rb +42 -0
- data/bin/cli/sqs.rb +188 -0
- data/bin/shoryuken +47 -9
- data/examples/default_worker.rb +1 -12
- data/lib/shoryuken/client.rb +3 -25
- data/lib/shoryuken/default_worker_registry.rb +9 -5
- data/lib/shoryuken/environment_loader.rb +29 -67
- data/lib/shoryuken/fetcher.rb +22 -53
- data/lib/shoryuken/launcher.rb +5 -29
- data/lib/shoryuken/manager.rb +72 -184
- data/lib/shoryuken/message.rb +4 -13
- data/lib/shoryuken/middleware/chain.rb +1 -18
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +21 -18
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +26 -19
- data/lib/shoryuken/polling.rb +204 -0
- data/lib/shoryuken/processor.rb +6 -14
- data/lib/shoryuken/queue.rb +36 -38
- data/lib/shoryuken/runner.rb +143 -0
- data/lib/shoryuken/util.rb +3 -9
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +1 -1
- data/lib/shoryuken.rb +78 -39
- data/shoryuken.gemspec +6 -6
- data/spec/integration/launcher_spec.rb +4 -3
- data/spec/shoryuken/client_spec.rb +2 -43
- data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
- data/spec/shoryuken/environment_loader_spec.rb +34 -0
- data/spec/shoryuken/fetcher_spec.rb +18 -52
- data/spec/shoryuken/manager_spec.rb +56 -97
- data/spec/shoryuken/middleware/chain_spec.rb +0 -24
- data/spec/shoryuken/middleware/server/auto_delete_spec.rb +2 -2
- data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +7 -3
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +56 -33
- data/spec/shoryuken/polling_spec.rb +239 -0
- data/spec/shoryuken/processor_spec.rb +5 -5
- data/spec/shoryuken/queue_spec.rb +110 -63
- data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +10 -24
- data/spec/shoryuken_spec.rb +13 -1
- data/spec/spec_helper.rb +8 -20
- data/test_workers/endless_interruptive_worker.rb +41 -0
- data/test_workers/endless_uninterruptive_worker.rb +44 -0
- metadata +34 -35
- data/.hound.yml +0 -6
- data/lib/shoryuken/cli.rb +0 -210
- data/lib/shoryuken/sns_arn.rb +0 -27
- data/lib/shoryuken/topic.rb +0 -17
- data/spec/shoryuken/sns_arn_spec.rb +0 -42
- data/spec/shoryuken/topic_spec.rb +0 -32
- data/spec/shoryuken_endpoint.yml +0 -6
- /data/{LICENSE.txt → LICENSE} +0 -0
|
@@ -2,13 +2,21 @@ require 'spec_helper'
|
|
|
2
2
|
|
|
3
3
|
describe Shoryuken::Queue do
|
|
4
4
|
let(:credentials) { Aws::Credentials.new('access_key_id', 'secret_access_key') }
|
|
5
|
-
let(:sqs)
|
|
6
|
-
let(:queue_name)
|
|
7
|
-
let(:queue_url)
|
|
5
|
+
let(:sqs) { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) }
|
|
6
|
+
let(:queue_name) { 'shoryuken' }
|
|
7
|
+
let(:queue_url) { 'https://eu-west-1.amazonaws.com:6059/123456789012/shoryuken' }
|
|
8
8
|
|
|
9
9
|
subject { described_class.new(sqs, queue_name) }
|
|
10
|
+
before {
|
|
11
|
+
# Required as Aws::SQS::Client.get_queue_url returns 'String' when responses are stubbed,
|
|
12
|
+
# which is not accepted by Aws::SQS::Client.get_queue_attributes for :queue_name parameter.
|
|
13
|
+
allow(subject).to receive(:url).and_return(queue_url)
|
|
14
|
+
}
|
|
10
15
|
|
|
11
16
|
describe '#send_message' do
|
|
17
|
+
before {
|
|
18
|
+
allow(subject).to receive(:fifo?).and_return(false)
|
|
19
|
+
}
|
|
12
20
|
it 'accepts SQS request parameters' do
|
|
13
21
|
# https://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#send_message-instance_method
|
|
14
22
|
expect(sqs).to receive(:send_message).with(hash_including(message_body: 'msg1'))
|
|
@@ -22,95 +30,134 @@ describe Shoryuken::Queue do
|
|
|
22
30
|
subject.send_message('msg1')
|
|
23
31
|
end
|
|
24
32
|
|
|
25
|
-
context 'when
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
}.to raise_error(ArgumentError, 'The message body must be a String and you passed a NilClass')
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
it 'raises ArgumentError for Fixnum' do
|
|
33
|
-
expect {
|
|
34
|
-
subject.send_message(message_body: 1)
|
|
35
|
-
}.to raise_error(ArgumentError, 'The message body must be a String and you passed a Fixnum')
|
|
36
|
-
end
|
|
33
|
+
context 'when a client middleware' do
|
|
34
|
+
class MyClientMiddleware
|
|
35
|
+
def call(options)
|
|
36
|
+
options[:message_body] = 'changed'
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
class MyClientMiddleware
|
|
40
|
-
def call(options)
|
|
41
|
-
options[:message_body] = 'changed'
|
|
42
|
-
|
|
43
|
-
yield
|
|
44
|
-
end
|
|
38
|
+
yield
|
|
45
39
|
end
|
|
40
|
+
end
|
|
46
41
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
42
|
+
before do
|
|
43
|
+
allow(Shoryuken).to receive(:server?).and_return(false)
|
|
44
|
+
Shoryuken.configure_client do |config|
|
|
45
|
+
config.client_middleware do |chain|
|
|
46
|
+
chain.add MyClientMiddleware
|
|
52
47
|
end
|
|
53
48
|
end
|
|
49
|
+
end
|
|
54
50
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
end
|
|
51
|
+
after do
|
|
52
|
+
Shoryuken.configure_client do |config|
|
|
53
|
+
config.client_middleware do |chain|
|
|
54
|
+
chain.remove MyClientMiddleware
|
|
60
55
|
end
|
|
61
56
|
end
|
|
57
|
+
end
|
|
62
58
|
|
|
63
|
-
|
|
64
|
-
|
|
59
|
+
it 'invokes MyClientMiddleware' do
|
|
60
|
+
expect(sqs).to receive(:send_message).with(hash_including(message_body: 'changed'))
|
|
65
61
|
|
|
66
|
-
|
|
67
|
-
end
|
|
62
|
+
subject.send_message(message_body: 'original')
|
|
68
63
|
end
|
|
69
64
|
end
|
|
70
65
|
end
|
|
71
66
|
|
|
72
67
|
describe '#send_messages' do
|
|
68
|
+
before {
|
|
69
|
+
allow(subject).to receive(:fifo?).and_return(false)
|
|
70
|
+
}
|
|
73
71
|
it 'accepts SQS request parameters' do
|
|
74
72
|
# https://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#send_message_batch-instance_method
|
|
75
|
-
expect(sqs).to receive(:send_message_batch).with(hash_including(entries: [{
|
|
73
|
+
expect(sqs).to receive(:send_message_batch).with(hash_including(entries: [{id: '0', message_body: 'msg1'}, {id: '1', message_body: 'msg2'}]))
|
|
76
74
|
|
|
77
|
-
subject.send_messages(entries: [{
|
|
75
|
+
subject.send_messages(entries: [{id: '0', message_body: 'msg1'}, {id: '1', message_body: 'msg2'}])
|
|
78
76
|
end
|
|
79
77
|
|
|
80
78
|
it 'accepts an array of messages' do
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
79
|
+
options = { entries: [{ id: '0',
|
|
80
|
+
message_body: 'msg1',
|
|
81
|
+
delay_seconds: 1,
|
|
82
|
+
message_attributes: { attr: 'attr1' } },
|
|
83
|
+
{ id: '1',
|
|
84
|
+
message_body: 'msg2',
|
|
85
|
+
delay_seconds: 1,
|
|
86
|
+
message_attributes: { attr: 'attr2' } }] }
|
|
87
|
+
|
|
88
|
+
expect(sqs).to receive(:send_message_batch).with(hash_including(options))
|
|
89
|
+
|
|
90
|
+
subject.send_messages([{ message_body: 'msg1',
|
|
91
|
+
delay_seconds: 1,
|
|
92
|
+
message_attributes: { attr: 'attr1' }
|
|
93
|
+
}, {
|
|
94
|
+
message_body: 'msg2',
|
|
95
|
+
delay_seconds: 1,
|
|
96
|
+
message_attributes: { attr: 'attr2' }
|
|
97
|
+
}])
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
context 'when FIFO' do
|
|
101
|
+
before do
|
|
102
|
+
allow(subject).to receive(:fifo?).and_return(true)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
context 'and message_group_id and message_deduplication_id are absent' do
|
|
106
|
+
it 'sets default values' do
|
|
107
|
+
expect(sqs).to receive(:send_message_batch) do |arg|
|
|
108
|
+
first_entry = arg[:entries].first
|
|
109
|
+
|
|
110
|
+
expect(first_entry[:message_group_id]).to eq described_class::MESSAGE_GROUP_ID
|
|
111
|
+
expect(first_entry[:message_deduplication_id]).to be
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
subject.send_messages([{ message_body: 'msg1', message_attributes: { attr: 'attr1' } }])
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
context 'and message_group_id and message_deduplication_id are present' do
|
|
119
|
+
it 'preserves existing values' do
|
|
120
|
+
expect(sqs).to receive(:send_message_batch) do |arg|
|
|
121
|
+
first_entry = arg[:entries].first
|
|
122
|
+
|
|
123
|
+
expect(first_entry[:message_group_id]).to eq 'my group'
|
|
124
|
+
expect(first_entry[:message_deduplication_id]).to eq 'my id'
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
subject.send_messages([{ message_body: 'msg1',
|
|
128
|
+
message_attributes: { attr: 'attr1' },
|
|
129
|
+
message_group_id: 'my group',
|
|
130
|
+
message_deduplication_id: 'my id' }])
|
|
131
|
+
end
|
|
132
|
+
end
|
|
94
133
|
end
|
|
95
134
|
|
|
96
135
|
it 'accepts an array of string' do
|
|
97
|
-
expect(sqs).to receive(:send_message_batch).with(hash_including(entries: [{ id: '0', message_body: 'msg1'}, { id: '1', message_body: 'msg2' }]))
|
|
136
|
+
expect(sqs).to receive(:send_message_batch).with(hash_including(entries: [{ id: '0', message_body: 'msg1' }, { id: '1', message_body: 'msg2' }]))
|
|
98
137
|
|
|
99
138
|
subject.send_messages(%w(msg1 msg2))
|
|
100
139
|
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
describe '#fifo?' do
|
|
143
|
+
before do
|
|
144
|
+
attribute_response = double 'Aws::SQS::Types::GetQueueAttributesResponse'
|
|
145
|
+
|
|
146
|
+
allow(attribute_response).to receive(:attributes).and_return('FifoQueue' => fifo.to_s, 'ContentBasedDeduplication' => 'true')
|
|
147
|
+
allow(subject).to receive(:url).and_return(queue_url)
|
|
148
|
+
allow(sqs).to receive(:get_queue_attributes).with(queue_url: queue_url, attribute_names: ['All']).and_return(attribute_response)
|
|
149
|
+
end
|
|
101
150
|
|
|
102
|
-
context 'when
|
|
103
|
-
|
|
104
|
-
expect {
|
|
105
|
-
subject.send_messages(entries: [message_body: nil])
|
|
106
|
-
}.to raise_error(ArgumentError, 'The message body must be a String and you passed a NilClass')
|
|
107
|
-
end
|
|
151
|
+
context 'when queue is FIFO' do
|
|
152
|
+
let(:fifo) { true }
|
|
108
153
|
|
|
109
|
-
it
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
154
|
+
it { expect(subject.fifo?).to be }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
context 'when queue is not FIFO' do
|
|
158
|
+
let(:fifo) { false }
|
|
159
|
+
|
|
160
|
+
it { expect(subject.fifo?).to_not be }
|
|
114
161
|
end
|
|
115
162
|
end
|
|
116
163
|
end
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
|
-
require 'shoryuken/
|
|
2
|
+
require 'shoryuken/runner'
|
|
3
3
|
require 'shoryuken/launcher'
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
# rubocop:disable Metrics/BlockLength
|
|
6
|
+
RSpec.describe Shoryuken::Runner do
|
|
7
|
+
let(:cli) { Shoryuken::Runner.instance }
|
|
7
8
|
|
|
8
|
-
before
|
|
9
|
+
before do
|
|
9
10
|
# make sure we do not bail
|
|
10
11
|
allow(cli).to receive(:exit)
|
|
11
12
|
|
|
@@ -23,43 +24,28 @@ RSpec.describe Shoryuken::CLI do
|
|
|
23
24
|
end
|
|
24
25
|
|
|
25
26
|
it 'does not raise' do
|
|
26
|
-
expect { cli.run(
|
|
27
|
+
expect { cli.run({}) }.to_not raise_error
|
|
27
28
|
end
|
|
28
29
|
|
|
29
30
|
it 'daemonizes with --daemon --logfile' do
|
|
30
|
-
expect(cli).to receive(:celluloid_loaded?).and_return(false)
|
|
31
31
|
expect(Process).to receive(:daemon)
|
|
32
|
-
cli.run(
|
|
32
|
+
cli.run(daemon: true, logfile: '/dev/null')
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
it 'does NOT daemonize with --
|
|
35
|
+
it 'does NOT daemonize with --logfile' do
|
|
36
36
|
expect(Process).to_not receive(:daemon)
|
|
37
|
-
cli.run(
|
|
37
|
+
cli.run(logfile: '/dev/null')
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
it 'writes PID file with --pidfile' do
|
|
41
41
|
pidfile = instance_double('File')
|
|
42
42
|
expect(File).to receive(:open).with('/dev/null', 'w').and_yield(pidfile)
|
|
43
43
|
expect(pidfile).to receive(:puts).with(Process.pid)
|
|
44
|
-
cli.run(
|
|
44
|
+
cli.run(pidfile: '/dev/null')
|
|
45
45
|
end
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
describe '#daemonize' do
|
|
49
|
-
before(:each) do
|
|
50
|
-
allow(cli).to receive(:celluloid_loaded?).and_return(false)
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
it 'raises if logfile is not set' do
|
|
54
|
-
expect { cli.send(:daemonize, daemon: true) }.to raise_error(ArgumentError)
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
it 'raises if Celluloid is already loaded' do
|
|
58
|
-
expect(cli).to receive(:celluloid_loaded?).and_return(true)
|
|
59
|
-
args = { daemon: true, logfile: '/dev/null' }
|
|
60
|
-
expect { cli.send(:daemonize, args) }.to raise_error(RuntimeError)
|
|
61
|
-
end
|
|
62
|
-
|
|
63
49
|
it 'calls Process.daemon' do
|
|
64
50
|
args = { daemon: true, logfile: '/dev/null' }
|
|
65
51
|
expect(Process).to receive(:daemon).with(true, true)
|
data/spec/shoryuken_spec.rb
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
|
-
describe Shoryuken do
|
|
3
|
+
RSpec.describe Shoryuken do
|
|
4
|
+
describe '.add_queue' do
|
|
5
|
+
after { Shoryuken.queues.clear }
|
|
6
|
+
|
|
7
|
+
it 'adds queues' do
|
|
8
|
+
described_class.add_queue('default')
|
|
9
|
+
expect(described_class.queues).to eq(['default'])
|
|
10
|
+
|
|
11
|
+
described_class.add_queue('high', 2)
|
|
12
|
+
expect(described_class.queues).to eq(%w(default high high))
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
4
16
|
describe '.register_worker' do
|
|
5
17
|
it 'registers a worker' do
|
|
6
18
|
described_class.worker_registry.clear
|
data/spec/spec_helper.rb
CHANGED
|
@@ -2,33 +2,21 @@ require 'bundler/setup'
|
|
|
2
2
|
Bundler.setup
|
|
3
3
|
|
|
4
4
|
require 'pry-byebug'
|
|
5
|
-
require 'celluloid'
|
|
6
5
|
require 'shoryuken'
|
|
7
6
|
require 'json'
|
|
8
|
-
require 'multi_xml'
|
|
9
7
|
require 'dotenv'
|
|
10
8
|
Dotenv.load
|
|
11
9
|
|
|
12
10
|
if ENV['CODECLIMATE_REPO_TOKEN']
|
|
13
|
-
require '
|
|
14
|
-
|
|
11
|
+
require 'simplecov'
|
|
12
|
+
SimpleCov.start
|
|
15
13
|
end
|
|
16
14
|
|
|
17
15
|
config_file = File.join(File.expand_path('../..', __FILE__), 'spec', 'shoryuken.yml')
|
|
18
16
|
|
|
19
|
-
Shoryuken::EnvironmentLoader.
|
|
17
|
+
Shoryuken::EnvironmentLoader.setup_options(config_file: config_file)
|
|
20
18
|
|
|
21
19
|
Shoryuken.logger.level = Logger::UNKNOWN
|
|
22
|
-
Celluloid.logger.level = Logger::UNKNOWN
|
|
23
|
-
|
|
24
|
-
# I'm not sure whether this is an issue specific to running Shoryuken against github.com/comcast/cmb
|
|
25
|
-
# as opposed to AWS itself, but sometimes the receive_messages call returns XML that looks like this:
|
|
26
|
-
#
|
|
27
|
-
# <ReceiveMessageResponse>\n\t<ReceiveMessageResult>\n\t</ReceiveMessageResult> ... </ReceiveMessageResponse>
|
|
28
|
-
#
|
|
29
|
-
# The default MultiXML parser is ReXML, which seems to mishandle \n\t chars. Nokogiri seems to be
|
|
30
|
-
# the only one that correctly ignore this whitespace.
|
|
31
|
-
MultiXml.parser = :nokogiri
|
|
32
20
|
|
|
33
21
|
class TestWorker
|
|
34
22
|
include Shoryuken::Worker
|
|
@@ -40,24 +28,24 @@ end
|
|
|
40
28
|
|
|
41
29
|
RSpec.configure do |config|
|
|
42
30
|
# Only run slow tests if SPEC_ALL=true and AWS_ACCESS_KEY_ID is present
|
|
43
|
-
# The AWS_ACCESS_KEY_ID checker is because Travis CI
|
|
31
|
+
# The AWS_ACCESS_KEY_ID checker is because Travis CI
|
|
32
|
+
# does not expose ENV variables to pull requests from forked repositories
|
|
44
33
|
# http://docs.travis-ci.com/user/pull-requests/
|
|
45
34
|
config.filter_run_excluding slow: true if ENV['SPEC_ALL'] != 'true' || ENV['AWS_ACCESS_KEY_ID'].nil?
|
|
46
35
|
|
|
47
36
|
config.before do
|
|
48
37
|
Shoryuken::Client.class_variable_set :@@queues, {}
|
|
49
|
-
Shoryuken::Client.class_variable_set :@@visibility_timeouts, {}
|
|
50
38
|
|
|
51
39
|
Shoryuken::Client.sqs = nil
|
|
52
|
-
Shoryuken::Client.sqs_resource = nil
|
|
53
|
-
Shoryuken::Client.sns = nil
|
|
54
40
|
|
|
55
41
|
Shoryuken.queues.clear
|
|
56
42
|
|
|
57
43
|
Shoryuken.options[:concurrency] = 1
|
|
58
44
|
Shoryuken.options[:delay] = 1
|
|
59
45
|
Shoryuken.options[:timeout] = 1
|
|
60
|
-
Shoryuken.options[:
|
|
46
|
+
Shoryuken.options[:daemon] = nil
|
|
47
|
+
Shoryuken.options[:logfile] = nil
|
|
48
|
+
Shoryuken.options[:queues] = nil
|
|
61
49
|
|
|
62
50
|
TestWorker.get_shoryuken_options.clear
|
|
63
51
|
TestWorker.get_shoryuken_options['queue'] = 'default'
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
class EndlessInterruptiveWorker
|
|
2
|
+
include Shoryuken::Worker
|
|
3
|
+
|
|
4
|
+
# Usage:
|
|
5
|
+
# QUEUE="super-q"
|
|
6
|
+
# MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
|
|
7
|
+
# bundle exec ./bin/shoryuken -r ./examples/endless_uninterruptive_worker.rb -q $QUEUE -c 8
|
|
8
|
+
|
|
9
|
+
class << self
|
|
10
|
+
def queue
|
|
11
|
+
ENV['QUEUE'] || 'default'
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def max_execution_time
|
|
15
|
+
ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def rng
|
|
19
|
+
@rng ||= Random.new
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# returns a random number between 0 and 100
|
|
23
|
+
def random_number(hi = 1000)
|
|
24
|
+
(rng.rand * hi).to_i
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def perform(sqs_msg, body)
|
|
29
|
+
Shoryuken.logger.info("Received message: '#{body}'")
|
|
30
|
+
|
|
31
|
+
execution_ms = self.class.random_number(self.class.max_execution_time)
|
|
32
|
+
Shoryuken.logger.info("Going to sleep for #{execution_ms}ms")
|
|
33
|
+
|
|
34
|
+
new_body = "#{execution_ms}-" + body.to_s
|
|
35
|
+
sleep(execution_ms.to_f / 1000)
|
|
36
|
+
|
|
37
|
+
self.class.perform_async(new_body.slice(0, 512))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
shoryuken_options queue: queue, auto_delete: true
|
|
41
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
class EndlessUninterruptiveWorker
|
|
2
|
+
include Shoryuken::Worker
|
|
3
|
+
|
|
4
|
+
# Usage:
|
|
5
|
+
# QUEUE="super-q"
|
|
6
|
+
# MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
|
|
7
|
+
# bundle exec ./bin/shoryuken -r ./examples/endless_interruptive_worker.rb -q $QUEUE -c 8
|
|
8
|
+
|
|
9
|
+
class << self
|
|
10
|
+
def queue
|
|
11
|
+
ENV['QUEUE'] || 'default'
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def max_execution_time
|
|
15
|
+
ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def rng
|
|
19
|
+
@rng ||= Random.new
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# returns a random number between 0 and 100
|
|
23
|
+
def random_number(hi = 1000)
|
|
24
|
+
(rng.rand * hi).to_i
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def perform(sqs_msg, body)
|
|
29
|
+
Shoryuken.logger.info("Received message: '#{body}'")
|
|
30
|
+
|
|
31
|
+
execution_ms = self.class.random_number(self.class.max_execution_time)
|
|
32
|
+
Shoryuken.logger.info("Going to burn metal for #{execution_ms}ms")
|
|
33
|
+
end_time = Time.now + execution_ms.to_f / 1000
|
|
34
|
+
while Time.now < end_time do
|
|
35
|
+
# burn metal
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
new_body = "#{execution_ms}-" + body.to_s
|
|
39
|
+
|
|
40
|
+
self.class.perform_async(new_body.slice(0, 512))
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
shoryuken_options queue: queue, auto_delete: true
|
|
44
|
+
end
|
metadata
CHANGED
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: shoryuken
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 3.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Pablo Cantero
|
|
8
|
-
- Mario Kostelac
|
|
9
8
|
autorequire:
|
|
10
9
|
bindir: bin
|
|
11
10
|
cert_chain: []
|
|
12
|
-
date:
|
|
11
|
+
date: 2017-03-12 00:00:00.000000000 Z
|
|
13
12
|
dependencies:
|
|
14
13
|
- !ruby/object:Gem::Dependency
|
|
15
14
|
name: bundler
|
|
@@ -68,7 +67,7 @@ dependencies:
|
|
|
68
67
|
- !ruby/object:Gem::Version
|
|
69
68
|
version: '0'
|
|
70
69
|
- !ruby/object:Gem::Dependency
|
|
71
|
-
name:
|
|
70
|
+
name: dotenv
|
|
72
71
|
requirement: !ruby/object:Gem::Requirement
|
|
73
72
|
requirements:
|
|
74
73
|
- - ">="
|
|
@@ -82,71 +81,71 @@ dependencies:
|
|
|
82
81
|
- !ruby/object:Gem::Version
|
|
83
82
|
version: '0'
|
|
84
83
|
- !ruby/object:Gem::Dependency
|
|
85
|
-
name:
|
|
84
|
+
name: aws-sdk-core
|
|
86
85
|
requirement: !ruby/object:Gem::Requirement
|
|
87
86
|
requirements:
|
|
88
|
-
- - "
|
|
87
|
+
- - "~>"
|
|
89
88
|
- !ruby/object:Gem::Version
|
|
90
|
-
version: '
|
|
91
|
-
type: :
|
|
89
|
+
version: '2'
|
|
90
|
+
type: :runtime
|
|
92
91
|
prerelease: false
|
|
93
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
94
93
|
requirements:
|
|
95
|
-
- - "
|
|
94
|
+
- - "~>"
|
|
96
95
|
- !ruby/object:Gem::Version
|
|
97
|
-
version: '
|
|
96
|
+
version: '2'
|
|
98
97
|
- !ruby/object:Gem::Dependency
|
|
99
|
-
name:
|
|
98
|
+
name: concurrent-ruby
|
|
100
99
|
requirement: !ruby/object:Gem::Requirement
|
|
101
100
|
requirements:
|
|
102
|
-
- - "
|
|
101
|
+
- - ">="
|
|
103
102
|
- !ruby/object:Gem::Version
|
|
104
|
-
version: '
|
|
103
|
+
version: '0'
|
|
105
104
|
type: :runtime
|
|
106
105
|
prerelease: false
|
|
107
106
|
version_requirements: !ruby/object:Gem::Requirement
|
|
108
107
|
requirements:
|
|
109
|
-
- - "
|
|
108
|
+
- - ">="
|
|
110
109
|
- !ruby/object:Gem::Version
|
|
111
|
-
version: '
|
|
110
|
+
version: '0'
|
|
112
111
|
- !ruby/object:Gem::Dependency
|
|
113
|
-
name:
|
|
112
|
+
name: thor
|
|
114
113
|
requirement: !ruby/object:Gem::Requirement
|
|
115
114
|
requirements:
|
|
116
|
-
- - "
|
|
115
|
+
- - ">="
|
|
117
116
|
- !ruby/object:Gem::Version
|
|
118
|
-
version: '0
|
|
117
|
+
version: '0'
|
|
119
118
|
type: :runtime
|
|
120
119
|
prerelease: false
|
|
121
120
|
version_requirements: !ruby/object:Gem::Requirement
|
|
122
121
|
requirements:
|
|
123
|
-
- - "
|
|
122
|
+
- - ">="
|
|
124
123
|
- !ruby/object:Gem::Version
|
|
125
|
-
version: '0
|
|
124
|
+
version: '0'
|
|
126
125
|
description: Shoryuken is a super efficient AWS SQS thread based message processor
|
|
127
126
|
email:
|
|
128
127
|
- pablo@pablocantero.com
|
|
129
|
-
- mariokostelac@gmail.com
|
|
130
128
|
executables:
|
|
131
129
|
- shoryuken
|
|
132
130
|
extensions: []
|
|
133
131
|
extra_rdoc_files: []
|
|
134
132
|
files:
|
|
133
|
+
- ".codeclimate.yml"
|
|
135
134
|
- ".gitignore"
|
|
136
|
-
- ".hound.yml"
|
|
137
135
|
- ".rspec"
|
|
138
136
|
- ".rubocop.yml"
|
|
139
137
|
- ".travis.yml"
|
|
140
138
|
- CHANGELOG.md
|
|
141
139
|
- Gemfile
|
|
142
|
-
- LICENSE
|
|
140
|
+
- LICENSE
|
|
143
141
|
- README.md
|
|
144
142
|
- Rakefile
|
|
143
|
+
- bin/cli/base.rb
|
|
144
|
+
- bin/cli/sqs.rb
|
|
145
145
|
- bin/shoryuken
|
|
146
146
|
- examples/bootstrap_queues.rb
|
|
147
147
|
- examples/default_worker.rb
|
|
148
148
|
- lib/shoryuken.rb
|
|
149
|
-
- lib/shoryuken/cli.rb
|
|
150
149
|
- lib/shoryuken/client.rb
|
|
151
150
|
- lib/shoryuken/core_ext.rb
|
|
152
151
|
- lib/shoryuken/default_worker_registry.rb
|
|
@@ -163,10 +162,10 @@ files:
|
|
|
163
162
|
- lib/shoryuken/middleware/server/auto_extend_visibility.rb
|
|
164
163
|
- lib/shoryuken/middleware/server/exponential_backoff_retry.rb
|
|
165
164
|
- lib/shoryuken/middleware/server/timing.rb
|
|
165
|
+
- lib/shoryuken/polling.rb
|
|
166
166
|
- lib/shoryuken/processor.rb
|
|
167
167
|
- lib/shoryuken/queue.rb
|
|
168
|
-
- lib/shoryuken/
|
|
169
|
-
- lib/shoryuken/topic.rb
|
|
168
|
+
- lib/shoryuken/runner.rb
|
|
170
169
|
- lib/shoryuken/util.rb
|
|
171
170
|
- lib/shoryuken/version.rb
|
|
172
171
|
- lib/shoryuken/worker.rb
|
|
@@ -175,10 +174,10 @@ files:
|
|
|
175
174
|
- shoryuken.jpg
|
|
176
175
|
- spec/integration/launcher_spec.rb
|
|
177
176
|
- spec/shoryuken.yml
|
|
178
|
-
- spec/shoryuken/cli_spec.rb
|
|
179
177
|
- spec/shoryuken/client_spec.rb
|
|
180
178
|
- spec/shoryuken/core_ext_spec.rb
|
|
181
179
|
- spec/shoryuken/default_worker_registry_spec.rb
|
|
180
|
+
- spec/shoryuken/environment_loader_spec.rb
|
|
182
181
|
- spec/shoryuken/fetcher_spec.rb
|
|
183
182
|
- spec/shoryuken/manager_spec.rb
|
|
184
183
|
- spec/shoryuken/middleware/chain_spec.rb
|
|
@@ -186,15 +185,16 @@ files:
|
|
|
186
185
|
- spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
|
|
187
186
|
- spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
|
|
188
187
|
- spec/shoryuken/middleware/server/timing_spec.rb
|
|
188
|
+
- spec/shoryuken/polling_spec.rb
|
|
189
189
|
- spec/shoryuken/processor_spec.rb
|
|
190
190
|
- spec/shoryuken/queue_spec.rb
|
|
191
|
-
- spec/shoryuken/
|
|
192
|
-
- spec/shoryuken/topic_spec.rb
|
|
191
|
+
- spec/shoryuken/runner_spec.rb
|
|
193
192
|
- spec/shoryuken/util_spec.rb
|
|
194
193
|
- spec/shoryuken/worker_spec.rb
|
|
195
|
-
- spec/shoryuken_endpoint.yml
|
|
196
194
|
- spec/shoryuken_spec.rb
|
|
197
195
|
- spec/spec_helper.rb
|
|
196
|
+
- test_workers/endless_interruptive_worker.rb
|
|
197
|
+
- test_workers/endless_uninterruptive_worker.rb
|
|
198
198
|
homepage: https://github.com/phstc/shoryuken
|
|
199
199
|
licenses:
|
|
200
200
|
- LGPL-3.0
|
|
@@ -215,17 +215,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
215
215
|
version: '0'
|
|
216
216
|
requirements: []
|
|
217
217
|
rubyforge_project:
|
|
218
|
-
rubygems_version: 2.5.
|
|
218
|
+
rubygems_version: 2.5.2
|
|
219
219
|
signing_key:
|
|
220
220
|
specification_version: 4
|
|
221
221
|
summary: Shoryuken is a super efficient AWS SQS thread based message processor
|
|
222
222
|
test_files:
|
|
223
223
|
- spec/integration/launcher_spec.rb
|
|
224
224
|
- spec/shoryuken.yml
|
|
225
|
-
- spec/shoryuken/cli_spec.rb
|
|
226
225
|
- spec/shoryuken/client_spec.rb
|
|
227
226
|
- spec/shoryuken/core_ext_spec.rb
|
|
228
227
|
- spec/shoryuken/default_worker_registry_spec.rb
|
|
228
|
+
- spec/shoryuken/environment_loader_spec.rb
|
|
229
229
|
- spec/shoryuken/fetcher_spec.rb
|
|
230
230
|
- spec/shoryuken/manager_spec.rb
|
|
231
231
|
- spec/shoryuken/middleware/chain_spec.rb
|
|
@@ -233,12 +233,11 @@ test_files:
|
|
|
233
233
|
- spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
|
|
234
234
|
- spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
|
|
235
235
|
- spec/shoryuken/middleware/server/timing_spec.rb
|
|
236
|
+
- spec/shoryuken/polling_spec.rb
|
|
236
237
|
- spec/shoryuken/processor_spec.rb
|
|
237
238
|
- spec/shoryuken/queue_spec.rb
|
|
238
|
-
- spec/shoryuken/
|
|
239
|
-
- spec/shoryuken/topic_spec.rb
|
|
239
|
+
- spec/shoryuken/runner_spec.rb
|
|
240
240
|
- spec/shoryuken/util_spec.rb
|
|
241
241
|
- spec/shoryuken/worker_spec.rb
|
|
242
|
-
- spec/shoryuken_endpoint.yml
|
|
243
242
|
- spec/shoryuken_spec.rb
|
|
244
243
|
- spec/spec_helper.rb
|