shoryuken 2.1.3 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +2 -0
- data/.rubocop.yml +8 -2
- data/.travis.yml +1 -0
- data/CHANGELOG.md +19 -0
- data/README.md +20 -104
- data/Rakefile +0 -1
- data/bin/cli/base.rb +42 -0
- data/bin/cli/sqs.rb +188 -0
- data/bin/shoryuken +47 -9
- data/examples/default_worker.rb +1 -1
- data/lib/shoryuken.rb +75 -55
- data/lib/shoryuken/client.rb +3 -15
- data/lib/shoryuken/default_worker_registry.rb +9 -5
- data/lib/shoryuken/environment_loader.rb +9 -40
- data/lib/shoryuken/fetcher.rb +16 -18
- data/lib/shoryuken/launcher.rb +5 -28
- data/lib/shoryuken/manager.rb +60 -140
- data/lib/shoryuken/message.rb +4 -13
- data/lib/shoryuken/middleware/chain.rb +1 -18
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +7 -16
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +25 -21
- data/lib/shoryuken/polling.rb +2 -4
- data/lib/shoryuken/processor.rb +2 -11
- data/lib/shoryuken/queue.rb +1 -3
- data/lib/shoryuken/runner.rb +143 -0
- data/lib/shoryuken/util.rb +0 -8
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +1 -1
- data/shoryuken.gemspec +6 -5
- data/spec/integration/launcher_spec.rb +4 -3
- data/spec/shoryuken/client_spec.rb +2 -45
- data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
- data/spec/shoryuken/environment_loader_spec.rb +34 -0
- data/spec/shoryuken/manager_spec.rb +11 -21
- data/spec/shoryuken/middleware/chain_spec.rb +0 -24
- data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +0 -2
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +46 -29
- data/spec/shoryuken/processor_spec.rb +5 -5
- data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +8 -22
- data/spec/shoryuken_spec.rb +13 -1
- data/spec/spec_helper.rb +3 -8
- metadata +29 -22
- data/lib/shoryuken/aws_config.rb +0 -64
- data/lib/shoryuken/cli.rb +0 -215
- data/lib/shoryuken/sns_arn.rb +0 -27
- data/lib/shoryuken/topic.rb +0 -17
- data/spec/shoryuken/sns_arn_spec.rb +0 -42
- data/spec/shoryuken/topic_spec.rb +0 -32
- data/spec/shoryuken_endpoint.yml +0 -6
data/lib/shoryuken/aws_config.rb
DELETED
@@ -1,64 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
module Shoryuken
|
3
|
-
class AwsConfig
|
4
|
-
class << self
|
5
|
-
attr_writer :options
|
6
|
-
|
7
|
-
def options
|
8
|
-
@options ||= {}
|
9
|
-
end
|
10
|
-
|
11
|
-
def setup(hash)
|
12
|
-
# aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
|
13
|
-
# when not explicit supplied
|
14
|
-
return if hash.empty?
|
15
|
-
|
16
|
-
self.options = hash
|
17
|
-
|
18
|
-
shoryuken_keys = %w(
|
19
|
-
account_id
|
20
|
-
sns_endpoint
|
21
|
-
sqs_endpoint
|
22
|
-
receive_message
|
23
|
-
).map(&:to_sym)
|
24
|
-
|
25
|
-
@aws_options = hash.reject do |k, _|
|
26
|
-
shoryuken_keys.include?(k)
|
27
|
-
end
|
28
|
-
|
29
|
-
# assume credentials based authentication
|
30
|
-
credentials = Aws::Credentials.new(
|
31
|
-
@aws_options.delete(:access_key_id),
|
32
|
-
@aws_options.delete(:secret_access_key)
|
33
|
-
)
|
34
|
-
|
35
|
-
# but only if the configuration options have valid values
|
36
|
-
@aws_options.merge!(credentials: credentials) if credentials.set?
|
37
|
-
|
38
|
-
if (callback = Shoryuken.aws_initialization_callback)
|
39
|
-
Shoryuken.logger.info { 'Calling Shoryuken.on_aws_initialization block' }
|
40
|
-
callback.call(@aws_options)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
def sns
|
45
|
-
Aws::SNS::Client.new(aws_client_options(:sns_endpoint))
|
46
|
-
end
|
47
|
-
|
48
|
-
def sqs
|
49
|
-
Aws::SQS::Client.new(aws_client_options(:sqs_endpoint))
|
50
|
-
end
|
51
|
-
|
52
|
-
private
|
53
|
-
|
54
|
-
def aws_client_options(service_endpoint_key)
|
55
|
-
environment_endpoint = ENV["AWS_#{service_endpoint_key.to_s.upcase}"]
|
56
|
-
explicit_endpoint = options[service_endpoint_key] || environment_endpoint
|
57
|
-
endpoint = {}.tap do |hash|
|
58
|
-
hash[:endpoint] = explicit_endpoint unless explicit_endpoint.to_s.empty?
|
59
|
-
end
|
60
|
-
@aws_options.to_h.merge(endpoint)
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
data/lib/shoryuken/cli.rb
DELETED
@@ -1,215 +0,0 @@
|
|
1
|
-
$stdout.sync = true
|
2
|
-
|
3
|
-
require 'singleton'
|
4
|
-
require 'optparse'
|
5
|
-
require 'erb'
|
6
|
-
|
7
|
-
require 'shoryuken'
|
8
|
-
|
9
|
-
module Shoryuken
|
10
|
-
# See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
|
11
|
-
class Shutdown < Interrupt; end
|
12
|
-
|
13
|
-
class CLI
|
14
|
-
include Util
|
15
|
-
include Singleton
|
16
|
-
|
17
|
-
attr_accessor :launcher
|
18
|
-
|
19
|
-
def run(args)
|
20
|
-
self_read, self_write = IO.pipe
|
21
|
-
|
22
|
-
%w(INT TERM USR1 USR2 TTIN).each do |sig|
|
23
|
-
begin
|
24
|
-
trap sig do
|
25
|
-
self_write.puts(sig)
|
26
|
-
end
|
27
|
-
rescue ArgumentError
|
28
|
-
puts "Signal #{sig} not supported"
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
options = parse_cli_args(args)
|
33
|
-
|
34
|
-
loader = EnvironmentLoader.setup_options(options)
|
35
|
-
|
36
|
-
# When cli args exist, override options in config file
|
37
|
-
Shoryuken.options.merge!(options)
|
38
|
-
|
39
|
-
daemonize(Shoryuken.options)
|
40
|
-
write_pid(Shoryuken.options)
|
41
|
-
|
42
|
-
loader.load
|
43
|
-
|
44
|
-
load_celluloid
|
45
|
-
|
46
|
-
require 'shoryuken/launcher'
|
47
|
-
@launcher = Shoryuken::Launcher.new
|
48
|
-
|
49
|
-
if callback = Shoryuken.start_callback
|
50
|
-
logger.info { 'Calling Shoryuken.on_start block' }
|
51
|
-
callback.call
|
52
|
-
end
|
53
|
-
|
54
|
-
fire_event(:startup)
|
55
|
-
|
56
|
-
begin
|
57
|
-
launcher.run
|
58
|
-
|
59
|
-
while (readable_io = IO.select([self_read]))
|
60
|
-
signal = readable_io.first[0].gets.strip
|
61
|
-
handle_signal(signal)
|
62
|
-
end
|
63
|
-
rescue Interrupt
|
64
|
-
launcher.stop(shutdown: true)
|
65
|
-
exit 0
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
private
|
70
|
-
|
71
|
-
def load_celluloid
|
72
|
-
require 'celluloid/current'
|
73
|
-
Celluloid.logger = (Shoryuken.options[:verbose] ? Shoryuken.logger : nil)
|
74
|
-
|
75
|
-
require 'shoryuken/manager'
|
76
|
-
end
|
77
|
-
|
78
|
-
def celluloid_loaded?
|
79
|
-
defined?(::Celluloid)
|
80
|
-
end
|
81
|
-
|
82
|
-
def daemonize(options)
|
83
|
-
return unless options[:daemon]
|
84
|
-
|
85
|
-
fail ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
86
|
-
|
87
|
-
if celluloid_loaded?
|
88
|
-
# Celluloid can't be loaded until after we've daemonized
|
89
|
-
# because it spins up threads and creates locks which get
|
90
|
-
# into a very bad state if forked.
|
91
|
-
raise "Celluloid cannot be required until here, or it will break Shoryuken's daemonization"
|
92
|
-
end
|
93
|
-
|
94
|
-
files_to_reopen = []
|
95
|
-
ObjectSpace.each_object(File) do |file|
|
96
|
-
files_to_reopen << file unless file.closed?
|
97
|
-
end
|
98
|
-
|
99
|
-
Process.daemon(true, true)
|
100
|
-
|
101
|
-
files_to_reopen.each do |file|
|
102
|
-
begin
|
103
|
-
file.reopen file.path, 'a+'
|
104
|
-
file.sync = true
|
105
|
-
rescue ::Exception
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
[$stdout, $stderr].each do |io|
|
110
|
-
File.open(options[:logfile], 'ab') do |f|
|
111
|
-
io.reopen(f)
|
112
|
-
end
|
113
|
-
io.sync = true
|
114
|
-
end
|
115
|
-
$stdin.reopen('/dev/null')
|
116
|
-
end
|
117
|
-
|
118
|
-
def write_pid(options)
|
119
|
-
if (path = options[:pidfile])
|
120
|
-
File.open(path, 'w') do |f|
|
121
|
-
f.puts Process.pid
|
122
|
-
end
|
123
|
-
end
|
124
|
-
end
|
125
|
-
|
126
|
-
def parse_cli_args(argv)
|
127
|
-
opts = {}
|
128
|
-
|
129
|
-
@parser = OptionParser.new do |o|
|
130
|
-
o.on '-c', '--concurrency INT', 'Processor threads to use' do |arg|
|
131
|
-
opts[:concurrency] = Integer(arg)
|
132
|
-
end
|
133
|
-
|
134
|
-
o.on '-d', '--daemon', 'Daemonize process' do |arg|
|
135
|
-
opts[:daemon] = arg
|
136
|
-
end
|
137
|
-
|
138
|
-
o.on '-q', '--queue QUEUE[,WEIGHT]...', 'Queues to process with optional weights' do |arg|
|
139
|
-
queue, weight = arg.split(',')
|
140
|
-
opts[:queues] = [] unless opts[:queues]
|
141
|
-
opts[:queues] << [queue, weight]
|
142
|
-
end
|
143
|
-
|
144
|
-
o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
|
145
|
-
opts[:require] = arg
|
146
|
-
end
|
147
|
-
|
148
|
-
o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
|
149
|
-
opts[:config_file] = arg
|
150
|
-
end
|
151
|
-
|
152
|
-
o.on '-R', '--rails', 'Load Rails' do |arg|
|
153
|
-
opts[:rails] = arg
|
154
|
-
end
|
155
|
-
|
156
|
-
o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
|
157
|
-
opts[:logfile] = arg
|
158
|
-
end
|
159
|
-
|
160
|
-
o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
|
161
|
-
opts[:pidfile] = arg
|
162
|
-
end
|
163
|
-
|
164
|
-
o.on '-v', '--verbose', 'Print more verbose output' do |arg|
|
165
|
-
opts[:verbose] = arg
|
166
|
-
end
|
167
|
-
|
168
|
-
o.on '-V', '--version', 'Print version and exit' do
|
169
|
-
puts "Shoryuken #{Shoryuken::VERSION}"
|
170
|
-
exit 0
|
171
|
-
end
|
172
|
-
end
|
173
|
-
|
174
|
-
@parser.banner = 'shoryuken [options]'
|
175
|
-
@parser.on_tail '-h', '--help', 'Show help' do
|
176
|
-
logger.info { @parser }
|
177
|
-
exit 1
|
178
|
-
end
|
179
|
-
@parser.parse!(argv)
|
180
|
-
opts
|
181
|
-
end
|
182
|
-
|
183
|
-
def handle_signal(sig)
|
184
|
-
logger.info { "Got #{sig} signal" }
|
185
|
-
|
186
|
-
case sig
|
187
|
-
when 'USR1'
|
188
|
-
logger.info { 'Received USR1, will soft shutdown down' }
|
189
|
-
|
190
|
-
launcher.stop
|
191
|
-
fire_event(:quiet, true)
|
192
|
-
exit 0
|
193
|
-
when 'TTIN'
|
194
|
-
Thread.list.each do |thread|
|
195
|
-
logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
|
196
|
-
if thread.backtrace
|
197
|
-
logger.info { thread.backtrace.join("\n") }
|
198
|
-
else
|
199
|
-
logger.info { '<no backtrace available>' }
|
200
|
-
end
|
201
|
-
end
|
202
|
-
|
203
|
-
ready = launcher.manager.instance_variable_get(:@ready).size
|
204
|
-
busy = launcher.manager.instance_variable_get(:@busy).size
|
205
|
-
queues = launcher.manager.instance_variable_get(:@queues)
|
206
|
-
|
207
|
-
logger.info { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{unparse_queues(queues)}" }
|
208
|
-
else
|
209
|
-
logger.info { "Received #{sig}, will shutdown down" }
|
210
|
-
|
211
|
-
raise Interrupt
|
212
|
-
end
|
213
|
-
end
|
214
|
-
end
|
215
|
-
end
|
data/lib/shoryuken/sns_arn.rb
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
module Shoryuken
|
2
|
-
class SnsArn
|
3
|
-
def initialize(topic)
|
4
|
-
@topic = topic
|
5
|
-
end
|
6
|
-
|
7
|
-
def to_s
|
8
|
-
@arn ||= "arn:aws:sns:#{region}:#{account_id}:#{@topic}"
|
9
|
-
end
|
10
|
-
|
11
|
-
private
|
12
|
-
|
13
|
-
def account_id
|
14
|
-
Shoryuken::Client.account_id.tap do |account_id|
|
15
|
-
if account_id.to_s.empty?
|
16
|
-
fail 'To generate SNS ARNs, you must assign an :account_id in your Shoryuken::Client.'
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
def region
|
22
|
-
Aws.config.fetch(:region) do
|
23
|
-
fail 'To generate SNS ARNs, you must include a :region in your AWS config.'
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
data/lib/shoryuken/topic.rb
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
module Shoryuken
|
2
|
-
class Topic
|
3
|
-
def initialize(name, sns)
|
4
|
-
@name, @sns = name, sns
|
5
|
-
end
|
6
|
-
|
7
|
-
def arn
|
8
|
-
@arn ||= Client.sns_arn.new(@name).to_s
|
9
|
-
end
|
10
|
-
|
11
|
-
def send_message(body, options = {})
|
12
|
-
body = JSON.dump(body) if body.is_a?(Hash)
|
13
|
-
|
14
|
-
@sns.publish(topic_arn: arn, message: body)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Shoryuken::SnsArn do
|
4
|
-
let(:account_id) { '1234567890' }
|
5
|
-
let(:region) { 'eu-west-1' }
|
6
|
-
let(:topic) { 'topic-x' }
|
7
|
-
|
8
|
-
before do
|
9
|
-
Shoryuken::Client.account_id = account_id
|
10
|
-
Aws.config = { region: region }
|
11
|
-
end
|
12
|
-
|
13
|
-
subject { described_class.new(topic).to_s }
|
14
|
-
|
15
|
-
describe '#to_s' do
|
16
|
-
context 'when the Aws config includes all the information necessary' do
|
17
|
-
it 'generates an SNS arn' do
|
18
|
-
expect(subject).to eq('arn:aws:sns:eu-west-1:1234567890:topic-x')
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
context 'when the Aws config does not include the account id' do
|
23
|
-
before do
|
24
|
-
Shoryuken::Client.account_id = nil
|
25
|
-
end
|
26
|
-
|
27
|
-
it 'fails' do
|
28
|
-
expect { subject }.to raise_error(/an :account_id/)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
context 'when the Aws config does not include the region' do
|
33
|
-
before do
|
34
|
-
Aws.config.delete :region
|
35
|
-
end
|
36
|
-
|
37
|
-
it 'fails' do
|
38
|
-
expect { subject }.to raise_error(/a :region/)
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
@@ -1,32 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe Shoryuken::Topic do
|
4
|
-
let(:sns) { Aws::SNS::Client.new stub_responses: true }
|
5
|
-
let(:topic_arn) { 'arn:aws:sns:us-east-1:0987654321:shoryuken' }
|
6
|
-
let(:topic_name) { 'shoryuken' }
|
7
|
-
|
8
|
-
before do
|
9
|
-
Shoryuken::Client.account_id = '0987654321'
|
10
|
-
Aws.config = { region: 'us-east-1' }
|
11
|
-
end
|
12
|
-
|
13
|
-
subject { described_class.new(topic_name, sns) }
|
14
|
-
|
15
|
-
describe '#send_message' do
|
16
|
-
it 'enqueues a message' do
|
17
|
-
sns.stub_responses(:publish, { message_id: 'msg1' })
|
18
|
-
expect(sns).to receive(:publish).with(topic_arn: topic_arn, message: 'test')
|
19
|
-
|
20
|
-
subject.send_message('test')
|
21
|
-
end
|
22
|
-
|
23
|
-
it 'parses as JSON by default' do
|
24
|
-
msg = { field: 'test', other_field: 'other' }
|
25
|
-
|
26
|
-
sns.stub_responses(:publish, { message_id: 'msg2' })
|
27
|
-
expect(sns).to receive(:publish).with(topic_arn: topic_arn, message: JSON.dump(msg))
|
28
|
-
|
29
|
-
subject.send_message(msg)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|