shoryuken 2.1.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +2 -0
  3. data/.rubocop.yml +8 -2
  4. data/.travis.yml +1 -0
  5. data/CHANGELOG.md +19 -0
  6. data/README.md +20 -104
  7. data/Rakefile +0 -1
  8. data/bin/cli/base.rb +42 -0
  9. data/bin/cli/sqs.rb +188 -0
  10. data/bin/shoryuken +47 -9
  11. data/examples/default_worker.rb +1 -1
  12. data/lib/shoryuken.rb +75 -55
  13. data/lib/shoryuken/client.rb +3 -15
  14. data/lib/shoryuken/default_worker_registry.rb +9 -5
  15. data/lib/shoryuken/environment_loader.rb +9 -40
  16. data/lib/shoryuken/fetcher.rb +16 -18
  17. data/lib/shoryuken/launcher.rb +5 -28
  18. data/lib/shoryuken/manager.rb +60 -140
  19. data/lib/shoryuken/message.rb +4 -13
  20. data/lib/shoryuken/middleware/chain.rb +1 -18
  21. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +7 -16
  22. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +25 -21
  23. data/lib/shoryuken/polling.rb +2 -4
  24. data/lib/shoryuken/processor.rb +2 -11
  25. data/lib/shoryuken/queue.rb +1 -3
  26. data/lib/shoryuken/runner.rb +143 -0
  27. data/lib/shoryuken/util.rb +0 -8
  28. data/lib/shoryuken/version.rb +1 -1
  29. data/lib/shoryuken/worker.rb +1 -1
  30. data/shoryuken.gemspec +6 -5
  31. data/spec/integration/launcher_spec.rb +4 -3
  32. data/spec/shoryuken/client_spec.rb +2 -45
  33. data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
  34. data/spec/shoryuken/environment_loader_spec.rb +34 -0
  35. data/spec/shoryuken/manager_spec.rb +11 -21
  36. data/spec/shoryuken/middleware/chain_spec.rb +0 -24
  37. data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +0 -2
  38. data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +46 -29
  39. data/spec/shoryuken/processor_spec.rb +5 -5
  40. data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +8 -22
  41. data/spec/shoryuken_spec.rb +13 -1
  42. data/spec/spec_helper.rb +3 -8
  43. metadata +29 -22
  44. data/lib/shoryuken/aws_config.rb +0 -64
  45. data/lib/shoryuken/cli.rb +0 -215
  46. data/lib/shoryuken/sns_arn.rb +0 -27
  47. data/lib/shoryuken/topic.rb +0 -17
  48. data/spec/shoryuken/sns_arn_spec.rb +0 -42
  49. data/spec/shoryuken/topic_spec.rb +0 -32
  50. data/spec/shoryuken_endpoint.yml +0 -6
@@ -1,64 +0,0 @@
1
- # frozen_string_literal: true
2
- module Shoryuken
3
- class AwsConfig
4
- class << self
5
- attr_writer :options
6
-
7
- def options
8
- @options ||= {}
9
- end
10
-
11
- def setup(hash)
12
- # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
13
- # when not explicit supplied
14
- return if hash.empty?
15
-
16
- self.options = hash
17
-
18
- shoryuken_keys = %w(
19
- account_id
20
- sns_endpoint
21
- sqs_endpoint
22
- receive_message
23
- ).map(&:to_sym)
24
-
25
- @aws_options = hash.reject do |k, _|
26
- shoryuken_keys.include?(k)
27
- end
28
-
29
- # assume credentials based authentication
30
- credentials = Aws::Credentials.new(
31
- @aws_options.delete(:access_key_id),
32
- @aws_options.delete(:secret_access_key)
33
- )
34
-
35
- # but only if the configuration options have valid values
36
- @aws_options.merge!(credentials: credentials) if credentials.set?
37
-
38
- if (callback = Shoryuken.aws_initialization_callback)
39
- Shoryuken.logger.info { 'Calling Shoryuken.on_aws_initialization block' }
40
- callback.call(@aws_options)
41
- end
42
- end
43
-
44
- def sns
45
- Aws::SNS::Client.new(aws_client_options(:sns_endpoint))
46
- end
47
-
48
- def sqs
49
- Aws::SQS::Client.new(aws_client_options(:sqs_endpoint))
50
- end
51
-
52
- private
53
-
54
- def aws_client_options(service_endpoint_key)
55
- environment_endpoint = ENV["AWS_#{service_endpoint_key.to_s.upcase}"]
56
- explicit_endpoint = options[service_endpoint_key] || environment_endpoint
57
- endpoint = {}.tap do |hash|
58
- hash[:endpoint] = explicit_endpoint unless explicit_endpoint.to_s.empty?
59
- end
60
- @aws_options.to_h.merge(endpoint)
61
- end
62
- end
63
- end
64
- end
data/lib/shoryuken/cli.rb DELETED
@@ -1,215 +0,0 @@
1
- $stdout.sync = true
2
-
3
- require 'singleton'
4
- require 'optparse'
5
- require 'erb'
6
-
7
- require 'shoryuken'
8
-
9
- module Shoryuken
10
- # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
11
- class Shutdown < Interrupt; end
12
-
13
- class CLI
14
- include Util
15
- include Singleton
16
-
17
- attr_accessor :launcher
18
-
19
- def run(args)
20
- self_read, self_write = IO.pipe
21
-
22
- %w(INT TERM USR1 USR2 TTIN).each do |sig|
23
- begin
24
- trap sig do
25
- self_write.puts(sig)
26
- end
27
- rescue ArgumentError
28
- puts "Signal #{sig} not supported"
29
- end
30
- end
31
-
32
- options = parse_cli_args(args)
33
-
34
- loader = EnvironmentLoader.setup_options(options)
35
-
36
- # When cli args exist, override options in config file
37
- Shoryuken.options.merge!(options)
38
-
39
- daemonize(Shoryuken.options)
40
- write_pid(Shoryuken.options)
41
-
42
- loader.load
43
-
44
- load_celluloid
45
-
46
- require 'shoryuken/launcher'
47
- @launcher = Shoryuken::Launcher.new
48
-
49
- if callback = Shoryuken.start_callback
50
- logger.info { 'Calling Shoryuken.on_start block' }
51
- callback.call
52
- end
53
-
54
- fire_event(:startup)
55
-
56
- begin
57
- launcher.run
58
-
59
- while (readable_io = IO.select([self_read]))
60
- signal = readable_io.first[0].gets.strip
61
- handle_signal(signal)
62
- end
63
- rescue Interrupt
64
- launcher.stop(shutdown: true)
65
- exit 0
66
- end
67
- end
68
-
69
- private
70
-
71
- def load_celluloid
72
- require 'celluloid/current'
73
- Celluloid.logger = (Shoryuken.options[:verbose] ? Shoryuken.logger : nil)
74
-
75
- require 'shoryuken/manager'
76
- end
77
-
78
- def celluloid_loaded?
79
- defined?(::Celluloid)
80
- end
81
-
82
- def daemonize(options)
83
- return unless options[:daemon]
84
-
85
- fail ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
86
-
87
- if celluloid_loaded?
88
- # Celluloid can't be loaded until after we've daemonized
89
- # because it spins up threads and creates locks which get
90
- # into a very bad state if forked.
91
- raise "Celluloid cannot be required until here, or it will break Shoryuken's daemonization"
92
- end
93
-
94
- files_to_reopen = []
95
- ObjectSpace.each_object(File) do |file|
96
- files_to_reopen << file unless file.closed?
97
- end
98
-
99
- Process.daemon(true, true)
100
-
101
- files_to_reopen.each do |file|
102
- begin
103
- file.reopen file.path, 'a+'
104
- file.sync = true
105
- rescue ::Exception
106
- end
107
- end
108
-
109
- [$stdout, $stderr].each do |io|
110
- File.open(options[:logfile], 'ab') do |f|
111
- io.reopen(f)
112
- end
113
- io.sync = true
114
- end
115
- $stdin.reopen('/dev/null')
116
- end
117
-
118
- def write_pid(options)
119
- if (path = options[:pidfile])
120
- File.open(path, 'w') do |f|
121
- f.puts Process.pid
122
- end
123
- end
124
- end
125
-
126
- def parse_cli_args(argv)
127
- opts = {}
128
-
129
- @parser = OptionParser.new do |o|
130
- o.on '-c', '--concurrency INT', 'Processor threads to use' do |arg|
131
- opts[:concurrency] = Integer(arg)
132
- end
133
-
134
- o.on '-d', '--daemon', 'Daemonize process' do |arg|
135
- opts[:daemon] = arg
136
- end
137
-
138
- o.on '-q', '--queue QUEUE[,WEIGHT]...', 'Queues to process with optional weights' do |arg|
139
- queue, weight = arg.split(',')
140
- opts[:queues] = [] unless opts[:queues]
141
- opts[:queues] << [queue, weight]
142
- end
143
-
144
- o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
145
- opts[:require] = arg
146
- end
147
-
148
- o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
149
- opts[:config_file] = arg
150
- end
151
-
152
- o.on '-R', '--rails', 'Load Rails' do |arg|
153
- opts[:rails] = arg
154
- end
155
-
156
- o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
157
- opts[:logfile] = arg
158
- end
159
-
160
- o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
161
- opts[:pidfile] = arg
162
- end
163
-
164
- o.on '-v', '--verbose', 'Print more verbose output' do |arg|
165
- opts[:verbose] = arg
166
- end
167
-
168
- o.on '-V', '--version', 'Print version and exit' do
169
- puts "Shoryuken #{Shoryuken::VERSION}"
170
- exit 0
171
- end
172
- end
173
-
174
- @parser.banner = 'shoryuken [options]'
175
- @parser.on_tail '-h', '--help', 'Show help' do
176
- logger.info { @parser }
177
- exit 1
178
- end
179
- @parser.parse!(argv)
180
- opts
181
- end
182
-
183
- def handle_signal(sig)
184
- logger.info { "Got #{sig} signal" }
185
-
186
- case sig
187
- when 'USR1'
188
- logger.info { 'Received USR1, will soft shutdown down' }
189
-
190
- launcher.stop
191
- fire_event(:quiet, true)
192
- exit 0
193
- when 'TTIN'
194
- Thread.list.each do |thread|
195
- logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
196
- if thread.backtrace
197
- logger.info { thread.backtrace.join("\n") }
198
- else
199
- logger.info { '<no backtrace available>' }
200
- end
201
- end
202
-
203
- ready = launcher.manager.instance_variable_get(:@ready).size
204
- busy = launcher.manager.instance_variable_get(:@busy).size
205
- queues = launcher.manager.instance_variable_get(:@queues)
206
-
207
- logger.info { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{unparse_queues(queues)}" }
208
- else
209
- logger.info { "Received #{sig}, will shutdown down" }
210
-
211
- raise Interrupt
212
- end
213
- end
214
- end
215
- end
@@ -1,27 +0,0 @@
1
- module Shoryuken
2
- class SnsArn
3
- def initialize(topic)
4
- @topic = topic
5
- end
6
-
7
- def to_s
8
- @arn ||= "arn:aws:sns:#{region}:#{account_id}:#{@topic}"
9
- end
10
-
11
- private
12
-
13
- def account_id
14
- Shoryuken::Client.account_id.tap do |account_id|
15
- if account_id.to_s.empty?
16
- fail 'To generate SNS ARNs, you must assign an :account_id in your Shoryuken::Client.'
17
- end
18
- end
19
- end
20
-
21
- def region
22
- Aws.config.fetch(:region) do
23
- fail 'To generate SNS ARNs, you must include a :region in your AWS config.'
24
- end
25
- end
26
- end
27
- end
@@ -1,17 +0,0 @@
1
- module Shoryuken
2
- class Topic
3
- def initialize(name, sns)
4
- @name, @sns = name, sns
5
- end
6
-
7
- def arn
8
- @arn ||= Client.sns_arn.new(@name).to_s
9
- end
10
-
11
- def send_message(body, options = {})
12
- body = JSON.dump(body) if body.is_a?(Hash)
13
-
14
- @sns.publish(topic_arn: arn, message: body)
15
- end
16
- end
17
- end
@@ -1,42 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Shoryuken::SnsArn do
4
- let(:account_id) { '1234567890' }
5
- let(:region) { 'eu-west-1' }
6
- let(:topic) { 'topic-x' }
7
-
8
- before do
9
- Shoryuken::Client.account_id = account_id
10
- Aws.config = { region: region }
11
- end
12
-
13
- subject { described_class.new(topic).to_s }
14
-
15
- describe '#to_s' do
16
- context 'when the Aws config includes all the information necessary' do
17
- it 'generates an SNS arn' do
18
- expect(subject).to eq('arn:aws:sns:eu-west-1:1234567890:topic-x')
19
- end
20
- end
21
-
22
- context 'when the Aws config does not include the account id' do
23
- before do
24
- Shoryuken::Client.account_id = nil
25
- end
26
-
27
- it 'fails' do
28
- expect { subject }.to raise_error(/an :account_id/)
29
- end
30
- end
31
-
32
- context 'when the Aws config does not include the region' do
33
- before do
34
- Aws.config.delete :region
35
- end
36
-
37
- it 'fails' do
38
- expect { subject }.to raise_error(/a :region/)
39
- end
40
- end
41
- end
42
- end
@@ -1,32 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Shoryuken::Topic do
4
- let(:sns) { Aws::SNS::Client.new stub_responses: true }
5
- let(:topic_arn) { 'arn:aws:sns:us-east-1:0987654321:shoryuken' }
6
- let(:topic_name) { 'shoryuken' }
7
-
8
- before do
9
- Shoryuken::Client.account_id = '0987654321'
10
- Aws.config = { region: 'us-east-1' }
11
- end
12
-
13
- subject { described_class.new(topic_name, sns) }
14
-
15
- describe '#send_message' do
16
- it 'enqueues a message' do
17
- sns.stub_responses(:publish, { message_id: 'msg1' })
18
- expect(sns).to receive(:publish).with(topic_arn: topic_arn, message: 'test')
19
-
20
- subject.send_message('test')
21
- end
22
-
23
- it 'parses as JSON by default' do
24
- msg = { field: 'test', other_field: 'other' }
25
-
26
- sns.stub_responses(:publish, { message_id: 'msg2' })
27
- expect(sns).to receive(:publish).with(topic_arn: topic_arn, message: JSON.dump(msg))
28
-
29
- subject.send_message(msg)
30
- end
31
- end
32
- end
@@ -1,6 +0,0 @@
1
- aws:
2
- access_key_id: <%= ENV['AWS_ACCESS_KEY_ID'] %>
3
- secret_access_key: <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
4
- region: us-east-1
5
- sqs_endpoint: https://github.com/phstc/shoryuken:4568
6
- sns_endpoint: http://127.0.0.1:4568