shoryuken 2.1.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +2 -0
  3. data/.rubocop.yml +8 -2
  4. data/.travis.yml +1 -0
  5. data/CHANGELOG.md +19 -0
  6. data/README.md +20 -104
  7. data/Rakefile +0 -1
  8. data/bin/cli/base.rb +42 -0
  9. data/bin/cli/sqs.rb +188 -0
  10. data/bin/shoryuken +47 -9
  11. data/examples/default_worker.rb +1 -1
  12. data/lib/shoryuken.rb +75 -55
  13. data/lib/shoryuken/client.rb +3 -15
  14. data/lib/shoryuken/default_worker_registry.rb +9 -5
  15. data/lib/shoryuken/environment_loader.rb +9 -40
  16. data/lib/shoryuken/fetcher.rb +16 -18
  17. data/lib/shoryuken/launcher.rb +5 -28
  18. data/lib/shoryuken/manager.rb +60 -140
  19. data/lib/shoryuken/message.rb +4 -13
  20. data/lib/shoryuken/middleware/chain.rb +1 -18
  21. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +7 -16
  22. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +25 -21
  23. data/lib/shoryuken/polling.rb +2 -4
  24. data/lib/shoryuken/processor.rb +2 -11
  25. data/lib/shoryuken/queue.rb +1 -3
  26. data/lib/shoryuken/runner.rb +143 -0
  27. data/lib/shoryuken/util.rb +0 -8
  28. data/lib/shoryuken/version.rb +1 -1
  29. data/lib/shoryuken/worker.rb +1 -1
  30. data/shoryuken.gemspec +6 -5
  31. data/spec/integration/launcher_spec.rb +4 -3
  32. data/spec/shoryuken/client_spec.rb +2 -45
  33. data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
  34. data/spec/shoryuken/environment_loader_spec.rb +34 -0
  35. data/spec/shoryuken/manager_spec.rb +11 -21
  36. data/spec/shoryuken/middleware/chain_spec.rb +0 -24
  37. data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +0 -2
  38. data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +46 -29
  39. data/spec/shoryuken/processor_spec.rb +5 -5
  40. data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +8 -22
  41. data/spec/shoryuken_spec.rb +13 -1
  42. data/spec/spec_helper.rb +3 -8
  43. metadata +29 -22
  44. data/lib/shoryuken/aws_config.rb +0 -64
  45. data/lib/shoryuken/cli.rb +0 -215
  46. data/lib/shoryuken/sns_arn.rb +0 -27
  47. data/lib/shoryuken/topic.rb +0 -17
  48. data/spec/shoryuken/sns_arn_spec.rb +0 -42
  49. data/spec/shoryuken/topic_spec.rb +0 -32
  50. data/spec/shoryuken_endpoint.yml +0 -6
@@ -65,7 +65,6 @@ module Shoryuken
65
65
  end
66
66
 
67
67
  class WeightedRoundRobin < BaseStrategy
68
-
69
68
  def initialize(queues)
70
69
  @initial_queues = queues
71
70
  @queues = queues.dup.uniq
@@ -106,7 +105,7 @@ module Shoryuken
106
105
  @paused_queues << [Time.now + delay, queue]
107
106
  logger.debug "Paused '#{queue}'"
108
107
  end
109
-
108
+
110
109
  def unpause_queues
111
110
  return if @paused_queues.empty?
112
111
  return if Time.now < @paused_queues.first[0]
@@ -129,7 +128,6 @@ module Shoryuken
129
128
  end
130
129
 
131
130
  class StrictPriority < BaseStrategy
132
-
133
131
  def initialize(queues)
134
132
  # Priority ordering of the queues, highest priority first
135
133
  @queues = queues
@@ -178,7 +176,7 @@ module Shoryuken
178
176
  return queue unless queue_paused?(queue)
179
177
  end
180
178
 
181
- return nil
179
+ nil
182
180
  end
183
181
 
184
182
  def queues_unpaused_since?
@@ -1,16 +1,11 @@
1
- require 'json'
2
-
3
1
  module Shoryuken
4
2
  class Processor
5
- include Celluloid
6
3
  include Util
7
4
 
8
5
  def initialize(manager)
9
6
  @manager = manager
10
7
  end
11
8
 
12
- attr_accessor :proxy_id
13
-
14
9
  def process(queue, sqs_msg)
15
10
  worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
16
11
  body = get_body(worker.class, sqs_msg)
@@ -18,12 +13,8 @@ module Shoryuken
18
13
  worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
19
14
  worker.perform(sqs_msg, body)
20
15
  end
21
-
22
- @manager.async.processor_done(queue, current_actor)
23
- end
24
-
25
- def running_thread
26
- Thread.current
16
+ ensure
17
+ @manager.processor_done(queue)
27
18
  end
28
19
 
29
20
  private
@@ -35,9 +35,7 @@ module Shoryuken
35
35
  end
36
36
 
37
37
  def receive_messages(options)
38
- client.receive_message(options.merge(queue_url: url)).
39
- messages.
40
- map { |m| Message.new(client, self, m) }
38
+ client.receive_message(options.merge(queue_url: url)).messages.map { |m| Message.new(client, self, m) }
41
39
  end
42
40
 
43
41
  def fifo?
@@ -0,0 +1,143 @@
1
+ $stdout.sync = true
2
+
3
+ require 'singleton'
4
+ require 'optparse'
5
+ require 'erb'
6
+
7
+ require 'shoryuken'
8
+
9
+ module Shoryuken
10
+ # rubocop:disable Lint/InheritException
11
+ # rubocop:disable Metrics/AbcSize
12
+ # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
13
+ class Shutdown < Interrupt; end
14
+
15
+ class Runner
16
+ include Util
17
+ include Singleton
18
+
19
+ def run(options)
20
+ self_read, self_write = IO.pipe
21
+
22
+ %w(INT TERM USR1 USR2 TTIN).each do |sig|
23
+ begin
24
+ trap sig do
25
+ self_write.puts(sig)
26
+ end
27
+ rescue ArgumentError
28
+ puts "Signal #{sig} not supported"
29
+ end
30
+ end
31
+
32
+ loader = EnvironmentLoader.setup_options(options)
33
+
34
+ # When cli args exist, override options in config file
35
+ Shoryuken.options.merge!(options)
36
+
37
+ daemonize(Shoryuken.options)
38
+ write_pid(Shoryuken.options)
39
+
40
+ loader.load
41
+
42
+ initialize_concurrent_logger
43
+
44
+ @launcher = Shoryuken::Launcher.new
45
+
46
+ if (callback = Shoryuken.start_callback)
47
+ logger.info { 'Calling Shoryuken.on_start block' }
48
+ callback.call
49
+ end
50
+
51
+ fire_event(:startup)
52
+
53
+ begin
54
+ @launcher.run
55
+
56
+ while (readable_io = IO.select([self_read]))
57
+ signal = readable_io.first[0].gets.strip
58
+ handle_signal(signal)
59
+ end
60
+ rescue Interrupt
61
+ @launcher.stop(shutdown: true)
62
+ exit 0
63
+ end
64
+ end
65
+
66
+ private
67
+
68
+ def initialize_concurrent_logger
69
+ return unless Shoryuken.logger
70
+
71
+ Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
72
+ Shoryuken.logger.log(level, msg, progname, &block)
73
+ end
74
+ end
75
+
76
+ def daemonize(options)
77
+ return unless options[:daemon]
78
+
79
+ files_to_reopen = []
80
+ ObjectSpace.each_object(File) do |file|
81
+ files_to_reopen << file unless file.closed?
82
+ end
83
+
84
+ Process.daemon(true, true)
85
+
86
+ files_to_reopen.each do |file|
87
+ begin
88
+ file.reopen file.path, 'a+'
89
+ file.sync = true
90
+ rescue ::Exception
91
+ end
92
+ end
93
+
94
+ [$stdout, $stderr].each do |io|
95
+ File.open(options[:logfile], 'ab') do |f|
96
+ io.reopen(f)
97
+ end
98
+ io.sync = true
99
+ end
100
+ $stdin.reopen('/dev/null')
101
+ end
102
+
103
+ def write_pid(options)
104
+ return unless (path = options[:pidfile])
105
+
106
+ File.open(path, 'w') { |f| f.puts(Process.pid) }
107
+ end
108
+
109
+ def execute_soft_shutdown
110
+ logger.info { 'Received USR1, will soft shutdown down' }
111
+
112
+ @launcher.stop
113
+ fire_event(:quiet, true)
114
+ exit 0
115
+ end
116
+
117
+ def print_threads_backtrace
118
+ Thread.list.each do |thread|
119
+ logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
120
+ if thread.backtrace
121
+ logger.info { thread.backtrace.join("\n") }
122
+ else
123
+ logger.info { '<no backtrace available>' }
124
+ end
125
+ end
126
+ end
127
+
128
+ def handle_signal(sig)
129
+ logger.info { "Got #{sig} signal" }
130
+
131
+ case sig
132
+ when 'USR1' then execute_soft_shutdown
133
+ when 'TTIN' then print_threads_backtrace
134
+ when 'USR2'
135
+ logger.warn { "Received #{sig}, will do nothing. To execute soft shutdown, please send USR1" }
136
+ else
137
+ logger.info { "Received #{sig}, will shutdown down" }
138
+
139
+ raise Interrupt
140
+ end
141
+ end
142
+ end
143
+ end
@@ -1,13 +1,5 @@
1
1
  module Shoryuken
2
2
  module Util
3
- def watchdog(last_words)
4
- yield
5
- rescue => ex
6
- logger.error { last_words }
7
- logger.error { ex }
8
- logger.error { ex.backtrace.join("\n") }
9
- end
10
-
11
3
  def logger
12
4
  Shoryuken.logger
13
5
  end
@@ -1,3 +1,3 @@
1
1
  module Shoryuken
2
- VERSION = '2.1.3'
2
+ VERSION = '3.0.0'.freeze
3
3
  end
@@ -63,7 +63,7 @@ module Shoryuken
63
63
 
64
64
  def normalize_worker_queue!
65
65
  queue = @shoryuken_options['queue']
66
- if queue.respond_to? :call
66
+ if queue.respond_to?(:call)
67
67
  queue = queue.call
68
68
  @shoryuken_options['queue'] = queue
69
69
  end
data/shoryuken.gemspec CHANGED
@@ -6,14 +6,14 @@ require 'shoryuken/version'
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = 'shoryuken'
8
8
  spec.version = Shoryuken::VERSION
9
- spec.authors = ['Pablo Cantero', 'Mario Kostelac']
10
- spec.email = ['pablo@pablocantero.com', 'mariokostelac@gmail.com']
11
- spec.description = spec.summary = %q(Shoryuken is a super efficient AWS SQS thread based message processor)
9
+ spec.authors = ['Pablo Cantero']
10
+ spec.email = ['pablo@pablocantero.com']
11
+ spec.description = spec.summary = 'Shoryuken is a super efficient AWS SQS thread based message processor'
12
12
  spec.homepage = 'https://github.com/phstc/shoryuken'
13
13
  spec.license = 'LGPL-3.0'
14
14
 
15
15
  spec.files = `git ls-files -z`.split("\x0")
16
- spec.executables = %w[shoryuken]
16
+ spec.executables = %w(shoryuken)
17
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
18
  spec.require_paths = ['lib']
19
19
 
@@ -24,5 +24,6 @@ Gem::Specification.new do |spec|
24
24
  spec.add_development_dependency 'dotenv'
25
25
 
26
26
  spec.add_dependency 'aws-sdk-core', '~> 2'
27
- spec.add_dependency 'celluloid', '~> 0.17'
27
+ spec.add_dependency 'concurrent-ruby'
28
+ spec.add_dependency 'thor'
28
29
  end
@@ -1,12 +1,11 @@
1
1
  require 'spec_helper'
2
2
  require 'shoryuken/manager'
3
3
  require 'shoryuken/launcher'
4
+ require 'securerandom'
4
5
 
5
6
  RSpec.describe Shoryuken::Launcher do
6
7
  describe 'Consuming messages', slow: :true do
7
8
  before do
8
- Shoryuken.options[:aws][:receive_message] = { wait_time_seconds: 5 }
9
-
10
9
  StandardWorker.received_messages = 0
11
10
 
12
11
  queue = "test_shoryuken#{StandardWorker}_#{SecureRandom.uuid}"
@@ -21,7 +20,9 @@ RSpec.describe Shoryuken::Launcher do
21
20
  end
22
21
 
23
22
  after do
24
- queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: StandardWorker.get_shoryuken_options['queue']).queue_url
23
+ queue_url = Shoryuken::Client.sqs.get_queue_url(
24
+ queue_name: StandardWorker.get_shoryuken_options['queue']
25
+ ).queue_url
25
26
 
26
27
  Shoryuken::Client.sqs.delete_queue queue_url: queue_url
27
28
  end
@@ -1,17 +1,16 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Shoryuken::Client do
3
+ RSpec.describe Shoryuken::Client do
4
4
  let(:credentials) { Aws::Credentials.new('access_key_id', 'secret_access_key') }
5
5
  let(:sqs) { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) }
6
6
  let(:queue_name) { 'shoryuken' }
7
7
  let(:queue_url) { 'https://eu-west-1.amazonaws.com:6059/123456789012/shoryuken' }
8
- let(:sqs_endpoint) { 'http://localhost:4568' }
9
- let(:sns_endpoint) { 'http://0.0.0.0:4568' }
10
8
 
11
9
  describe '.queue' do
12
10
  before do
13
11
  described_class.sqs = sqs
14
12
  end
13
+
15
14
  it 'memoizes queues' do
16
15
  sqs.stub_responses(:get_queue_url, { queue_url: queue_url }, { queue_url: 'xyz' })
17
16
 
@@ -19,46 +18,4 @@ describe Shoryuken::Client do
19
18
  expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
20
19
  end
21
20
  end
22
-
23
- describe 'environment variable endpoints' do
24
- before do
25
- ENV['AWS_SQS_ENDPOINT'] = sqs_endpoint
26
- ENV['AWS_SNS_ENDPOINT'] = sns_endpoint
27
- ENV['AWS_REGION'] = 'us-east-1'
28
- Shoryuken.options[:aws] = {}
29
- Shoryuken::AwsConfig.options = {}
30
- end
31
-
32
- it 'will use config file settings if set' do
33
- load_config_file_by_file_name('shoryuken_endpoint.yml')
34
- expect(described_class.sqs.config.endpoint.to_s).to eql('https://github.com/phstc/shoryuken:4568')
35
- expect(described_class.sns.config.endpoint.to_s).to eq('http://127.0.0.1:4568')
36
- end
37
-
38
- it 'should fallback to environment variable if config file not found or set' do
39
- load_config_file_by_file_name(nil)
40
- expect(described_class.sqs.config.endpoint.to_s).to eql(sqs_endpoint)
41
- expect(described_class.sns.config.endpoint.to_s).to eq(sns_endpoint)
42
- end
43
-
44
- it 'should fallback to environment variable if config file found but settings not set' do
45
- load_config_file_by_file_name('shoryuken.yml')
46
- expect(described_class.sqs.config.endpoint.to_s).to eql(sqs_endpoint)
47
- expect(described_class.sns.config.endpoint.to_s).to eq(sns_endpoint)
48
- end
49
-
50
- it 'will fallback to default settings if no config file settings or environment variables found' do
51
- ENV['AWS_SQS_ENDPOINT'] = nil
52
- ENV['AWS_SNS_ENDPOINT'] = nil
53
- load_config_file_by_file_name('shoryuken.yml')
54
- expect(described_class.sqs.config.endpoint.to_s).to eql('https://sqs.us-east-1.amazonaws.com')
55
- expect(described_class.sns.config.endpoint.to_s).to eq('https://sns.us-east-1.amazonaws.com')
56
- end
57
- end
58
-
59
- def load_config_file_by_file_name(file_name)
60
- path_name = file_name ? File.join(File.expand_path('../../..', __FILE__), 'spec', file_name) : nil
61
- loader = Shoryuken::EnvironmentLoader.setup_options(config_file: path_name)
62
- loader.load
63
- end
64
21
  end
@@ -1,6 +1,7 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Shoryuken::DefaultWorkerRegistry do
3
+ # rubocop:disable Metrics/BlockLength
4
+ RSpec.describe Shoryuken::DefaultWorkerRegistry do
4
5
  class RegistryTestWorker
5
6
  include Shoryuken::Worker
6
7
 
@@ -42,16 +43,17 @@ describe Shoryuken::DefaultWorkerRegistry do
42
43
  end
43
44
 
44
45
  describe 'a registry with workers is handling messages' do
45
- def build_message queue, explicit_worker = nil
46
+ def build_message(queue, explicit_worker = nil)
46
47
  attributes = {}
47
- attributes['shoryuken_class'] = {
48
- string_value: explicit_worker.to_s,
49
- data_type: 'String' } if explicit_worker
50
-
51
- double Shoryuken::Message,
52
- body: 'test',
53
- message_attributes: attributes,
54
- message_id: SecureRandom.uuid
48
+
49
+ if explicit_worker
50
+ attributes['shoryuken_class'] = { string_value: explicit_worker.to_s, data_type: 'String' }
51
+ end
52
+
53
+ double(Shoryuken::Message,
54
+ body: 'test',
55
+ message_attributes: attributes,
56
+ message_id: SecureRandom.uuid)
55
57
  end
56
58
 
57
59
  context 'a batch of messages is being processed' do
@@ -0,0 +1,34 @@
1
+ require 'spec_helper'
2
+
3
+ # rubocop:disable Metrics/BlockLength
4
+ RSpec.describe Shoryuken::EnvironmentLoader do
5
+ subject { described_class.new({}) }
6
+
7
+ describe '#parse_queues' do
8
+ before do
9
+ # TODO proper test other methods
10
+ allow(subject).to receive(:load_rails).with(anything)
11
+ allow(subject).to receive(:prefix_active_job_queue_names)
12
+ allow(subject).to receive(:require_workers)
13
+ allow(subject).to receive(:validate_queues)
14
+ allow(subject).to receive(:validate_workers)
15
+ allow(subject).to receive(:patch_deprecated_workers)
16
+ end
17
+
18
+ it 'parses' do
19
+ Shoryuken.options[:queues] = ['queue_1']
20
+ subject.load
21
+
22
+ expect(Shoryuken.queues).to eq(%w(queue_1))
23
+ end
24
+
25
+ context 'with priority' do
26
+ it 'parses' do
27
+ Shoryuken.options[:queues] = ['queue_1', ['queue_2', 2]]
28
+ subject.load
29
+
30
+ expect(Shoryuken.queues).to eq(%w(queue_1 queue_2 queue_2))
31
+ end
32
+ end
33
+ end
34
+ end
@@ -12,21 +12,12 @@ RSpec.describe Shoryuken::Manager do
12
12
  let(:queues) { [queue] }
13
13
  let(:polling_strategy) { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
14
14
  let(:fetcher) { Shoryuken::Fetcher.new }
15
- let(:condvar) do
16
- condvar = double(:condvar)
17
- allow(condvar).to receive(:signal).and_return(nil)
18
- condvar
19
- end
20
- let(:async_manager) { instance_double(described_class.name) }
21
15
  let(:concurrency) { 1 }
22
16
 
23
- subject { Shoryuken::Manager.new(condvar) }
17
+ subject { Shoryuken::Manager.new(fetcher, polling_strategy) }
24
18
 
25
19
  before(:each) do
26
20
  Shoryuken.options[:concurrency] = concurrency
27
- subject.fetcher = fetcher
28
- subject.polling_strategy = polling_strategy
29
- allow_any_instance_of(described_class).to receive(:async).and_return(async_manager)
30
21
  end
31
22
 
32
23
  after(:each) do
@@ -37,31 +28,30 @@ RSpec.describe Shoryuken::Manager do
37
28
  describe 'Invalid concurrency setting' do
38
29
  it 'raises ArgumentError if concurrency is not positive number' do
39
30
  Shoryuken.options[:concurrency] = -1
40
- expect { Shoryuken::Manager.new(nil) }
31
+ expect { Shoryuken::Manager.new(nil, nil) }
41
32
  .to raise_error(ArgumentError, 'Concurrency value -1 is invalid, it needs to be a positive number')
42
33
  end
43
34
  end
44
35
 
45
- describe '#dispatch' do
46
- it 'pauses when there are no active queues' do
36
+ describe '#start' do
37
+ xit 'pauses when there are no active queues' do
47
38
  expect(polling_strategy).to receive(:next_queue).and_return(nil)
48
39
  expect_any_instance_of(described_class).to receive(:after)
49
- subject.dispatch
40
+ subject.start
50
41
  end
51
42
 
52
- it 'calls dispatch_batch if worker wants batches' do
43
+ xit 'calls dispatch_batch if worker wants batches' do
53
44
  TestWorker.get_shoryuken_options['batch'] = true
54
45
  expect_any_instance_of(described_class).to receive(:dispatch_batch).with(queue_config_of(queue))
55
- expect_any_instance_of(described_class).to receive(:async).and_return(async_manager)
56
- expect(async_manager).to receive(:dispatch)
57
- subject.dispatch
46
+ expect(subject).to receive(:dispatch_later)
47
+ subject.start
58
48
  end
59
49
 
60
- it 'calls dispatch_single_messages if worker wants single messages' do
50
+ xit 'calls dispatch_single_messages if worker wants single messages' do
61
51
  expect_any_instance_of(described_class).to receive(:dispatch_single_messages).
62
52
  with(queue_config_of(queue))
63
- expect(async_manager).to receive(:dispatch)
64
- subject.dispatch
53
+ expect(subject).to receive(:dispatch_later)
54
+ subject.start
65
55
  end
66
56
  end
67
57