shoryuken 2.1.3 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +2 -0
- data/.rubocop.yml +8 -2
- data/.travis.yml +1 -0
- data/CHANGELOG.md +19 -0
- data/README.md +20 -104
- data/Rakefile +0 -1
- data/bin/cli/base.rb +42 -0
- data/bin/cli/sqs.rb +188 -0
- data/bin/shoryuken +47 -9
- data/examples/default_worker.rb +1 -1
- data/lib/shoryuken.rb +75 -55
- data/lib/shoryuken/client.rb +3 -15
- data/lib/shoryuken/default_worker_registry.rb +9 -5
- data/lib/shoryuken/environment_loader.rb +9 -40
- data/lib/shoryuken/fetcher.rb +16 -18
- data/lib/shoryuken/launcher.rb +5 -28
- data/lib/shoryuken/manager.rb +60 -140
- data/lib/shoryuken/message.rb +4 -13
- data/lib/shoryuken/middleware/chain.rb +1 -18
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +7 -16
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +25 -21
- data/lib/shoryuken/polling.rb +2 -4
- data/lib/shoryuken/processor.rb +2 -11
- data/lib/shoryuken/queue.rb +1 -3
- data/lib/shoryuken/runner.rb +143 -0
- data/lib/shoryuken/util.rb +0 -8
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +1 -1
- data/shoryuken.gemspec +6 -5
- data/spec/integration/launcher_spec.rb +4 -3
- data/spec/shoryuken/client_spec.rb +2 -45
- data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
- data/spec/shoryuken/environment_loader_spec.rb +34 -0
- data/spec/shoryuken/manager_spec.rb +11 -21
- data/spec/shoryuken/middleware/chain_spec.rb +0 -24
- data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +0 -2
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +46 -29
- data/spec/shoryuken/processor_spec.rb +5 -5
- data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +8 -22
- data/spec/shoryuken_spec.rb +13 -1
- data/spec/spec_helper.rb +3 -8
- metadata +29 -22
- data/lib/shoryuken/aws_config.rb +0 -64
- data/lib/shoryuken/cli.rb +0 -215
- data/lib/shoryuken/sns_arn.rb +0 -27
- data/lib/shoryuken/topic.rb +0 -17
- data/spec/shoryuken/sns_arn_spec.rb +0 -42
- data/spec/shoryuken/topic_spec.rb +0 -32
- data/spec/shoryuken_endpoint.yml +0 -6
data/bin/shoryuken
CHANGED
@@ -1,12 +1,50 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
3
|
+
require 'rubygems'
|
4
|
+
|
5
|
+
require 'thor'
|
6
|
+
require 'aws-sdk-core'
|
7
|
+
require_relative 'cli/base'
|
8
|
+
require_relative 'cli/sqs'
|
9
|
+
require_relative '../lib/shoryuken/runner'
|
10
|
+
|
11
|
+
# rubocop:disable Metrics/AbcSize
|
12
|
+
module Shoryuken
|
13
|
+
module CLI
|
14
|
+
class Runner < Base
|
15
|
+
default_task :start
|
16
|
+
|
17
|
+
register(Shoryuken::CLI::SQS, 'sqs', 'sqs COMMAND', 'SQS commands')
|
18
|
+
|
19
|
+
desc 'start', 'Start shoryuken'
|
20
|
+
method_option :concurrency, aliases: '-c', type: :numeric, desc: 'Processor threads to use'
|
21
|
+
method_option :daemon, aliases: '-d', type: :boolean, desc: 'Daemonize process'
|
22
|
+
method_option :queues, aliases: '-q', type: :array, desc: 'Queues to process with optional weights'
|
23
|
+
method_option :require, aliases: '-r', type: :string, desc: 'Dir or path of the workers'
|
24
|
+
method_option :config_file, aliases: '-C', type: :string, desc: 'Path to config file'
|
25
|
+
method_option :rails, aliases: '-R', type: :boolean, desc: 'Load Rails'
|
26
|
+
method_option :logfile, aliases: '-L', type: :string, desc: 'Path to logfile'
|
27
|
+
method_option :pidfile, aliases: '-P', type: :string, desc: 'Path to pidfile'
|
28
|
+
method_option :verbose, aliases: '-v', type: :boolean, desc: 'Print more verbose output'
|
29
|
+
def start
|
30
|
+
opts = options.to_h.symbolize_keys
|
31
|
+
|
32
|
+
# Keep compatibility with old CLI queue format
|
33
|
+
opts[:queues] = options.queues.map { |q| q.split(',') } if options.queues
|
34
|
+
|
35
|
+
if options.daemon && options.logfile.nil?
|
36
|
+
fail_task "You should set a logfile if you're going to daemonize"
|
37
|
+
end
|
38
|
+
|
39
|
+
Shoryuken::Runner.instance.run(opts.freeze)
|
40
|
+
end
|
41
|
+
|
42
|
+
desc 'version', 'Print version'
|
43
|
+
def version
|
44
|
+
say "Shoryuken #{Shoryuken::VERSION}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
12
48
|
end
|
49
|
+
|
50
|
+
Shoryuken::CLI::Runner.start
|
data/examples/default_worker.rb
CHANGED
data/lib/shoryuken.rb
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
require 'yaml'
|
2
|
+
require 'json'
|
2
3
|
require 'aws-sdk-core'
|
3
4
|
require 'time'
|
5
|
+
require 'concurrent'
|
4
6
|
|
5
7
|
require 'shoryuken/version'
|
6
8
|
require 'shoryuken/core_ext'
|
7
9
|
require 'shoryuken/util'
|
8
10
|
require 'shoryuken/logging'
|
9
|
-
require 'shoryuken/aws_config'
|
10
11
|
require 'shoryuken/environment_loader'
|
11
12
|
require 'shoryuken/queue'
|
12
13
|
require 'shoryuken/message'
|
@@ -19,9 +20,11 @@ require 'shoryuken/middleware/server/auto_delete'
|
|
19
20
|
Shoryuken::Middleware::Server.autoload :AutoExtendVisibility, 'shoryuken/middleware/server/auto_extend_visibility'
|
20
21
|
require 'shoryuken/middleware/server/exponential_backoff_retry'
|
21
22
|
require 'shoryuken/middleware/server/timing'
|
22
|
-
require 'shoryuken/sns_arn'
|
23
|
-
require 'shoryuken/topic'
|
24
23
|
require 'shoryuken/polling'
|
24
|
+
require 'shoryuken/manager'
|
25
|
+
require 'shoryuken/launcher'
|
26
|
+
require 'shoryuken/processor'
|
27
|
+
require 'shoryuken/fetcher'
|
25
28
|
|
26
29
|
module Shoryuken
|
27
30
|
DEFAULTS = {
|
@@ -33,78 +36,106 @@ module Shoryuken
|
|
33
36
|
lifecycle_events: {
|
34
37
|
startup: [],
|
35
38
|
quiet: [],
|
36
|
-
shutdown: []
|
39
|
+
shutdown: []
|
37
40
|
},
|
38
|
-
polling_strategy: Polling::WeightedRoundRobin
|
39
|
-
}
|
41
|
+
polling_strategy: Polling::WeightedRoundRobin
|
42
|
+
}.freeze
|
40
43
|
|
41
|
-
@@queues
|
42
|
-
@@worker_registry
|
44
|
+
@@queues = []
|
45
|
+
@@worker_registry = DefaultWorkerRegistry.new
|
43
46
|
@@active_job_queue_name_prefixing = false
|
47
|
+
@@sqs_client = nil
|
48
|
+
@@sqs_client_receive_message_opts = {}
|
49
|
+
@@start_callback = nil
|
50
|
+
@@stop_callback = nil
|
44
51
|
|
45
52
|
class << self
|
46
|
-
def options
|
47
|
-
@options ||= DEFAULTS.dup
|
48
|
-
end
|
49
|
-
|
50
53
|
def queues
|
51
54
|
@@queues
|
52
55
|
end
|
53
56
|
|
54
|
-
def
|
55
|
-
|
57
|
+
def add_queue(queue, priority = 1)
|
58
|
+
priority.times { queues << queue }
|
56
59
|
end
|
57
60
|
|
58
|
-
def
|
59
|
-
worker_registry
|
61
|
+
def worker_registry
|
62
|
+
@@worker_registry
|
60
63
|
end
|
61
64
|
|
62
65
|
def worker_registry=(worker_registry)
|
63
66
|
@@worker_registry = worker_registry
|
64
67
|
end
|
65
68
|
|
66
|
-
def
|
67
|
-
@@
|
69
|
+
def start_callback
|
70
|
+
@@start_callback
|
71
|
+
end
|
72
|
+
|
73
|
+
def start_callback=(start_callback)
|
74
|
+
@@start_callback = start_callback
|
75
|
+
end
|
76
|
+
|
77
|
+
def stop_callback
|
78
|
+
@@stop_callback
|
79
|
+
end
|
80
|
+
|
81
|
+
def stop_callback=(stop_callback)
|
82
|
+
@@stop_callback = stop_callback
|
68
83
|
end
|
69
84
|
|
70
85
|
def active_job_queue_name_prefixing
|
71
86
|
@@active_job_queue_name_prefixing
|
72
87
|
end
|
73
88
|
|
74
|
-
def active_job_queue_name_prefixing=(
|
75
|
-
@@active_job_queue_name_prefixing =
|
89
|
+
def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
|
90
|
+
@@active_job_queue_name_prefixing = active_job_queue_name_prefixing
|
91
|
+
end
|
92
|
+
|
93
|
+
def sqs_client
|
94
|
+
@@sqs_client ||= Aws::SQS::Client.new
|
95
|
+
end
|
96
|
+
|
97
|
+
def sqs_client=(sqs_client)
|
98
|
+
@@sqs_client
|
99
|
+
end
|
100
|
+
|
101
|
+
def sqs_client_receive_message_opts
|
102
|
+
@@sqs_client_receive_message_opts
|
103
|
+
end
|
104
|
+
|
105
|
+
def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
|
106
|
+
@@sqs_client_receive_message_opts
|
107
|
+
end
|
108
|
+
|
109
|
+
def options
|
110
|
+
@@options ||= DEFAULTS.dup
|
111
|
+
end
|
112
|
+
|
113
|
+
def logger
|
114
|
+
Shoryuken::Logging.logger
|
115
|
+
end
|
116
|
+
|
117
|
+
def register_worker(*args)
|
118
|
+
@@worker_registry.register_worker(*args)
|
76
119
|
end
|
77
120
|
|
78
|
-
##
|
79
|
-
# Configuration for Shoryuken server, use like:
|
80
|
-
#
|
81
|
-
# Shoryuken.configure_server do |config|
|
82
|
-
# config.aws = { :sqs_endpoint => '...', :access_key_id: '...', :secret_access_key: '...', region: '...' }
|
83
|
-
# end
|
84
121
|
def configure_server
|
85
122
|
yield self if server?
|
86
123
|
end
|
87
124
|
|
88
125
|
def server_middleware
|
89
|
-
|
90
|
-
yield
|
91
|
-
|
126
|
+
@@server_chain ||= default_server_middleware
|
127
|
+
yield @@server_chain if block_given?
|
128
|
+
@@server_chain
|
92
129
|
end
|
93
130
|
|
94
|
-
##
|
95
|
-
# Configuration for Shoryuken client, use like:
|
96
|
-
#
|
97
|
-
# Shoryuken.configure_client do |config|
|
98
|
-
# config.aws = { :sqs_endpoint => '...', :access_key_id: '...', :secret_access_key: '...', region: '...' }
|
99
|
-
# end
|
100
131
|
def configure_client
|
101
132
|
yield self unless server?
|
102
133
|
end
|
103
134
|
|
104
135
|
def client_middleware
|
105
|
-
|
106
|
-
yield
|
107
|
-
|
136
|
+
@@client_chain ||= default_client_middleware
|
137
|
+
yield @@client_chain if block_given?
|
138
|
+
@@client_chain
|
108
139
|
end
|
109
140
|
|
110
141
|
def default_worker_options
|
@@ -114,27 +145,20 @@ module Shoryuken
|
|
114
145
|
'auto_delete' => false,
|
115
146
|
'auto_visibility_timeout' => false,
|
116
147
|
'retry_intervals' => nil,
|
117
|
-
'batch' => false
|
148
|
+
'batch' => false
|
149
|
+
}
|
118
150
|
end
|
119
151
|
|
120
|
-
def default_worker_options=(
|
121
|
-
@@default_worker_options =
|
122
|
-
end
|
123
|
-
|
124
|
-
def on_aws_initialization(&block)
|
125
|
-
@aws_initialization_callback = block
|
152
|
+
def default_worker_options=(default_worker_options)
|
153
|
+
@@default_worker_options = default_worker_options
|
126
154
|
end
|
127
155
|
|
128
156
|
def on_start(&block)
|
129
|
-
|
157
|
+
@@start_callback = block
|
130
158
|
end
|
131
159
|
|
132
160
|
def on_stop(&block)
|
133
|
-
|
134
|
-
end
|
135
|
-
|
136
|
-
def aws=(hash)
|
137
|
-
Shoryuken::AwsConfig.setup(hash)
|
161
|
+
@@stop_callback = block
|
138
162
|
end
|
139
163
|
|
140
164
|
# Register a block to run at a point in the Shoryuken lifecycle.
|
@@ -151,10 +175,6 @@ module Shoryuken
|
|
151
175
|
options[:lifecycle_events][event] << block
|
152
176
|
end
|
153
177
|
|
154
|
-
attr_reader :aws_initialization_callback,
|
155
|
-
:start_callback,
|
156
|
-
:stop_callback
|
157
|
-
|
158
178
|
private
|
159
179
|
|
160
180
|
def default_server_middleware
|
data/lib/shoryuken/client.rb
CHANGED
@@ -1,31 +1,19 @@
|
|
1
1
|
module Shoryuken
|
2
2
|
class Client
|
3
3
|
@@queues = {}
|
4
|
-
@@topics = {}
|
5
4
|
|
6
5
|
class << self
|
7
6
|
def queues(name)
|
8
7
|
@@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name)
|
9
8
|
end
|
10
9
|
|
11
|
-
def sns
|
12
|
-
@sns ||= Shoryuken::AwsConfig.sns
|
13
|
-
end
|
14
|
-
|
15
|
-
def sns_arn
|
16
|
-
@sns_arn ||= SnsArn
|
17
|
-
end
|
18
|
-
|
19
10
|
def sqs
|
20
|
-
|
11
|
+
@@sqs ||= Shoryuken.sqs_client
|
21
12
|
end
|
22
13
|
|
23
|
-
def
|
24
|
-
@@
|
14
|
+
def sqs=(sqs)
|
15
|
+
@@sqs = sqs
|
25
16
|
end
|
26
|
-
|
27
|
-
attr_accessor :account_id
|
28
|
-
attr_writer :sns, :sqs, :sqs_resource, :sns_arn
|
29
17
|
end
|
30
18
|
end
|
31
19
|
end
|
@@ -14,11 +14,15 @@ module Shoryuken
|
|
14
14
|
|
15
15
|
def fetch_worker(queue, message)
|
16
16
|
worker_class = !message.is_a?(Array) &&
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
worker_class =
|
17
|
+
message.message_attributes &&
|
18
|
+
message.message_attributes['shoryuken_class'] &&
|
19
|
+
message.message_attributes['shoryuken_class'][:string_value]
|
20
|
+
|
21
|
+
worker_class = begin
|
22
|
+
worker_class.constantize
|
23
|
+
rescue
|
24
|
+
@workers[queue]
|
25
|
+
end
|
22
26
|
|
23
27
|
worker_class.new
|
24
28
|
end
|
@@ -28,10 +28,8 @@ module Shoryuken
|
|
28
28
|
prefix_active_job_queue_names
|
29
29
|
parse_queues
|
30
30
|
require_workers
|
31
|
-
initialize_aws
|
32
31
|
validate_queues
|
33
32
|
validate_workers
|
34
|
-
patch_deprecated_workers
|
35
33
|
end
|
36
34
|
|
37
35
|
private
|
@@ -49,16 +47,6 @@ module Shoryuken
|
|
49
47
|
YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
|
50
48
|
end
|
51
49
|
|
52
|
-
# DEPRECATED: Please use configure_server and configure_client in
|
53
|
-
# https://github.com/phstc/shoryuken/blob/a81637d577b36c5cf245882733ea91a335b6602f/lib/shoryuken.rb#L82
|
54
|
-
# Please delete this method afert next release (v2.0.12 or later)
|
55
|
-
def initialize_aws
|
56
|
-
unless Shoryuken.options[:aws].to_h.empty?
|
57
|
-
Shoryuken.logger.warn { '[DEPRECATION] aws in shoryuken.yml is deprecated. Please use configure_server and configure_client in your initializer' }
|
58
|
-
end
|
59
|
-
Shoryuken::AwsConfig.setup(Shoryuken.options[:aws])
|
60
|
-
end
|
61
|
-
|
62
50
|
def initialize_logger
|
63
51
|
Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
|
64
52
|
Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
|
@@ -83,12 +71,11 @@ module Shoryuken
|
|
83
71
|
end
|
84
72
|
|
85
73
|
def merge_cli_defined_queues
|
86
|
-
cli_defined_queues = options
|
74
|
+
cli_defined_queues = options[:queues].to_a
|
87
75
|
|
88
76
|
cli_defined_queues.each do |cli_defined_queue|
|
89
|
-
|
90
|
-
|
91
|
-
end
|
77
|
+
# CLI defined queues override config_file defined queues
|
78
|
+
Shoryuken.options[:queues].delete_if { |config_file_queue| config_file_queue[0] == cli_defined_queue[0] }
|
92
79
|
|
93
80
|
Shoryuken.options[:queues] << cli_defined_queue
|
94
81
|
end
|
@@ -110,7 +97,7 @@ module Shoryuken
|
|
110
97
|
end
|
111
98
|
|
112
99
|
def parse_queue(queue, weight = nil)
|
113
|
-
[weight.to_i, 1].max
|
100
|
+
Shoryuken.add_queue(queue, [weight.to_i, 1].max)
|
114
101
|
end
|
115
102
|
|
116
103
|
def parse_queues
|
@@ -119,24 +106,6 @@ module Shoryuken
|
|
119
106
|
end
|
120
107
|
end
|
121
108
|
|
122
|
-
def patch_deprecated_workers
|
123
|
-
Shoryuken.worker_registry.queues.each do |queue|
|
124
|
-
Shoryuken.worker_registry.workers(queue).each do |worker_class|
|
125
|
-
if worker_class.instance_method(:perform).arity == 1
|
126
|
-
Shoryuken.logger.warn { "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)" }
|
127
|
-
|
128
|
-
worker_class.class_eval do
|
129
|
-
alias_method :deprecated_perform, :perform
|
130
|
-
|
131
|
-
def perform(sqs_msg, body = nil)
|
132
|
-
deprecated_perform(sqs_msg)
|
133
|
-
end
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
109
|
def require_workers
|
141
110
|
required = Shoryuken.options[:require]
|
142
111
|
|
@@ -156,7 +125,7 @@ module Shoryuken
|
|
156
125
|
|
157
126
|
Shoryuken.queues.uniq.each do |queue|
|
158
127
|
begin
|
159
|
-
Shoryuken::Client.queues
|
128
|
+
Shoryuken::Client.queues(queue)
|
160
129
|
rescue Aws::SQS::Errors::NonExistentQueue
|
161
130
|
non_existent_queues << queue
|
162
131
|
end
|
@@ -166,13 +135,13 @@ module Shoryuken
|
|
166
135
|
end
|
167
136
|
|
168
137
|
def validate_workers
|
138
|
+
return if defined?(::ActiveJob)
|
139
|
+
|
169
140
|
all_queues = Shoryuken.queues
|
170
141
|
queues_with_workers = Shoryuken.worker_registry.queues
|
171
142
|
|
172
|
-
|
173
|
-
|
174
|
-
Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
|
175
|
-
end
|
143
|
+
(all_queues - queues_with_workers).each do |queue|
|
144
|
+
Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
|
176
145
|
end
|
177
146
|
end
|
178
147
|
end
|
data/lib/shoryuken/fetcher.rb
CHANGED
@@ -5,23 +5,21 @@ module Shoryuken
|
|
5
5
|
FETCH_LIMIT = 10
|
6
6
|
|
7
7
|
def fetch(queue, available_processors)
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
[]
|
24
|
-
end
|
8
|
+
started_at = Time.now
|
9
|
+
|
10
|
+
logger.debug { "Looking for new messages in '#{queue}'" }
|
11
|
+
|
12
|
+
begin
|
13
|
+
limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
|
14
|
+
|
15
|
+
sqs_msgs = Array(receive_messages(queue, limit))
|
16
|
+
logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } unless sqs_msgs.empty?
|
17
|
+
logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
|
18
|
+
sqs_msgs
|
19
|
+
rescue => ex
|
20
|
+
logger.error { "Error fetching message: #{ex}" }
|
21
|
+
logger.error { ex.backtrace.first }
|
22
|
+
[]
|
25
23
|
end
|
26
24
|
end
|
27
25
|
|
@@ -31,7 +29,7 @@ module Shoryuken
|
|
31
29
|
# AWS limits the batch size by 10
|
32
30
|
limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
|
33
31
|
|
34
|
-
options =
|
32
|
+
options = Shoryuken.sqs_client_receive_message_opts.to_h.dup
|
35
33
|
options[:max_number_of_messages] = limit
|
36
34
|
options[:message_attribute_names] = %w(All)
|
37
35
|
options[:attribute_names] = %w(All)
|