shoryuken 3.2.3 → 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.rubocop.yml +89 -23
- data/CHANGELOG.md +14 -0
- data/Gemfile +4 -0
- data/Rakefile +1 -1
- data/bin/cli/base.rb +0 -1
- data/bin/cli/sqs.rb +3 -3
- data/bin/shoryuken +3 -9
- data/examples/bootstrap_queues.rb +3 -3
- data/examples/default_worker.rb +1 -1
- data/lib/shoryuken.rb +0 -1
- data/lib/shoryuken/core_ext.rb +1 -1
- data/lib/shoryuken/extensions/active_job_adapter.rb +1 -1
- data/lib/shoryuken/fetcher.rb +3 -3
- data/lib/shoryuken/logging.rb +1 -1
- data/lib/shoryuken/middleware/server/active_record.rb +1 -1
- data/lib/shoryuken/middleware/server/auto_delete.rb +1 -1
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +1 -1
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +1 -1
- data/lib/shoryuken/middleware/server/timing.rb +12 -14
- data/lib/shoryuken/polling/base.rb +1 -1
- data/lib/shoryuken/polling/strict_priority.rb +8 -8
- data/lib/shoryuken/queue.rb +3 -3
- data/lib/shoryuken/runner.rb +9 -3
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker/inline_executor.rb +12 -3
- data/lib/shoryuken/worker_registry.rb +4 -4
- data/shoryuken.gemspec +5 -5
- data/spec/integration/launcher_spec.rb +2 -2
- data/spec/shoryuken/body_parser_spec.rb +2 -2
- data/spec/shoryuken/client_spec.rb +1 -1
- data/spec/shoryuken/core_ext_spec.rb +6 -6
- data/spec/shoryuken/default_worker_registry_spec.rb +2 -4
- data/spec/shoryuken/environment_loader_spec.rb +4 -4
- data/spec/shoryuken/extensions/active_job_adapter_spec.rb +3 -3
- data/spec/shoryuken/manager_spec.rb +1 -1
- data/spec/shoryuken/middleware/chain_spec.rb +2 -2
- data/spec/shoryuken/middleware/server/auto_delete_spec.rb +9 -7
- data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +3 -3
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +5 -3
- data/spec/shoryuken/options_spec.rb +5 -5
- data/spec/shoryuken/polling/strict_priority_spec.rb +1 -1
- data/spec/shoryuken/polling/weighted_round_robin_spec.rb +1 -1
- data/spec/shoryuken/processor_spec.rb +1 -1
- data/spec/shoryuken/queue_spec.rb +3 -3
- data/spec/shoryuken/util_spec.rb +1 -1
- data/spec/shoryuken/worker/default_executor_spec.rb +10 -5
- data/spec/shoryuken/worker/inline_executor_spec.rb +28 -2
- data/spec/shoryuken/worker_spec.rb +7 -9
- data/spec/spec_helper.rb +1 -1
- data/test_workers/endless_interruptive_worker.rb +2 -2
- data/test_workers/endless_uninterruptive_worker.rb +3 -3
- metadata +7 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 68aa225742d398731903e759938a4c3a8ae662ab6a0846b91252a9a381f29bd5
|
4
|
+
data.tar.gz: 0e93f5155fbfeef146adb1f9e26b9943c0e60da6b314b1d1b5f507adcbbbed10
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f7b70803c03bf5aebc3215f0b2f6b4275bd56e738bf10133b9c67ae0a3675335eee1bd818b38667cec4ea6c5b57032a6ab35a2fe6cc5d336d2247ea30b4cdc1e
|
7
|
+
data.tar.gz: a874aaa3f126760d6ac6b95c9e32bd97a6aa173bb3e8a1a4064084d10fc88883d0ec2e5e8e0cee57b27f70d3225cc62ad4317735d76df2f19ee349886eefc720
|
data/.rubocop.yml
CHANGED
@@ -1,13 +1,61 @@
|
|
1
1
|
AllCops:
|
2
|
+
Exclude:
|
3
|
+
- '**/Gemfile'
|
2
4
|
TargetRubyVersion: 2.1
|
3
5
|
|
4
|
-
|
6
|
+
Metrics/PerceivedComplexity:
|
5
7
|
Enabled: false
|
6
8
|
|
7
|
-
|
9
|
+
Metrics/CyclomaticComplexity:
|
10
|
+
Enabled: false
|
11
|
+
|
12
|
+
Metrics/ClassLength:
|
13
|
+
Enabled: false
|
14
|
+
|
15
|
+
Metrics/ParameterLists:
|
16
|
+
Enabled: false
|
17
|
+
|
18
|
+
Metrics/MethodLength:
|
19
|
+
Enabled: false
|
20
|
+
|
21
|
+
Metrics/AbcSize:
|
22
|
+
# Disable "Assignment Branch Condition size for update_attributes is too high" from houndci
|
23
|
+
# because codeclimate already give that for us with more details
|
24
|
+
Enabled: false
|
25
|
+
|
26
|
+
Metrics/LineLength:
|
27
|
+
Max: 125
|
28
|
+
|
29
|
+
Style/Alias:
|
8
30
|
Enabled: false
|
9
31
|
|
10
|
-
Style/
|
32
|
+
Style/PerlBackrefs:
|
33
|
+
Enabled: false
|
34
|
+
|
35
|
+
Style/TrailingBlankLines:
|
36
|
+
Enabled: false
|
37
|
+
|
38
|
+
# Override the HoundCI custom rules (they do not use Rubocop defaults)
|
39
|
+
Style/StringLiterals:
|
40
|
+
EnforcedStyle: single_quotes
|
41
|
+
|
42
|
+
Style/StringLiteralsInInterpolation:
|
43
|
+
EnforcedStyle: single_quotes
|
44
|
+
|
45
|
+
Style/ExtraSpacing:
|
46
|
+
# disabling that in favour of using:
|
47
|
+
# long_field_test_1 = 1
|
48
|
+
# field_test_2 = 2
|
49
|
+
# etc = 3
|
50
|
+
Enabled: false
|
51
|
+
|
52
|
+
Style/BlockDelimiters:
|
53
|
+
Enabled: false
|
54
|
+
|
55
|
+
Style/CollectionMethods:
|
56
|
+
Enabled: false
|
57
|
+
|
58
|
+
Style/SignalException:
|
11
59
|
Enabled: false
|
12
60
|
|
13
61
|
Style/Documentation:
|
@@ -16,41 +64,59 @@ Style/Documentation:
|
|
16
64
|
Style/ClassAndModuleChildren:
|
17
65
|
Enabled: false
|
18
66
|
|
19
|
-
|
67
|
+
Style/CommentAnnotation:
|
20
68
|
Enabled: false
|
21
69
|
|
22
|
-
|
70
|
+
Style/DotPosition:
|
71
|
+
EnforcedStyle: leading
|
72
|
+
|
73
|
+
Style/GuardClause:
|
23
74
|
Enabled: false
|
24
75
|
|
25
|
-
Style/
|
76
|
+
Style/RegexpLiteral:
|
26
77
|
Enabled: false
|
27
78
|
|
28
|
-
|
79
|
+
Lint/HandleExceptions:
|
29
80
|
Enabled: false
|
30
81
|
|
31
|
-
|
82
|
+
Lint/AssignmentInCondition:
|
32
83
|
Enabled: false
|
33
84
|
|
34
|
-
|
35
|
-
|
85
|
+
Style/DoubleNegation:
|
86
|
+
Enabled: false
|
36
87
|
|
37
|
-
|
88
|
+
Style/AndOr:
|
38
89
|
Enabled: false
|
39
90
|
|
40
|
-
Style/
|
91
|
+
Style/ClassVars:
|
41
92
|
Enabled: false
|
42
93
|
|
43
|
-
Style/
|
44
|
-
|
45
|
-
SupportedStyles:
|
46
|
-
- single_quotes
|
47
|
-
- double_quotes
|
94
|
+
Style/GlobalVars:
|
95
|
+
Enabled: false
|
48
96
|
|
49
|
-
Style/
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
97
|
+
Style/SingleLineBlockParams:
|
98
|
+
Enabled: false
|
99
|
+
|
100
|
+
Style/RescueModifier:
|
101
|
+
Enabled: false
|
102
|
+
|
103
|
+
Style/ColonMethodCall:
|
104
|
+
Enabled: false
|
105
|
+
|
106
|
+
Style/FileName:
|
107
|
+
Enabled: false
|
108
|
+
|
109
|
+
Style/FrozenStringLiteralComment:
|
110
|
+
Enabled: false
|
111
|
+
|
112
|
+
Style/RescueStandardError:
|
113
|
+
Enabled: false
|
114
|
+
|
115
|
+
Security/YAMLLoad:
|
116
|
+
Enabled: false
|
117
|
+
|
118
|
+
Naming/MemoizedInstanceVariableName:
|
119
|
+
Enabled: false
|
54
120
|
|
55
|
-
|
121
|
+
Performance/RedundantBlockCall:
|
56
122
|
Enabled: false
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
## [v3.3.0] - 2018-09-30
|
2
|
+
|
3
|
+
- Add support for TSTP
|
4
|
+
- [#492](https://github.com/phstc/shoryuken/pull/492)
|
5
|
+
|
6
|
+
- Support an empty list of queues as a CLI argument
|
7
|
+
- [#507](https://github.com/phstc/shoryuken/pull/507)
|
8
|
+
|
9
|
+
- Add batch support for inline workers
|
10
|
+
- [#514](https://github.com/phstc/shoryuken/pull/514)
|
11
|
+
|
12
|
+
- Make InlineExecutor to behave as the DefaultExecutor when calling perform_in
|
13
|
+
- [#518](https://github.com/phstc/shoryuken/pull/518)
|
14
|
+
|
1
15
|
## [v3.2.3] - 2018-03-25
|
2
16
|
|
3
17
|
- Don't force eager load for Rails 5
|
data/Gemfile
CHANGED
data/Rakefile
CHANGED
@@ -12,7 +12,7 @@ task :console do
|
|
12
12
|
require 'pry'
|
13
13
|
require 'shoryuken'
|
14
14
|
|
15
|
-
config_file = File.join File.expand_path(
|
15
|
+
config_file = File.join File.expand_path(__dir__), 'shoryuken.yml'
|
16
16
|
|
17
17
|
if File.exist? config_file
|
18
18
|
config = YAML.load File.read(config_file)
|
data/bin/cli/base.rb
CHANGED
data/bin/cli/sqs.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'date'
|
2
2
|
|
3
|
-
# rubocop:disable Metrics/
|
3
|
+
# rubocop:disable Metrics/BlockLength
|
4
4
|
module Shoryuken
|
5
5
|
module CLI
|
6
6
|
class SQS < Base
|
@@ -51,7 +51,7 @@ module Shoryuken
|
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
-
def find_all(url, limit
|
54
|
+
def find_all(url, limit)
|
55
55
|
count = 0
|
56
56
|
batch_size = limit > 10 ? 10 : limit
|
57
57
|
|
@@ -78,7 +78,7 @@ module Shoryuken
|
|
78
78
|
end
|
79
79
|
|
80
80
|
def list_and_print_queues(urls)
|
81
|
-
attrs = %w
|
81
|
+
attrs = %w[QueueArn ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible LastModifiedTimestamp]
|
82
82
|
|
83
83
|
entries = urls.map { |u| sqs.get_queue_attributes(queue_url: u, attribute_names: attrs).attributes }.map do |q|
|
84
84
|
[
|
data/bin/shoryuken
CHANGED
@@ -7,8 +7,6 @@ require 'aws-sdk-core'
|
|
7
7
|
require_relative 'cli/base'
|
8
8
|
require_relative 'cli/sqs'
|
9
9
|
require_relative '../lib/shoryuken/runner'
|
10
|
-
|
11
|
-
# rubocop:disable Metrics/AbcSize
|
12
10
|
module Shoryuken
|
13
11
|
module CLI
|
14
12
|
class Runner < Base
|
@@ -32,18 +30,14 @@ module Shoryuken
|
|
32
30
|
def start
|
33
31
|
opts = options.to_h.symbolize_keys
|
34
32
|
|
35
|
-
if opts[:config_file]
|
36
|
-
say '[DEPRECATED] Please use --config instead of --config-file', :yellow
|
37
|
-
end
|
33
|
+
say '[DEPRECATED] Please use --config instead of --config-file', :yellow if opts[:config_file]
|
38
34
|
|
39
35
|
opts[:config_file] = opts.delete(:config) if opts[:config]
|
40
36
|
|
41
37
|
# Keep compatibility with old CLI queue format
|
42
|
-
opts[:queues] =
|
38
|
+
opts[:queues] = opts[:queues].reject(&:empty?).map { |q| q.split(',') } if opts[:queues]
|
43
39
|
|
44
|
-
if
|
45
|
-
fail_task "You should set a logfile if you're going to daemonize"
|
46
|
-
end
|
40
|
+
fail_task "You should set a logfile if you're going to daemonize" if opts[:daemon] && opts[:logfile].nil?
|
47
41
|
|
48
42
|
Shoryuken::Runner.instance.run(opts.freeze)
|
49
43
|
end
|
@@ -2,7 +2,7 @@ require 'yaml'
|
|
2
2
|
require 'shoryuken'
|
3
3
|
|
4
4
|
# load SQS credentials
|
5
|
-
config = YAML.load File.read(File.join(File.expand_path(
|
5
|
+
config = YAML.load File.read(File.join(File.expand_path(__dir__), 'shoryuken.yml'))
|
6
6
|
|
7
7
|
Aws.config = config['aws']
|
8
8
|
|
@@ -18,11 +18,11 @@ if sqs.config['endpoint'] =~ /amazonaws.com/
|
|
18
18
|
|
19
19
|
dead_letter_queue_arn = sqs.get_queue_attributes(
|
20
20
|
queue_url: dead_letter_queue_url,
|
21
|
-
attribute_names: %w
|
21
|
+
attribute_names: %w[QueueArn]
|
22
22
|
).attributes['QueueArn']
|
23
23
|
|
24
24
|
attributes = {}
|
25
|
-
attributes['RedrivePolicy'] = %
|
25
|
+
attributes['RedrivePolicy'] = %({"maxReceiveCount":"7", "deadLetterTargetArn":"#{dead_letter_queue_arn}"})
|
26
26
|
|
27
27
|
sqs.set_queue_attributes queue_url: default_queue_url, attributes: attributes
|
28
28
|
end
|
data/examples/default_worker.rb
CHANGED
data/lib/shoryuken.rb
CHANGED
data/lib/shoryuken/core_ext.rb
CHANGED
data/lib/shoryuken/fetcher.rb
CHANGED
@@ -25,7 +25,7 @@ module Shoryuken
|
|
25
25
|
|
26
26
|
private
|
27
27
|
|
28
|
-
def fetch_with_auto_retry(max_attempts
|
28
|
+
def fetch_with_auto_retry(max_attempts)
|
29
29
|
attempts = 0
|
30
30
|
|
31
31
|
begin
|
@@ -48,8 +48,8 @@ module Shoryuken
|
|
48
48
|
options = receive_options(queue)
|
49
49
|
|
50
50
|
options[:max_number_of_messages] = max_number_of_messages(limit, options)
|
51
|
-
options[:message_attribute_names] = %w
|
52
|
-
options[:attribute_names] = %w
|
51
|
+
options[:message_attribute_names] = %w[All]
|
52
|
+
options[:attribute_names] = %w[All]
|
53
53
|
|
54
54
|
options.merge!(queue.options)
|
55
55
|
|
data/lib/shoryuken/logging.rb
CHANGED
@@ -5,7 +5,7 @@ module Shoryuken
|
|
5
5
|
module Logging
|
6
6
|
class Pretty < Logger::Formatter
|
7
7
|
# Provide a call() method that returns the formatted message.
|
8
|
-
def call(severity, time,
|
8
|
+
def call(severity, time, _program_name, message)
|
9
9
|
"#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
|
10
10
|
end
|
11
11
|
|
@@ -25,7 +25,7 @@ module Shoryuken
|
|
25
25
|
class MessageVisibilityExtender
|
26
26
|
include Util
|
27
27
|
|
28
|
-
def auto_extend(
|
28
|
+
def auto_extend(_worker, queue, sqs_msg, _body)
|
29
29
|
queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
|
30
30
|
|
31
31
|
Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
|
@@ -4,25 +4,23 @@ module Shoryuken
|
|
4
4
|
class Timing
|
5
5
|
include Util
|
6
6
|
|
7
|
-
def call(
|
8
|
-
|
9
|
-
started_at = Time.now
|
7
|
+
def call(_worker, queue, _sqs_msg, _body)
|
8
|
+
started_at = Time.now
|
10
9
|
|
11
|
-
|
10
|
+
logger.info { "started at #{started_at}" }
|
12
11
|
|
13
|
-
|
12
|
+
yield
|
14
13
|
|
15
|
-
|
14
|
+
total_time = elapsed(started_at)
|
16
15
|
|
17
|
-
|
18
|
-
|
19
|
-
end
|
20
|
-
|
21
|
-
logger.info { "completed in: #{total_time} ms" }
|
22
|
-
rescue
|
23
|
-
logger.info { "failed in: #{elapsed(started_at)} ms" }
|
24
|
-
raise
|
16
|
+
if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
|
17
|
+
logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
|
25
18
|
end
|
19
|
+
|
20
|
+
logger.info { "completed in: #{total_time} ms" }
|
21
|
+
rescue
|
22
|
+
logger.info { "failed in: #{elapsed(started_at)} ms" }
|
23
|
+
raise
|
26
24
|
end
|
27
25
|
end
|
28
26
|
end
|
@@ -4,13 +4,13 @@ module Shoryuken
|
|
4
4
|
def initialize(queues)
|
5
5
|
# Priority ordering of the queues, highest priority first
|
6
6
|
@queues = queues
|
7
|
-
|
8
|
-
|
9
|
-
|
7
|
+
.group_by { |q| q }
|
8
|
+
.sort_by { |_, qs| -qs.count }
|
9
|
+
.map(&:first)
|
10
10
|
|
11
11
|
# Pause status of the queues, default to past time (unpaused)
|
12
12
|
@paused_until = queues
|
13
|
-
|
13
|
+
.each_with_object({}) { |queue, h| h[queue] = Time.at(0) }
|
14
14
|
|
15
15
|
# Start queues at 0
|
16
16
|
reset_next_queue
|
@@ -31,10 +31,10 @@ module Shoryuken
|
|
31
31
|
|
32
32
|
def active_queues
|
33
33
|
@queues
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
34
|
+
.reverse
|
35
|
+
.map.with_index(1)
|
36
|
+
.reject { |q, _| queue_paused?(q) }
|
37
|
+
.reverse
|
38
38
|
end
|
39
39
|
|
40
40
|
private
|