shoryuken 7.0.2 → 7.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/push.yml +3 -3
- data/.github/workflows/specs.yml +32 -5
- data/.github/workflows/verify-action-pins.yml +1 -1
- data/.ruby-version +1 -1
- data/.yard-lint.yml +29 -1
- data/CHANGELOG.md +171 -0
- data/Gemfile.lint.lock +8 -7
- data/bin/cli/sqs.rb +67 -3
- data/bin/integrations +52 -34
- data/lib/active_job/extensions.rb +5 -8
- data/lib/active_job/queue_adapters/shoryuken_adapter.rb +6 -5
- data/lib/shoryuken/active_job/current_attributes.rb +33 -7
- data/lib/shoryuken/body_parser.rb +1 -2
- data/lib/shoryuken/client.rb +15 -6
- data/lib/shoryuken/default_exception_handler.rb +2 -3
- data/lib/shoryuken/errors.rb +2 -4
- data/lib/shoryuken/fetcher.rb +7 -1
- data/lib/shoryuken/helpers/atomic_counter.rb +6 -9
- data/lib/shoryuken/helpers/atomic_hash.rb +9 -15
- data/lib/shoryuken/helpers/hash_utils.rb +5 -7
- data/lib/shoryuken/helpers/string_utils.rb +6 -8
- data/lib/shoryuken/helpers/timer_task.rb +19 -2
- data/lib/shoryuken/inline_message.rb +3 -4
- data/lib/shoryuken/launcher.rb +19 -8
- data/lib/shoryuken/manager.rb +65 -6
- data/lib/shoryuken/message.rb +2 -3
- data/lib/shoryuken/middleware/chain.rb +7 -13
- data/lib/shoryuken/middleware/entry.rb +1 -2
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +30 -2
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +40 -18
- data/lib/shoryuken/middleware/server/non_retryable_exception.rb +17 -8
- data/lib/shoryuken/middleware/server/timing.rb +2 -3
- data/lib/shoryuken/options.rb +37 -4
- data/lib/shoryuken/polling/base_strategy.rb +16 -23
- data/lib/shoryuken/polling/queue_configuration.rb +8 -12
- data/lib/shoryuken/polling/strict_priority.rb +26 -14
- data/lib/shoryuken/polling/weighted_round_robin.rb +40 -27
- data/lib/shoryuken/queue.rb +41 -4
- data/lib/shoryuken/util.rb +4 -1
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker/default_executor.rb +11 -5
- data/lib/shoryuken/worker/inline_executor.rb +10 -5
- data/lib/shoryuken/worker.rb +15 -13
- data/lib/shoryuken.rb +6 -3
- data/renovate.json +16 -2
- data/shoryuken.gemspec +9 -0
- data/spec/integration/active_job/current_attributes/cross_job_reset_spec.rb +47 -0
- data/spec/integration/active_job/current_attributes/incremental_persist_spec.rb +76 -0
- data/spec/integration/active_job/fifo_dedup_opt_out/fifo_dedup_opt_out_spec.rb +67 -0
- data/spec/integration/auto_extend_visibility/short_visibility_timeout_spec.rb +52 -0
- data/spec/integration/cli/find_all_spec.rb +120 -0
- data/spec/integration/concurrent_processing/processor_accounting_spec.rb +94 -0
- data/spec/integration/fifo_ordering/fifo_max_messages_cap_spec.rb +96 -0
- data/spec/integration/launcher/double_graceful_stop_spec.rb +71 -0
- data/spec/integration/launcher/embedded_dispatch_error_spec.rb +85 -0
- data/spec/integration/launcher/global_executor_preserved_spec.rb +76 -0
- data/spec/integration/launcher/graceful_stop_timeout_spec.rb +74 -0
- data/spec/integration/message_operations/partial_batch_delete_spec.rb +67 -0
- data/spec/integration/non_retryable_exception/non_retryable_exception_spec.rb +1 -1
- data/spec/integration/non_retryable_exception/with_retry_intervals_spec.rb +115 -0
- data/spec/integrations_helper.rb +10 -9
- data/spec/lib/shoryuken/client_spec.rb +62 -0
- data/spec/lib/shoryuken/fetcher_spec.rb +13 -0
- data/spec/lib/shoryuken/helpers/timer_task_spec.rb +24 -0
- data/spec/lib/shoryuken/launcher_spec.rb +38 -0
- data/spec/lib/shoryuken/manager_spec.rb +147 -0
- data/spec/lib/shoryuken/middleware/server/auto_extend_visibility_spec.rb +35 -0
- data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +88 -0
- data/spec/lib/shoryuken/polling/strict_priority_spec.rb +25 -0
- data/spec/lib/shoryuken/polling/weighted_round_robin_spec.rb +50 -0
- data/spec/lib/shoryuken/queue_spec.rb +123 -0
- data/spec/lib/shoryuken/util_spec.rb +26 -0
- data/spec/lib/shoryuken/worker/default_executor_spec.rb +13 -0
- data/spec/lib/shoryuken/worker/inline_executor_spec.rb +12 -0
- data/spec/shared_examples_for_active_job.rb +18 -0
- data/spec/spec_helper.rb +40 -20
- metadata +35 -3
data/lib/shoryuken/queue.rb
CHANGED
|
@@ -52,11 +52,18 @@ module Shoryuken
|
|
|
52
52
|
failed_messages = client.delete_message_batch(
|
|
53
53
|
options.merge(queue_url: url)
|
|
54
54
|
).failed || []
|
|
55
|
-
|
|
55
|
+
|
|
56
|
+
# Log every failure (not just the first) and return a plain boolean.
|
|
57
|
+
# The previous `any? { logger.error ... }` short-circuited after the first
|
|
58
|
+
# failure - so the rest went unlogged - and only returned truthy because
|
|
59
|
+
# Logger#error happens to return true.
|
|
60
|
+
failed_messages.each do |failure|
|
|
56
61
|
logger.error do
|
|
57
62
|
"Could not delete #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
|
|
58
63
|
end
|
|
59
64
|
end
|
|
65
|
+
|
|
66
|
+
failed_messages.any?
|
|
60
67
|
end
|
|
61
68
|
|
|
62
69
|
# Sends a single message to the queue
|
|
@@ -82,7 +89,11 @@ module Shoryuken
|
|
|
82
89
|
# @option options [Array<Hash>] :entries message entries to send
|
|
83
90
|
# @return [Aws::SQS::Types::SendMessageBatchResult] the batch send result
|
|
84
91
|
def send_messages(options)
|
|
85
|
-
client.send_message_batch(sanitize_messages!(options).merge(queue_url: url))
|
|
92
|
+
response = client.send_message_batch(sanitize_messages!(options).merge(queue_url: url))
|
|
93
|
+
|
|
94
|
+
log_failed_sends(response)
|
|
95
|
+
|
|
96
|
+
response
|
|
86
97
|
end
|
|
87
98
|
|
|
88
99
|
# Receives messages from the queue
|
|
@@ -113,6 +124,24 @@ module Shoryuken
|
|
|
113
124
|
|
|
114
125
|
private
|
|
115
126
|
|
|
127
|
+
# Logs any per-entry failures from a batch send. SQS reports these in
|
|
128
|
+
# response.failed (throttling, oversize/malformed entries, or a same-batch
|
|
129
|
+
# duplicate message_deduplication_id) rather than raising, so without this
|
|
130
|
+
# they are silently swallowed - notably by ShoryukenAdapter#enqueue_all,
|
|
131
|
+
# which reads only response.successful. Mirrors delete_messages.
|
|
132
|
+
#
|
|
133
|
+
# @param response [Aws::SQS::Types::SendMessageBatchResult] the batch result
|
|
134
|
+
# @return [void]
|
|
135
|
+
def log_failed_sends(response)
|
|
136
|
+
return unless response.respond_to?(:failed)
|
|
137
|
+
|
|
138
|
+
Array(response.failed).each do |failure|
|
|
139
|
+
logger.error do
|
|
140
|
+
"Could not send #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
116
145
|
# Initializes the FIFO attribute by calling fifo?
|
|
117
146
|
#
|
|
118
147
|
# @return [Boolean] whether the queue is FIFO
|
|
@@ -217,8 +246,16 @@ module Shoryuken
|
|
|
217
246
|
def add_fifo_attributes!(options)
|
|
218
247
|
return unless fifo?
|
|
219
248
|
|
|
220
|
-
options[:message_group_id]
|
|
221
|
-
|
|
249
|
+
options[:message_group_id] ||= MESSAGE_GROUP_ID
|
|
250
|
+
|
|
251
|
+
# Auto-generate a content-based dedup id unless disabled. With it on, two
|
|
252
|
+
# sends of an identical body within SQS's 5-minute window are deduplicated
|
|
253
|
+
# (the second is silently dropped); disable it to send such messages
|
|
254
|
+
# distinctly (requires an explicit message_deduplication_id or the queue's
|
|
255
|
+
# ContentBasedDeduplication attribute).
|
|
256
|
+
if Shoryuken.fifo_message_deduplication?
|
|
257
|
+
options[:message_deduplication_id] ||= Digest::SHA256.hexdigest(options[:message_body].to_s)
|
|
258
|
+
end
|
|
222
259
|
|
|
223
260
|
options
|
|
224
261
|
end
|
data/lib/shoryuken/util.rb
CHANGED
|
@@ -20,7 +20,10 @@ module Shoryuken
|
|
|
20
20
|
def fire_event(event, reverse = false, event_options = {})
|
|
21
21
|
logger.debug { "Firing '#{event}' lifecycle event" }
|
|
22
22
|
arr = Shoryuken.options[:lifecycle_events][event]
|
|
23
|
-
|
|
23
|
+
# reverse (not reverse!) so the stored handler array keeps its original
|
|
24
|
+
# order: events fired more than once with reverse: true (e.g. :shutdown
|
|
25
|
+
# via stop then stop!) must not alternate their handler order.
|
|
26
|
+
arr = arr.reverse if reverse
|
|
24
27
|
arr.each do |block|
|
|
25
28
|
block.call(event_options)
|
|
26
29
|
rescue => e
|
data/lib/shoryuken/version.rb
CHANGED
|
@@ -15,11 +15,17 @@ module Shoryuken
|
|
|
15
15
|
# @option options [String] :queue override the default queue
|
|
16
16
|
# @return [Aws::SQS::Types::SendMessageResult] the send result
|
|
17
17
|
def perform_async(worker_class, body, options = {})
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
18
|
+
# Work on a copy: callers may reuse the same options hash across
|
|
19
|
+
# enqueues, and mutating it (deleting :queue, adding :message_body)
|
|
20
|
+
# would silently reroute or corrupt later jobs. Merge - rather than
|
|
21
|
+
# mutate - the nested :message_attributes hash for the same reason.
|
|
22
|
+
options = options.dup
|
|
23
|
+
options[:message_attributes] = (options[:message_attributes] || {}).merge(
|
|
24
|
+
'shoryuken_class' => {
|
|
25
|
+
string_value: worker_class.to_s,
|
|
26
|
+
data_type: 'String'
|
|
27
|
+
}
|
|
28
|
+
)
|
|
23
29
|
|
|
24
30
|
options[:message_body] = body
|
|
25
31
|
|
|
@@ -15,13 +15,18 @@ module Shoryuken
|
|
|
15
15
|
# @option options [Hash] :message_attributes custom message attributes
|
|
16
16
|
# @return [Object] the result of the worker's perform method
|
|
17
17
|
def perform_async(worker_class, body, options = {})
|
|
18
|
+
# Work on a copy so reused/shared options hashes aren't mutated across
|
|
19
|
+
# calls (deleting :queue would reroute later jobs to the default
|
|
20
|
+
# queue). Merge - rather than mutate - the nested message_attributes.
|
|
21
|
+
options = options.dup
|
|
18
22
|
body = JSON.dump(body) if body.is_a?(Hash)
|
|
19
23
|
queue_name = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
|
|
20
|
-
message_attributes = options.delete(:message_attributes) || {}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
message_attributes = (options.delete(:message_attributes) || {}).merge(
|
|
25
|
+
'shoryuken_class' => {
|
|
26
|
+
string_value: worker_class.to_s,
|
|
27
|
+
data_type: 'String'
|
|
28
|
+
}
|
|
29
|
+
)
|
|
25
30
|
|
|
26
31
|
sqs_msg = InlineMessage.new(
|
|
27
32
|
body: body,
|
data/lib/shoryuken/worker.rb
CHANGED
|
@@ -4,10 +4,9 @@ module Shoryuken
|
|
|
4
4
|
# Worker module provides the core functionality for creating Shoryuken workers
|
|
5
5
|
# that process messages from Amazon SQS queues.
|
|
6
6
|
#
|
|
7
|
-
# Including this module in a class provides methods for configuring queue processing,
|
|
8
|
-
#
|
|
9
|
-
# processing
|
|
10
|
-
# various retry and visibility timeout strategies.
|
|
7
|
+
# Including this module in a class provides methods for configuring queue processing, enqueueing jobs, and setting up
|
|
8
|
+
# middleware. Workers can be configured for different processing patterns including single message processing, batch
|
|
9
|
+
# processing, and various retry and visibility timeout strategies.
|
|
11
10
|
#
|
|
12
11
|
# @example Basic worker implementation
|
|
13
12
|
# class EmailWorker
|
|
@@ -123,7 +122,8 @@ module Shoryuken
|
|
|
123
122
|
# @option opts [Boolean] :auto_delete (false) Automatically delete messages after processing
|
|
124
123
|
# @option opts [Boolean] :auto_visibility_timeout (false) Automatically extend message visibility
|
|
125
124
|
# @option opts [Array<Integer>] :retry_intervals Exponential backoff retry intervals in seconds
|
|
126
|
-
# @option opts [Array<Class>, Proc] :non_retryable_exceptions Exception classes or lambda that should
|
|
125
|
+
# @option opts [Array<Class>, Proc] :non_retryable_exceptions Exception classes or lambda that should
|
|
126
|
+
# skip retries and delete message immediately
|
|
127
127
|
# @option opts [Hash] :sqs Additional SQS client options
|
|
128
128
|
#
|
|
129
129
|
# @example Basic worker configuration
|
|
@@ -216,25 +216,27 @@ module Shoryuken
|
|
|
216
216
|
!!get_shoryuken_options['auto_visibility_timeout']
|
|
217
217
|
end
|
|
218
218
|
|
|
219
|
-
# Checks if exponential backoff retry is configured for this worker.
|
|
220
|
-
#
|
|
221
|
-
# increasing delays between attempts.
|
|
219
|
+
# Checks if exponential backoff retry is configured for this worker. When retry intervals are specified, failed jobs
|
|
220
|
+
# will be retried with increasing delays between attempts.
|
|
222
221
|
#
|
|
223
222
|
# @return [Boolean] true if retry intervals are configured
|
|
224
223
|
#
|
|
225
224
|
# @example Configuring exponential backoff
|
|
226
225
|
# shoryuken_options retry_intervals: [1, 5, 25, 125, 625]
|
|
227
|
-
# #
|
|
226
|
+
# # Retries after 1s, 5s, 25s, 125s, then 625s for every later attempt.
|
|
227
|
+
# # Shoryuken does not stop retrying on its own once the intervals are
|
|
228
|
+
# # exhausted - it keeps reusing the last interval. Configure an SQS
|
|
229
|
+
# # redrive policy (maxReceiveCount) to send exhausted messages to a
|
|
230
|
+
# # dead-letter queue.
|
|
228
231
|
#
|
|
229
232
|
# @see #shoryuken_options Documentation for configuring retry_intervals
|
|
230
233
|
def exponential_backoff?
|
|
231
234
|
!!get_shoryuken_options['retry_intervals']
|
|
232
235
|
end
|
|
233
236
|
|
|
234
|
-
# Checks if automatic message deletion is enabled for this worker.
|
|
235
|
-
# When
|
|
236
|
-
#
|
|
237
|
-
# or they will become visible again after the visibility timeout.
|
|
237
|
+
# Checks if automatic message deletion is enabled for this worker. When enabled, successfully processed messages are
|
|
238
|
+
# automatically deleted from the SQS queue. When disabled, you must manually delete messages or they will become
|
|
239
|
+
# visible again after the visibility timeout.
|
|
238
240
|
#
|
|
239
241
|
# @return [Boolean] true if auto delete is enabled
|
|
240
242
|
#
|
data/lib/shoryuken.rb
CHANGED
|
@@ -27,9 +27,8 @@ module Shoryuken
|
|
|
27
27
|
@_shoryuken_options ||= Shoryuken::Options.new
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
-
# Checks if the Shoryuken server is running and healthy.
|
|
31
|
-
#
|
|
32
|
-
# are running and able to process messages.
|
|
30
|
+
# Checks if the Shoryuken server is running and healthy. A server is considered healthy when all configured processing
|
|
31
|
+
# groups are running and able to process messages.
|
|
33
32
|
#
|
|
34
33
|
# @return [Boolean] true if the server is healthy
|
|
35
34
|
def self.healthy?
|
|
@@ -58,6 +57,10 @@ module Shoryuken
|
|
|
58
57
|
:stop_callback=,
|
|
59
58
|
:active_job_queue_name_prefixing?,
|
|
60
59
|
:active_job_queue_name_prefixing=,
|
|
60
|
+
:active_job_fifo_message_deduplication?,
|
|
61
|
+
:active_job_fifo_message_deduplication=,
|
|
62
|
+
:fifo_message_deduplication?,
|
|
63
|
+
:fifo_message_deduplication=,
|
|
61
64
|
:sqs_client,
|
|
62
65
|
:sqs_client=,
|
|
63
66
|
:sqs_client_receive_message_opts,
|
data/renovate.json
CHANGED
|
@@ -56,7 +56,21 @@
|
|
|
56
56
|
"ruby/setup-ruby",
|
|
57
57
|
"ruby"
|
|
58
58
|
],
|
|
59
|
-
"groupName": "ruby setup"
|
|
59
|
+
"groupName": "ruby setup",
|
|
60
|
+
"internalChecksFilter": "strict"
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
"description": "Let setup-ruby pass age gate before ruby so it is ready when the group PR is created",
|
|
64
|
+
"matchPackageNames": [
|
|
65
|
+
"ruby/setup-ruby"
|
|
66
|
+
],
|
|
67
|
+
"minimumReleaseAge": "5 days"
|
|
60
68
|
}
|
|
61
|
-
]
|
|
69
|
+
],
|
|
70
|
+
"lockFileMaintenance": {
|
|
71
|
+
"enabled": true,
|
|
72
|
+
"schedule": [
|
|
73
|
+
"before 4am on the first day of the month"
|
|
74
|
+
]
|
|
75
|
+
}
|
|
62
76
|
}
|
data/shoryuken.gemspec
CHANGED
|
@@ -11,6 +11,15 @@ Gem::Specification.new do |spec|
|
|
|
11
11
|
spec.homepage = 'https://github.com/ruby-shoryuken/shoryuken'
|
|
12
12
|
spec.license = 'LGPL-3.0'
|
|
13
13
|
|
|
14
|
+
spec.metadata = {
|
|
15
|
+
'homepage_uri' => spec.homepage,
|
|
16
|
+
'source_code_uri' => spec.homepage,
|
|
17
|
+
'changelog_uri' => "#{spec.homepage}/blob/main/CHANGELOG.md",
|
|
18
|
+
'bug_tracker_uri' => "#{spec.homepage}/issues",
|
|
19
|
+
'documentation_uri' => spec.homepage,
|
|
20
|
+
'rubygems_mfa_required' => 'true'
|
|
21
|
+
}
|
|
22
|
+
|
|
14
23
|
spec.files = `git ls-files -z`.split("\x0")
|
|
15
24
|
spec.executables = %w[shoryuken]
|
|
16
25
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# A message without a CurrentAttributes key must not inherit a previous job's
|
|
4
|
+
# context. Every registered class is reset after each job - even keyless ones -
|
|
5
|
+
# so values set during one job don't leak into the next on a reused worker
|
|
6
|
+
# thread.
|
|
7
|
+
|
|
8
|
+
setup_sqs
|
|
9
|
+
setup_active_job
|
|
10
|
+
|
|
11
|
+
require 'active_support/current_attributes'
|
|
12
|
+
require 'shoryuken/active_job/current_attributes'
|
|
13
|
+
|
|
14
|
+
queue_name = DT.queue
|
|
15
|
+
create_test_queue(queue_name)
|
|
16
|
+
|
|
17
|
+
class LeakCurrent < ActiveSupport::CurrentAttributes
|
|
18
|
+
attribute :user_id
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
Shoryuken::ActiveJob::CurrentAttributes.persist(LeakCurrent)
|
|
22
|
+
|
|
23
|
+
class CrossJobResetTestJob < ActiveJob::Base
|
|
24
|
+
def perform
|
|
25
|
+
# Record what the worker thread sees at the start of the job, then dirty
|
|
26
|
+
# Current. Without a reset after a keyless job, the second execution on the
|
|
27
|
+
# same thread would observe 'leaked' instead of nil.
|
|
28
|
+
DT[:observed] << LeakCurrent.user_id
|
|
29
|
+
LeakCurrent.user_id = 'leaked'
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
CrossJobResetTestJob.queue_as(queue_name)
|
|
34
|
+
|
|
35
|
+
# Concurrency 1 so the two jobs run sequentially on the same worker thread.
|
|
36
|
+
Shoryuken.add_group('default', 1)
|
|
37
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
38
|
+
Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
|
|
39
|
+
|
|
40
|
+
# Enqueued with an empty context, so both messages carry no cattr key.
|
|
41
|
+
CrossJobResetTestJob.perform_later
|
|
42
|
+
CrossJobResetTestJob.perform_later
|
|
43
|
+
|
|
44
|
+
poll_queues_until(timeout: 30) { DT[:observed].size >= 2 }
|
|
45
|
+
|
|
46
|
+
assert_equal([nil, nil], DT[:observed].first(2),
|
|
47
|
+
"CurrentAttributes leaked across jobs: #{DT[:observed].inspect}")
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# CurrentAttributes registered one class per persist call are all preserved.
|
|
4
|
+
#
|
|
5
|
+
# persist derived each storage key from the per-call index, so calling it once
|
|
6
|
+
# per class (as an app might across several initializers) made the third call
|
|
7
|
+
# reuse "cattr_0" and silently overwrite the second class - its attributes were
|
|
8
|
+
# then never serialized or restored.
|
|
9
|
+
|
|
10
|
+
setup_sqs
|
|
11
|
+
setup_active_job
|
|
12
|
+
|
|
13
|
+
require 'active_support/current_attributes'
|
|
14
|
+
require 'shoryuken/active_job/current_attributes'
|
|
15
|
+
|
|
16
|
+
queue_name = DT.queue
|
|
17
|
+
create_test_queue(queue_name)
|
|
18
|
+
|
|
19
|
+
class CurrentA < ActiveSupport::CurrentAttributes
|
|
20
|
+
attribute :a_val
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
class CurrentB < ActiveSupport::CurrentAttributes
|
|
24
|
+
attribute :b_val
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
class CurrentC < ActiveSupport::CurrentAttributes
|
|
28
|
+
attribute :c_val
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Registered incrementally, one class per call.
|
|
32
|
+
Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentA)
|
|
33
|
+
Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentB)
|
|
34
|
+
Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentC)
|
|
35
|
+
|
|
36
|
+
# Every class must be registered, each under a distinct storage key.
|
|
37
|
+
registered = Shoryuken::ActiveJob::CurrentAttributes.cattrs
|
|
38
|
+
assert_equal(
|
|
39
|
+
%w[CurrentA CurrentB CurrentC],
|
|
40
|
+
registered.values.sort,
|
|
41
|
+
'every persisted CurrentAttributes class must be registered (none silently dropped)'
|
|
42
|
+
)
|
|
43
|
+
assert_equal(3, registered.keys.uniq.size, 'each class must use a distinct storage key')
|
|
44
|
+
|
|
45
|
+
class IncrementalCurrentJob < ActiveJob::Base
|
|
46
|
+
def perform
|
|
47
|
+
DT[:executions] << {
|
|
48
|
+
a_val: CurrentA.a_val,
|
|
49
|
+
b_val: CurrentB.b_val,
|
|
50
|
+
c_val: CurrentC.c_val
|
|
51
|
+
}
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
IncrementalCurrentJob.queue_as(queue_name)
|
|
56
|
+
|
|
57
|
+
Shoryuken.add_group('default', 1)
|
|
58
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
59
|
+
Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
|
|
60
|
+
|
|
61
|
+
CurrentA.a_val = 'a-value'
|
|
62
|
+
CurrentB.b_val = 'b-value'
|
|
63
|
+
CurrentC.c_val = 'c-value'
|
|
64
|
+
|
|
65
|
+
IncrementalCurrentJob.perform_later
|
|
66
|
+
|
|
67
|
+
CurrentA.reset
|
|
68
|
+
CurrentB.reset
|
|
69
|
+
CurrentC.reset
|
|
70
|
+
|
|
71
|
+
poll_queues_until(timeout: 30) { DT[:executions].size >= 1 }
|
|
72
|
+
|
|
73
|
+
result = DT[:executions].first
|
|
74
|
+
assert_equal('a-value', result[:a_val])
|
|
75
|
+
assert_equal('b-value', result[:b_val], 'the second incrementally-persisted class must not be dropped')
|
|
76
|
+
assert_equal('c-value', result[:c_val])
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
|
|
5
|
+
# This spec tests opting out of the automatic FIFO message_deduplication_id that
|
|
6
|
+
# the ActiveJob adapter generates.
|
|
7
|
+
#
|
|
8
|
+
# For FIFO queues the adapter derives a content-based message_deduplication_id
|
|
9
|
+
# from the serialized job minus job_id/enqueued_at (see issues #457 / #750). That
|
|
10
|
+
# means two *distinct* enqueues of the same job class + args within SQS's 5-minute
|
|
11
|
+
# window collapse into one - the second is silently dropped. That is intentional,
|
|
12
|
+
# but it is a "skipped message" trap for users who legitimately enqueue the same
|
|
13
|
+
# work twice.
|
|
14
|
+
#
|
|
15
|
+
# Expected behavior: setting Shoryuken.active_job_fifo_message_deduplication = false
|
|
16
|
+
# stops the adapter from generating the id, so those enqueues are no longer
|
|
17
|
+
# silently deduplicated. The default (true) preserves the existing behavior.
|
|
18
|
+
#
|
|
19
|
+
# Regression: there was no way to disable the auto-generated deduplication id.
|
|
20
|
+
|
|
21
|
+
setup_active_job
|
|
22
|
+
|
|
23
|
+
class FifoOptOutJob < ActiveJob::Base
|
|
24
|
+
queue_as :fifo_opt_out
|
|
25
|
+
|
|
26
|
+
def perform(_arg); end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
fifo_queue_mock = Object.new
|
|
30
|
+
fifo_queue_mock.define_singleton_method(:fifo?) { true }
|
|
31
|
+
fifo_queue_mock.define_singleton_method(:name) { 'fifo_opt_out.fifo' }
|
|
32
|
+
|
|
33
|
+
captured = nil
|
|
34
|
+
fifo_queue_mock.define_singleton_method(:send_message) { |params| captured = params }
|
|
35
|
+
|
|
36
|
+
Shoryuken::Client.define_singleton_method(:queues) { |_queue_name = nil| fifo_queue_mock }
|
|
37
|
+
Shoryuken.define_singleton_method(:register_worker) { |*| nil }
|
|
38
|
+
|
|
39
|
+
# Default behavior: the adapter sets a content-based dedup id (excluding the
|
|
40
|
+
# per-enqueue job_id/enqueued_at), so two identical jobs would collapse to one.
|
|
41
|
+
FifoOptOutJob.perform_later('same-args')
|
|
42
|
+
|
|
43
|
+
assert(
|
|
44
|
+
captured.key?(:message_deduplication_id),
|
|
45
|
+
'by default a FIFO ActiveJob send gets an auto-generated message_deduplication_id'
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
body = captured[:message_body]
|
|
49
|
+
expected = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id', 'enqueued_at')))
|
|
50
|
+
assert_equal(expected, captured[:message_deduplication_id])
|
|
51
|
+
|
|
52
|
+
# Opt-out: with deduplication disabled the adapter no longer sets a dedup id, so
|
|
53
|
+
# distinct enqueues of identical jobs are not silently dropped.
|
|
54
|
+
Shoryuken.active_job_fifo_message_deduplication = false
|
|
55
|
+
captured = nil
|
|
56
|
+
FifoOptOutJob.perform_later('same-args')
|
|
57
|
+
|
|
58
|
+
refute(
|
|
59
|
+
captured.key?(:message_deduplication_id),
|
|
60
|
+
'with deduplication disabled the adapter must not set message_deduplication_id'
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# An explicit deduplication id must still be honored even when auto-generation is off.
|
|
64
|
+
captured = nil
|
|
65
|
+
FifoOptOutJob.set(message_deduplication_id: 'explicit-id').perform_later('same-args')
|
|
66
|
+
|
|
67
|
+
assert_equal('explicit-id', captured[:message_deduplication_id])
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests that auto_visibility_timeout still processes messages when the
|
|
4
|
+
# queue's visibility timeout is short (<= EXTEND_UPFRONT_SECONDS).
|
|
5
|
+
#
|
|
6
|
+
# AutoExtendVisibility schedules a TimerTask at execution_interval =
|
|
7
|
+
# visibility_timeout - EXTEND_UPFRONT_SECONDS (5s). When the queue's visibility
|
|
8
|
+
# timeout is <= 5s that interval is <= 0, and TimerTask#initialize raises
|
|
9
|
+
# ArgumentError *before* the worker runs - so every receive fails and the
|
|
10
|
+
# message is reprocessed (and re-fails) until it hits a DLQ.
|
|
11
|
+
#
|
|
12
|
+
# Expected behavior: a short visibility timeout no longer breaks processing; the
|
|
13
|
+
# extension interval is clamped to a positive value and the worker runs normally.
|
|
14
|
+
#
|
|
15
|
+
# Regression: auto_visibility_timeout + a <= 5s queue visibility timeout broke
|
|
16
|
+
# processing entirely.
|
|
17
|
+
|
|
18
|
+
require 'timeout'
|
|
19
|
+
|
|
20
|
+
setup_sqs
|
|
21
|
+
|
|
22
|
+
DT.clear
|
|
23
|
+
|
|
24
|
+
queue_name = DT.queue
|
|
25
|
+
# 5s visibility timeout -> interval would be 5 - 5 = 0 -> TimerTask raised pre-fix.
|
|
26
|
+
create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '5' })
|
|
27
|
+
Shoryuken.add_group('default', 1)
|
|
28
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
29
|
+
|
|
30
|
+
short_vt_worker = Class.new do
|
|
31
|
+
include Shoryuken::Worker
|
|
32
|
+
|
|
33
|
+
shoryuken_options auto_delete: true, auto_visibility_timeout: true
|
|
34
|
+
|
|
35
|
+
def perform(_sqs_msg, body)
|
|
36
|
+
DT[:processed] << body
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
short_vt_worker.get_shoryuken_options['queue'] = queue_name
|
|
41
|
+
Shoryuken.register_worker(queue_name, short_vt_worker)
|
|
42
|
+
|
|
43
|
+
queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
|
|
44
|
+
Shoryuken::Client.sqs.send_message(queue_url: queue_url, message_body: 'hello')
|
|
45
|
+
|
|
46
|
+
poll_queues_until(timeout: 20) { DT[:processed].size >= 1 }
|
|
47
|
+
|
|
48
|
+
assert_equal(
|
|
49
|
+
['hello'],
|
|
50
|
+
DT[:processed].to_a,
|
|
51
|
+
'auto_visibility_timeout must not break processing on a queue with a short visibility timeout'
|
|
52
|
+
)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# `shoryuken sqs dump`/`mv` drain a queue via CLI::SQS#find_all. With short
|
|
4
|
+
# polling, real (distributed) SQS routinely returns an empty batch while the
|
|
5
|
+
# queue still has messages, so find_all must use long polling and must not stop
|
|
6
|
+
# on the first empty response - otherwise dump/mv silently process only a
|
|
7
|
+
# fraction of the queue.
|
|
8
|
+
#
|
|
9
|
+
# ElasticMQ never produces those false-empties (nor visibility-timeout re-reads),
|
|
10
|
+
# so this drives find_all with scripted clients to verify the behavior
|
|
11
|
+
# deterministically.
|
|
12
|
+
#
|
|
13
|
+
# Lives under spec/integration (not spec/lib) because requiring bin/cli defines
|
|
14
|
+
# Shoryuken::CLI, and Shoryuken#server? is `defined?(Shoryuken::CLI)` - loading
|
|
15
|
+
# it in the unit suite would flip server? for every other spec. Integration
|
|
16
|
+
# specs each run in their own process, so that's contained here.
|
|
17
|
+
|
|
18
|
+
require 'thor'
|
|
19
|
+
require_relative '../../../bin/cli/base'
|
|
20
|
+
require_relative '../../../bin/cli/sqs'
|
|
21
|
+
|
|
22
|
+
Msg = Struct.new(:message_id, :receipt_handle)
|
|
23
|
+
|
|
24
|
+
# A scripted SQS client: hands out pre-canned batches in order (to simulate
|
|
25
|
+
# false-empties and visibility-timeout re-reads), ignoring max_number_of_messages
|
|
26
|
+
# and recording the wait_time_seconds it was asked for.
|
|
27
|
+
class ScriptedSqsClient
|
|
28
|
+
attr_reader :wait_times
|
|
29
|
+
|
|
30
|
+
def initialize(batches)
|
|
31
|
+
@batches = batches.dup
|
|
32
|
+
@wait_times = []
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def receive_message(params)
|
|
36
|
+
@wait_times << params[:wait_time_seconds]
|
|
37
|
+
Struct.new(:messages).new(@batches.shift || [])
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# A more realistic client backed by a flat list that honours
|
|
42
|
+
# max_number_of_messages, so the finite-limit / batch-sizing path can be driven.
|
|
43
|
+
class FlatSqsClient
|
|
44
|
+
def initialize(messages)
|
|
45
|
+
@messages = messages.dup
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def receive_message(params)
|
|
49
|
+
Struct.new(:messages).new(@messages.shift(params[:max_number_of_messages]))
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def find_all_with(client, limit)
|
|
54
|
+
cli = Shoryuken::CLI::SQS.allocate
|
|
55
|
+
cli.instance_variable_set(:@_sqs, client)
|
|
56
|
+
|
|
57
|
+
collected = []
|
|
58
|
+
count = cli.send(:find_all, 'http://example.com/q', limit) { |msg| collected << msg }
|
|
59
|
+
[collected, count]
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# --- Drains past a false-empty batch, using long polling -------------------
|
|
63
|
+
client = ScriptedSqsClient.new(
|
|
64
|
+
[
|
|
65
|
+
[Msg.new('a'), Msg.new('b'), Msg.new('c')],
|
|
66
|
+
[Msg.new('d'), Msg.new('e')],
|
|
67
|
+
[], # false-empty while 'f' is still queued
|
|
68
|
+
[Msg.new('f')]
|
|
69
|
+
]
|
|
70
|
+
)
|
|
71
|
+
collected, = find_all_with(client, Float::INFINITY)
|
|
72
|
+
|
|
73
|
+
assert_equal(%w[a b c d e f], collected.map(&:message_id), 'find_all should drain past a false-empty batch')
|
|
74
|
+
assert(
|
|
75
|
+
client.wait_times.any? && client.wait_times.all? { |w| w && w.positive? },
|
|
76
|
+
"find_all should use long polling, saw wait_time_seconds: #{client.wait_times.uniq.inspect}"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# --- Does not re-yield a message that reappears after its visibility timeout ---
|
|
80
|
+
# 'a' is handed back on a later receive (its visibility timeout lapsed before
|
|
81
|
+
# dump/mv deleted it). find_all must yield and count it only once.
|
|
82
|
+
reappearing = ScriptedSqsClient.new(
|
|
83
|
+
[
|
|
84
|
+
[Msg.new('a'), Msg.new('b')],
|
|
85
|
+
[Msg.new('a')], # re-read of 'a' while it is still in the queue
|
|
86
|
+
[Msg.new('c')]
|
|
87
|
+
]
|
|
88
|
+
)
|
|
89
|
+
collected, count = find_all_with(reappearing, Float::INFINITY)
|
|
90
|
+
|
|
91
|
+
assert_equal(%w[a b c], collected.map(&:message_id), 'find_all must not re-yield a re-read message')
|
|
92
|
+
assert_equal(3, count, 'find_all count must not double-count a re-read message')
|
|
93
|
+
|
|
94
|
+
# --- Retains the newest receipt handle for a re-read message ----------------
|
|
95
|
+
# SQS only honors the most recently received receipt handle for deletion, and
|
|
96
|
+
# dump/mv batch_delete after the drain, so the yielded (and later deleted)
|
|
97
|
+
# message must carry the handle from the latest receive, not the first.
|
|
98
|
+
handles = ScriptedSqsClient.new(
|
|
99
|
+
[
|
|
100
|
+
[Msg.new('x', 'rh-x-1'), Msg.new('y', 'rh-y')],
|
|
101
|
+
[Msg.new('x', 'rh-x-2')], # re-read of 'x' with a fresh handle
|
|
102
|
+
[Msg.new('z', 'rh-z')]
|
|
103
|
+
]
|
|
104
|
+
)
|
|
105
|
+
collected, = find_all_with(handles, Float::INFINITY)
|
|
106
|
+
|
|
107
|
+
x = collected.find { |msg| msg.message_id == 'x' }
|
|
108
|
+
assert_equal('rh-x-2', x.receipt_handle, 'find_all must retain the newest receipt handle for a re-read message')
|
|
109
|
+
assert_equal(%w[x y z], collected.map(&:message_id), 'a re-read must not be yielded twice')
|
|
110
|
+
|
|
111
|
+
# --- Stops at a finite limit without over-fetching -------------------------
|
|
112
|
+
flat = FlatSqsClient.new((1..25).map { |i| Msg.new("m#{i}") })
|
|
113
|
+
collected, count = find_all_with(flat, 15)
|
|
114
|
+
|
|
115
|
+
assert_equal(15, count, 'find_all must stop once the limit is reached')
|
|
116
|
+
assert_equal(
|
|
117
|
+
(1..15).map { |i| "m#{i}" },
|
|
118
|
+
collected.map(&:message_id),
|
|
119
|
+
'find_all must yield exactly the first `limit` messages'
|
|
120
|
+
)
|