shoryuken 7.0.2 → 7.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/push.yml +3 -3
  3. data/.github/workflows/specs.yml +32 -5
  4. data/.github/workflows/verify-action-pins.yml +1 -1
  5. data/.ruby-version +1 -1
  6. data/.yard-lint.yml +29 -1
  7. data/CHANGELOG.md +171 -0
  8. data/Gemfile.lint.lock +8 -7
  9. data/bin/cli/sqs.rb +67 -3
  10. data/bin/integrations +52 -34
  11. data/lib/active_job/extensions.rb +5 -8
  12. data/lib/active_job/queue_adapters/shoryuken_adapter.rb +6 -5
  13. data/lib/shoryuken/active_job/current_attributes.rb +33 -7
  14. data/lib/shoryuken/body_parser.rb +1 -2
  15. data/lib/shoryuken/client.rb +15 -6
  16. data/lib/shoryuken/default_exception_handler.rb +2 -3
  17. data/lib/shoryuken/errors.rb +2 -4
  18. data/lib/shoryuken/fetcher.rb +7 -1
  19. data/lib/shoryuken/helpers/atomic_counter.rb +6 -9
  20. data/lib/shoryuken/helpers/atomic_hash.rb +9 -15
  21. data/lib/shoryuken/helpers/hash_utils.rb +5 -7
  22. data/lib/shoryuken/helpers/string_utils.rb +6 -8
  23. data/lib/shoryuken/helpers/timer_task.rb +19 -2
  24. data/lib/shoryuken/inline_message.rb +3 -4
  25. data/lib/shoryuken/launcher.rb +19 -8
  26. data/lib/shoryuken/manager.rb +65 -6
  27. data/lib/shoryuken/message.rb +2 -3
  28. data/lib/shoryuken/middleware/chain.rb +7 -13
  29. data/lib/shoryuken/middleware/entry.rb +1 -2
  30. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +30 -2
  31. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +40 -18
  32. data/lib/shoryuken/middleware/server/non_retryable_exception.rb +17 -8
  33. data/lib/shoryuken/middleware/server/timing.rb +2 -3
  34. data/lib/shoryuken/options.rb +37 -4
  35. data/lib/shoryuken/polling/base_strategy.rb +16 -23
  36. data/lib/shoryuken/polling/queue_configuration.rb +8 -12
  37. data/lib/shoryuken/polling/strict_priority.rb +26 -14
  38. data/lib/shoryuken/polling/weighted_round_robin.rb +40 -27
  39. data/lib/shoryuken/queue.rb +41 -4
  40. data/lib/shoryuken/util.rb +4 -1
  41. data/lib/shoryuken/version.rb +1 -1
  42. data/lib/shoryuken/worker/default_executor.rb +11 -5
  43. data/lib/shoryuken/worker/inline_executor.rb +10 -5
  44. data/lib/shoryuken/worker.rb +15 -13
  45. data/lib/shoryuken.rb +6 -3
  46. data/renovate.json +16 -2
  47. data/shoryuken.gemspec +9 -0
  48. data/spec/integration/active_job/current_attributes/cross_job_reset_spec.rb +47 -0
  49. data/spec/integration/active_job/current_attributes/incremental_persist_spec.rb +76 -0
  50. data/spec/integration/active_job/fifo_dedup_opt_out/fifo_dedup_opt_out_spec.rb +67 -0
  51. data/spec/integration/auto_extend_visibility/short_visibility_timeout_spec.rb +52 -0
  52. data/spec/integration/cli/find_all_spec.rb +120 -0
  53. data/spec/integration/concurrent_processing/processor_accounting_spec.rb +94 -0
  54. data/spec/integration/fifo_ordering/fifo_max_messages_cap_spec.rb +96 -0
  55. data/spec/integration/launcher/double_graceful_stop_spec.rb +71 -0
  56. data/spec/integration/launcher/embedded_dispatch_error_spec.rb +85 -0
  57. data/spec/integration/launcher/global_executor_preserved_spec.rb +76 -0
  58. data/spec/integration/launcher/graceful_stop_timeout_spec.rb +74 -0
  59. data/spec/integration/message_operations/partial_batch_delete_spec.rb +67 -0
  60. data/spec/integration/non_retryable_exception/non_retryable_exception_spec.rb +1 -1
  61. data/spec/integration/non_retryable_exception/with_retry_intervals_spec.rb +115 -0
  62. data/spec/integrations_helper.rb +10 -9
  63. data/spec/lib/shoryuken/client_spec.rb +62 -0
  64. data/spec/lib/shoryuken/fetcher_spec.rb +13 -0
  65. data/spec/lib/shoryuken/helpers/timer_task_spec.rb +24 -0
  66. data/spec/lib/shoryuken/launcher_spec.rb +38 -0
  67. data/spec/lib/shoryuken/manager_spec.rb +147 -0
  68. data/spec/lib/shoryuken/middleware/server/auto_extend_visibility_spec.rb +35 -0
  69. data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +88 -0
  70. data/spec/lib/shoryuken/polling/strict_priority_spec.rb +25 -0
  71. data/spec/lib/shoryuken/polling/weighted_round_robin_spec.rb +50 -0
  72. data/spec/lib/shoryuken/queue_spec.rb +123 -0
  73. data/spec/lib/shoryuken/util_spec.rb +26 -0
  74. data/spec/lib/shoryuken/worker/default_executor_spec.rb +13 -0
  75. data/spec/lib/shoryuken/worker/inline_executor_spec.rb +12 -0
  76. data/spec/shared_examples_for_active_job.rb +18 -0
  77. data/spec/spec_helper.rb +40 -20
  78. metadata +35 -3
@@ -52,11 +52,18 @@ module Shoryuken
52
52
  failed_messages = client.delete_message_batch(
53
53
  options.merge(queue_url: url)
54
54
  ).failed || []
55
- failed_messages.any? do |failure|
55
+
56
+ # Log every failure (not just the first) and return a plain boolean.
57
+ # The previous `any? { logger.error ... }` short-circuited after the first
58
+ # failure - so the rest went unlogged - and only returned truthy because
59
+ # Logger#error happens to return true.
60
+ failed_messages.each do |failure|
56
61
  logger.error do
57
62
  "Could not delete #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
58
63
  end
59
64
  end
65
+
66
+ failed_messages.any?
60
67
  end
61
68
 
62
69
  # Sends a single message to the queue
@@ -82,7 +89,11 @@ module Shoryuken
82
89
  # @option options [Array<Hash>] :entries message entries to send
83
90
  # @return [Aws::SQS::Types::SendMessageBatchResult] the batch send result
84
91
  def send_messages(options)
85
- client.send_message_batch(sanitize_messages!(options).merge(queue_url: url))
92
+ response = client.send_message_batch(sanitize_messages!(options).merge(queue_url: url))
93
+
94
+ log_failed_sends(response)
95
+
96
+ response
86
97
  end
87
98
 
88
99
  # Receives messages from the queue
@@ -113,6 +124,24 @@ module Shoryuken
113
124
 
114
125
  private
115
126
 
127
+ # Logs any per-entry failures from a batch send. SQS reports these in
128
+ # response.failed (throttling, oversize/malformed entries, or a same-batch
129
+ # duplicate message_deduplication_id) rather than raising, so without this
130
+ # they are silently swallowed - notably by ShoryukenAdapter#enqueue_all,
131
+ # which reads only response.successful. Mirrors delete_messages.
132
+ #
133
+ # @param response [Aws::SQS::Types::SendMessageBatchResult] the batch result
134
+ # @return [void]
135
+ def log_failed_sends(response)
136
+ return unless response.respond_to?(:failed)
137
+
138
+ Array(response.failed).each do |failure|
139
+ logger.error do
140
+ "Could not send #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
141
+ end
142
+ end
143
+ end
144
+
116
145
  # Initializes the FIFO attribute by calling fifo?
117
146
  #
118
147
  # @return [Boolean] whether the queue is FIFO
@@ -217,8 +246,16 @@ module Shoryuken
217
246
  def add_fifo_attributes!(options)
218
247
  return unless fifo?
219
248
 
220
- options[:message_group_id] ||= MESSAGE_GROUP_ID
221
- options[:message_deduplication_id] ||= Digest::SHA256.hexdigest(options[:message_body].to_s)
249
+ options[:message_group_id] ||= MESSAGE_GROUP_ID
250
+
251
+ # Auto-generate a content-based dedup id unless disabled. With it on, two
252
+ # sends of an identical body within SQS's 5-minute window are deduplicated
253
+ # (the second is silently dropped); disable it to send such messages
254
+ # distinctly (requires an explicit message_deduplication_id or the queue's
255
+ # ContentBasedDeduplication attribute).
256
+ if Shoryuken.fifo_message_deduplication?
257
+ options[:message_deduplication_id] ||= Digest::SHA256.hexdigest(options[:message_body].to_s)
258
+ end
222
259
 
223
260
  options
224
261
  end
@@ -20,7 +20,10 @@ module Shoryuken
20
20
  def fire_event(event, reverse = false, event_options = {})
21
21
  logger.debug { "Firing '#{event}' lifecycle event" }
22
22
  arr = Shoryuken.options[:lifecycle_events][event]
23
- arr.reverse! if reverse
23
+ # reverse (not reverse!) so the stored handler array keeps its original
24
+ # order: events fired more than once with reverse: true (e.g. :shutdown
25
+ # via stop then stop!) must not alternate their handler order.
26
+ arr = arr.reverse if reverse
24
27
  arr.each do |block|
25
28
  block.call(event_options)
26
29
  rescue => e
@@ -2,5 +2,5 @@
2
2
 
3
3
  module Shoryuken
4
4
  # Current version of the Shoryuken gem
5
- VERSION = '7.0.2'
5
+ VERSION = '7.0.4'
6
6
  end
@@ -15,11 +15,17 @@ module Shoryuken
15
15
  # @option options [String] :queue override the default queue
16
16
  # @return [Aws::SQS::Types::SendMessageResult] the send result
17
17
  def perform_async(worker_class, body, options = {})
18
- options[:message_attributes] ||= {}
19
- options[:message_attributes]['shoryuken_class'] = {
20
- string_value: worker_class.to_s,
21
- data_type: 'String'
22
- }
18
+ # Work on a copy: callers may reuse the same options hash across
19
+ # enqueues, and mutating it (deleting :queue, adding :message_body)
20
+ # would silently reroute or corrupt later jobs. Merge - rather than
21
+ # mutate - the nested :message_attributes hash for the same reason.
22
+ options = options.dup
23
+ options[:message_attributes] = (options[:message_attributes] || {}).merge(
24
+ 'shoryuken_class' => {
25
+ string_value: worker_class.to_s,
26
+ data_type: 'String'
27
+ }
28
+ )
23
29
 
24
30
  options[:message_body] = body
25
31
 
@@ -15,13 +15,18 @@ module Shoryuken
15
15
  # @option options [Hash] :message_attributes custom message attributes
16
16
  # @return [Object] the result of the worker's perform method
17
17
  def perform_async(worker_class, body, options = {})
18
+ # Work on a copy so reused/shared options hashes aren't mutated across
19
+ # calls (deleting :queue would reroute later jobs to the default
20
+ # queue). Merge - rather than mutate - the nested message_attributes.
21
+ options = options.dup
18
22
  body = JSON.dump(body) if body.is_a?(Hash)
19
23
  queue_name = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
20
- message_attributes = options.delete(:message_attributes) || {}
21
- message_attributes['shoryuken_class'] = {
22
- string_value: worker_class.to_s,
23
- data_type: 'String'
24
- }
24
+ message_attributes = (options.delete(:message_attributes) || {}).merge(
25
+ 'shoryuken_class' => {
26
+ string_value: worker_class.to_s,
27
+ data_type: 'String'
28
+ }
29
+ )
25
30
 
26
31
  sqs_msg = InlineMessage.new(
27
32
  body: body,
@@ -4,10 +4,9 @@ module Shoryuken
4
4
  # Worker module provides the core functionality for creating Shoryuken workers
5
5
  # that process messages from Amazon SQS queues.
6
6
  #
7
- # Including this module in a class provides methods for configuring queue processing,
8
- # enqueueing jobs, and setting up middleware. Workers can be configured for different
9
- # processing patterns including single message processing, batch processing, and
10
- # various retry and visibility timeout strategies.
7
+ # Including this module in a class provides methods for configuring queue processing, enqueueing jobs, and setting up
8
+ # middleware. Workers can be configured for different processing patterns including single message processing, batch
9
+ # processing, and various retry and visibility timeout strategies.
11
10
  #
12
11
  # @example Basic worker implementation
13
12
  # class EmailWorker
@@ -123,7 +122,8 @@ module Shoryuken
123
122
  # @option opts [Boolean] :auto_delete (false) Automatically delete messages after processing
124
123
  # @option opts [Boolean] :auto_visibility_timeout (false) Automatically extend message visibility
125
124
  # @option opts [Array<Integer>] :retry_intervals Exponential backoff retry intervals in seconds
126
- # @option opts [Array<Class>, Proc] :non_retryable_exceptions Exception classes or lambda that should skip retries and delete message immediately
125
+ # @option opts [Array<Class>, Proc] :non_retryable_exceptions Exception classes or lambda that should
126
+ # skip retries and delete message immediately
127
127
  # @option opts [Hash] :sqs Additional SQS client options
128
128
  #
129
129
  # @example Basic worker configuration
@@ -216,25 +216,27 @@ module Shoryuken
216
216
  !!get_shoryuken_options['auto_visibility_timeout']
217
217
  end
218
218
 
219
- # Checks if exponential backoff retry is configured for this worker.
220
- # When retry intervals are specified, failed jobs will be retried with
221
- # increasing delays between attempts.
219
+ # Checks if exponential backoff retry is configured for this worker. When retry intervals are specified, failed jobs
220
+ # will be retried with increasing delays between attempts.
222
221
  #
223
222
  # @return [Boolean] true if retry intervals are configured
224
223
  #
225
224
  # @example Configuring exponential backoff
226
225
  # shoryuken_options retry_intervals: [1, 5, 25, 125, 625]
227
- # # Will retry after 1s, 5s, 25s, 125s, then 625s before giving up
226
+ # # Retries after 1s, 5s, 25s, 125s, then 625s for every later attempt.
227
+ # # Shoryuken does not stop retrying on its own once the intervals are
228
+ # # exhausted - it keeps reusing the last interval. Configure an SQS
229
+ # # redrive policy (maxReceiveCount) to send exhausted messages to a
230
+ # # dead-letter queue.
228
231
  #
229
232
  # @see #shoryuken_options Documentation for configuring retry_intervals
230
233
  def exponential_backoff?
231
234
  !!get_shoryuken_options['retry_intervals']
232
235
  end
233
236
 
234
- # Checks if automatic message deletion is enabled for this worker.
235
- # When enabled, successfully processed messages are automatically deleted
236
- # from the SQS queue. When disabled, you must manually delete messages
237
- # or they will become visible again after the visibility timeout.
237
+ # Checks if automatic message deletion is enabled for this worker. When enabled, successfully processed messages are
238
+ # automatically deleted from the SQS queue. When disabled, you must manually delete messages or they will become
239
+ # visible again after the visibility timeout.
238
240
  #
239
241
  # @return [Boolean] true if auto delete is enabled
240
242
  #
data/lib/shoryuken.rb CHANGED
@@ -27,9 +27,8 @@ module Shoryuken
27
27
  @_shoryuken_options ||= Shoryuken::Options.new
28
28
  end
29
29
 
30
- # Checks if the Shoryuken server is running and healthy.
31
- # A server is considered healthy when all configured processing groups
32
- # are running and able to process messages.
30
+ # Checks if the Shoryuken server is running and healthy. A server is considered healthy when all configured processing
31
+ # groups are running and able to process messages.
33
32
  #
34
33
  # @return [Boolean] true if the server is healthy
35
34
  def self.healthy?
@@ -58,6 +57,10 @@ module Shoryuken
58
57
  :stop_callback=,
59
58
  :active_job_queue_name_prefixing?,
60
59
  :active_job_queue_name_prefixing=,
60
+ :active_job_fifo_message_deduplication?,
61
+ :active_job_fifo_message_deduplication=,
62
+ :fifo_message_deduplication?,
63
+ :fifo_message_deduplication=,
61
64
  :sqs_client,
62
65
  :sqs_client=,
63
66
  :sqs_client_receive_message_opts,
data/renovate.json CHANGED
@@ -56,7 +56,21 @@
56
56
  "ruby/setup-ruby",
57
57
  "ruby"
58
58
  ],
59
- "groupName": "ruby setup"
59
+ "groupName": "ruby setup",
60
+ "internalChecksFilter": "strict"
61
+ },
62
+ {
63
+ "description": "Let setup-ruby pass age gate before ruby so it is ready when the group PR is created",
64
+ "matchPackageNames": [
65
+ "ruby/setup-ruby"
66
+ ],
67
+ "minimumReleaseAge": "5 days"
60
68
  }
61
- ]
69
+ ],
70
+ "lockFileMaintenance": {
71
+ "enabled": true,
72
+ "schedule": [
73
+ "before 4am on the first day of the month"
74
+ ]
75
+ }
62
76
  }
data/shoryuken.gemspec CHANGED
@@ -11,6 +11,15 @@ Gem::Specification.new do |spec|
11
11
  spec.homepage = 'https://github.com/ruby-shoryuken/shoryuken'
12
12
  spec.license = 'LGPL-3.0'
13
13
 
14
+ spec.metadata = {
15
+ 'homepage_uri' => spec.homepage,
16
+ 'source_code_uri' => spec.homepage,
17
+ 'changelog_uri' => "#{spec.homepage}/blob/main/CHANGELOG.md",
18
+ 'bug_tracker_uri' => "#{spec.homepage}/issues",
19
+ 'documentation_uri' => spec.homepage,
20
+ 'rubygems_mfa_required' => 'true'
21
+ }
22
+
14
23
  spec.files = `git ls-files -z`.split("\x0")
15
24
  spec.executables = %w[shoryuken]
16
25
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ # A message without a CurrentAttributes key must not inherit a previous job's
4
+ # context. Every registered class is reset after each job - even keyless ones -
5
+ # so values set during one job don't leak into the next on a reused worker
6
+ # thread.
7
+
8
+ setup_sqs
9
+ setup_active_job
10
+
11
+ require 'active_support/current_attributes'
12
+ require 'shoryuken/active_job/current_attributes'
13
+
14
+ queue_name = DT.queue
15
+ create_test_queue(queue_name)
16
+
17
+ class LeakCurrent < ActiveSupport::CurrentAttributes
18
+ attribute :user_id
19
+ end
20
+
21
+ Shoryuken::ActiveJob::CurrentAttributes.persist(LeakCurrent)
22
+
23
+ class CrossJobResetTestJob < ActiveJob::Base
24
+ def perform
25
+ # Record what the worker thread sees at the start of the job, then dirty
26
+ # Current. Without a reset after a keyless job, the second execution on the
27
+ # same thread would observe 'leaked' instead of nil.
28
+ DT[:observed] << LeakCurrent.user_id
29
+ LeakCurrent.user_id = 'leaked'
30
+ end
31
+ end
32
+
33
+ CrossJobResetTestJob.queue_as(queue_name)
34
+
35
+ # Concurrency 1 so the two jobs run sequentially on the same worker thread.
36
+ Shoryuken.add_group('default', 1)
37
+ Shoryuken.add_queue(queue_name, 1, 'default')
38
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
39
+
40
+ # Enqueued with an empty context, so both messages carry no cattr key.
41
+ CrossJobResetTestJob.perform_later
42
+ CrossJobResetTestJob.perform_later
43
+
44
+ poll_queues_until(timeout: 30) { DT[:observed].size >= 2 }
45
+
46
+ assert_equal([nil, nil], DT[:observed].first(2),
47
+ "CurrentAttributes leaked across jobs: #{DT[:observed].inspect}")
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ # CurrentAttributes registered one class per persist call are all preserved.
4
+ #
5
+ # persist derived each storage key from the per-call index, so calling it once
6
+ # per class (as an app might across several initializers) made the third call
7
+ # reuse "cattr_0" and silently overwrite the second class - its attributes were
8
+ # then never serialized or restored.
9
+
10
+ setup_sqs
11
+ setup_active_job
12
+
13
+ require 'active_support/current_attributes'
14
+ require 'shoryuken/active_job/current_attributes'
15
+
16
+ queue_name = DT.queue
17
+ create_test_queue(queue_name)
18
+
19
+ class CurrentA < ActiveSupport::CurrentAttributes
20
+ attribute :a_val
21
+ end
22
+
23
+ class CurrentB < ActiveSupport::CurrentAttributes
24
+ attribute :b_val
25
+ end
26
+
27
+ class CurrentC < ActiveSupport::CurrentAttributes
28
+ attribute :c_val
29
+ end
30
+
31
+ # Registered incrementally, one class per call.
32
+ Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentA)
33
+ Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentB)
34
+ Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentC)
35
+
36
+ # Every class must be registered, each under a distinct storage key.
37
+ registered = Shoryuken::ActiveJob::CurrentAttributes.cattrs
38
+ assert_equal(
39
+ %w[CurrentA CurrentB CurrentC],
40
+ registered.values.sort,
41
+ 'every persisted CurrentAttributes class must be registered (none silently dropped)'
42
+ )
43
+ assert_equal(3, registered.keys.uniq.size, 'each class must use a distinct storage key')
44
+
45
+ class IncrementalCurrentJob < ActiveJob::Base
46
+ def perform
47
+ DT[:executions] << {
48
+ a_val: CurrentA.a_val,
49
+ b_val: CurrentB.b_val,
50
+ c_val: CurrentC.c_val
51
+ }
52
+ end
53
+ end
54
+
55
+ IncrementalCurrentJob.queue_as(queue_name)
56
+
57
+ Shoryuken.add_group('default', 1)
58
+ Shoryuken.add_queue(queue_name, 1, 'default')
59
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
60
+
61
+ CurrentA.a_val = 'a-value'
62
+ CurrentB.b_val = 'b-value'
63
+ CurrentC.c_val = 'c-value'
64
+
65
+ IncrementalCurrentJob.perform_later
66
+
67
+ CurrentA.reset
68
+ CurrentB.reset
69
+ CurrentC.reset
70
+
71
+ poll_queues_until(timeout: 30) { DT[:executions].size >= 1 }
72
+
73
+ result = DT[:executions].first
74
+ assert_equal('a-value', result[:a_val])
75
+ assert_equal('b-value', result[:b_val], 'the second incrementally-persisted class must not be dropped')
76
+ assert_equal('c-value', result[:c_val])
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'digest'
4
+
5
+ # This spec tests opting out of the automatic FIFO message_deduplication_id that
6
+ # the ActiveJob adapter generates.
7
+ #
8
+ # For FIFO queues the adapter derives a content-based message_deduplication_id
9
+ # from the serialized job minus job_id/enqueued_at (see issues #457 / #750). That
10
+ # means two *distinct* enqueues of the same job class + args within SQS's 5-minute
11
+ # window collapse into one - the second is silently dropped. That is intentional,
12
+ # but it is a "skipped message" trap for users who legitimately enqueue the same
13
+ # work twice.
14
+ #
15
+ # Expected behavior: setting Shoryuken.active_job_fifo_message_deduplication = false
16
+ # stops the adapter from generating the id, so those enqueues are no longer
17
+ # silently deduplicated. The default (true) preserves the existing behavior.
18
+ #
19
+ # Regression: there was no way to disable the auto-generated deduplication id.
20
+
21
+ setup_active_job
22
+
23
+ class FifoOptOutJob < ActiveJob::Base
24
+ queue_as :fifo_opt_out
25
+
26
+ def perform(_arg); end
27
+ end
28
+
29
+ fifo_queue_mock = Object.new
30
+ fifo_queue_mock.define_singleton_method(:fifo?) { true }
31
+ fifo_queue_mock.define_singleton_method(:name) { 'fifo_opt_out.fifo' }
32
+
33
+ captured = nil
34
+ fifo_queue_mock.define_singleton_method(:send_message) { |params| captured = params }
35
+
36
+ Shoryuken::Client.define_singleton_method(:queues) { |_queue_name = nil| fifo_queue_mock }
37
+ Shoryuken.define_singleton_method(:register_worker) { |*| nil }
38
+
39
+ # Default behavior: the adapter sets a content-based dedup id (excluding the
40
+ # per-enqueue job_id/enqueued_at), so two identical jobs would collapse to one.
41
+ FifoOptOutJob.perform_later('same-args')
42
+
43
+ assert(
44
+ captured.key?(:message_deduplication_id),
45
+ 'by default a FIFO ActiveJob send gets an auto-generated message_deduplication_id'
46
+ )
47
+
48
+ body = captured[:message_body]
49
+ expected = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id', 'enqueued_at')))
50
+ assert_equal(expected, captured[:message_deduplication_id])
51
+
52
+ # Opt-out: with deduplication disabled the adapter no longer sets a dedup id, so
53
+ # distinct enqueues of identical jobs are not silently dropped.
54
+ Shoryuken.active_job_fifo_message_deduplication = false
55
+ captured = nil
56
+ FifoOptOutJob.perform_later('same-args')
57
+
58
+ refute(
59
+ captured.key?(:message_deduplication_id),
60
+ 'with deduplication disabled the adapter must not set message_deduplication_id'
61
+ )
62
+
63
+ # An explicit deduplication id must still be honored even when auto-generation is off.
64
+ captured = nil
65
+ FifoOptOutJob.set(message_deduplication_id: 'explicit-id').perform_later('same-args')
66
+
67
+ assert_equal('explicit-id', captured[:message_deduplication_id])
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests that auto_visibility_timeout still processes messages when the
4
+ # queue's visibility timeout is short (<= EXTEND_UPFRONT_SECONDS).
5
+ #
6
+ # AutoExtendVisibility schedules a TimerTask at execution_interval =
7
+ # visibility_timeout - EXTEND_UPFRONT_SECONDS (5s). When the queue's visibility
8
+ # timeout is <= 5s that interval is <= 0, and TimerTask#initialize raises
9
+ # ArgumentError *before* the worker runs - so every receive fails and the
10
+ # message is reprocessed (and re-fails) until it hits a DLQ.
11
+ #
12
+ # Expected behavior: a short visibility timeout no longer breaks processing; the
13
+ # extension interval is clamped to a positive value and the worker runs normally.
14
+ #
15
+ # Regression: auto_visibility_timeout + a <= 5s queue visibility timeout broke
16
+ # processing entirely.
17
+
18
+ require 'timeout'
19
+
20
+ setup_sqs
21
+
22
+ DT.clear
23
+
24
+ queue_name = DT.queue
25
+ # 5s visibility timeout -> interval would be 5 - 5 = 0 -> TimerTask raised pre-fix.
26
+ create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '5' })
27
+ Shoryuken.add_group('default', 1)
28
+ Shoryuken.add_queue(queue_name, 1, 'default')
29
+
30
+ short_vt_worker = Class.new do
31
+ include Shoryuken::Worker
32
+
33
+ shoryuken_options auto_delete: true, auto_visibility_timeout: true
34
+
35
+ def perform(_sqs_msg, body)
36
+ DT[:processed] << body
37
+ end
38
+ end
39
+
40
+ short_vt_worker.get_shoryuken_options['queue'] = queue_name
41
+ Shoryuken.register_worker(queue_name, short_vt_worker)
42
+
43
+ queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
44
+ Shoryuken::Client.sqs.send_message(queue_url: queue_url, message_body: 'hello')
45
+
46
+ poll_queues_until(timeout: 20) { DT[:processed].size >= 1 }
47
+
48
+ assert_equal(
49
+ ['hello'],
50
+ DT[:processed].to_a,
51
+ 'auto_visibility_timeout must not break processing on a queue with a short visibility timeout'
52
+ )
@@ -0,0 +1,120 @@
1
+ # frozen_string_literal: true
2
+
3
+ # `shoryuken sqs dump`/`mv` drain a queue via CLI::SQS#find_all. With short
4
+ # polling, real (distributed) SQS routinely returns an empty batch while the
5
+ # queue still has messages, so find_all must use long polling and must not stop
6
+ # on the first empty response - otherwise dump/mv silently process only a
7
+ # fraction of the queue.
8
+ #
9
+ # ElasticMQ never produces those false-empties (nor visibility-timeout re-reads),
10
+ # so this drives find_all with scripted clients to verify the behavior
11
+ # deterministically.
12
+ #
13
+ # Lives under spec/integration (not spec/lib) because requiring bin/cli defines
14
+ # Shoryuken::CLI, and Shoryuken#server? is `defined?(Shoryuken::CLI)` - loading
15
+ # it in the unit suite would flip server? for every other spec. Integration
16
+ # specs each run in their own process, so that's contained here.
17
+
18
+ require 'thor'
19
+ require_relative '../../../bin/cli/base'
20
+ require_relative '../../../bin/cli/sqs'
21
+
22
+ Msg = Struct.new(:message_id, :receipt_handle)
23
+
24
+ # A scripted SQS client: hands out pre-canned batches in order (to simulate
25
+ # false-empties and visibility-timeout re-reads), ignoring max_number_of_messages
26
+ # and recording the wait_time_seconds it was asked for.
27
+ class ScriptedSqsClient
28
+ attr_reader :wait_times
29
+
30
+ def initialize(batches)
31
+ @batches = batches.dup
32
+ @wait_times = []
33
+ end
34
+
35
+ def receive_message(params)
36
+ @wait_times << params[:wait_time_seconds]
37
+ Struct.new(:messages).new(@batches.shift || [])
38
+ end
39
+ end
40
+
41
+ # A more realistic client backed by a flat list that honours
42
+ # max_number_of_messages, so the finite-limit / batch-sizing path can be driven.
43
+ class FlatSqsClient
44
+ def initialize(messages)
45
+ @messages = messages.dup
46
+ end
47
+
48
+ def receive_message(params)
49
+ Struct.new(:messages).new(@messages.shift(params[:max_number_of_messages]))
50
+ end
51
+ end
52
+
53
+ def find_all_with(client, limit)
54
+ cli = Shoryuken::CLI::SQS.allocate
55
+ cli.instance_variable_set(:@_sqs, client)
56
+
57
+ collected = []
58
+ count = cli.send(:find_all, 'http://example.com/q', limit) { |msg| collected << msg }
59
+ [collected, count]
60
+ end
61
+
62
+ # --- Drains past a false-empty batch, using long polling -------------------
63
+ client = ScriptedSqsClient.new(
64
+ [
65
+ [Msg.new('a'), Msg.new('b'), Msg.new('c')],
66
+ [Msg.new('d'), Msg.new('e')],
67
+ [], # false-empty while 'f' is still queued
68
+ [Msg.new('f')]
69
+ ]
70
+ )
71
+ collected, = find_all_with(client, Float::INFINITY)
72
+
73
+ assert_equal(%w[a b c d e f], collected.map(&:message_id), 'find_all should drain past a false-empty batch')
74
+ assert(
75
+ client.wait_times.any? && client.wait_times.all? { |w| w && w.positive? },
76
+ "find_all should use long polling, saw wait_time_seconds: #{client.wait_times.uniq.inspect}"
77
+ )
78
+
79
+ # --- Does not re-yield a message that reappears after its visibility timeout ---
80
+ # 'a' is handed back on a later receive (its visibility timeout lapsed before
81
+ # dump/mv deleted it). find_all must yield and count it only once.
82
+ reappearing = ScriptedSqsClient.new(
83
+ [
84
+ [Msg.new('a'), Msg.new('b')],
85
+ [Msg.new('a')], # re-read of 'a' while it is still in the queue
86
+ [Msg.new('c')]
87
+ ]
88
+ )
89
+ collected, count = find_all_with(reappearing, Float::INFINITY)
90
+
91
+ assert_equal(%w[a b c], collected.map(&:message_id), 'find_all must not re-yield a re-read message')
92
+ assert_equal(3, count, 'find_all count must not double-count a re-read message')
93
+
94
+ # --- Retains the newest receipt handle for a re-read message ----------------
95
+ # SQS only honors the most recently received receipt handle for deletion, and
96
+ # dump/mv batch_delete after the drain, so the yielded (and later deleted)
97
+ # message must carry the handle from the latest receive, not the first.
98
+ handles = ScriptedSqsClient.new(
99
+ [
100
+ [Msg.new('x', 'rh-x-1'), Msg.new('y', 'rh-y')],
101
+ [Msg.new('x', 'rh-x-2')], # re-read of 'x' with a fresh handle
102
+ [Msg.new('z', 'rh-z')]
103
+ ]
104
+ )
105
+ collected, = find_all_with(handles, Float::INFINITY)
106
+
107
+ x = collected.find { |msg| msg.message_id == 'x' }
108
+ assert_equal('rh-x-2', x.receipt_handle, 'find_all must retain the newest receipt handle for a re-read message')
109
+ assert_equal(%w[x y z], collected.map(&:message_id), 'a re-read must not be yielded twice')
110
+
111
+ # --- Stops at a finite limit without over-fetching -------------------------
112
+ flat = FlatSqsClient.new((1..25).map { |i| Msg.new("m#{i}") })
113
+ collected, count = find_all_with(flat, 15)
114
+
115
+ assert_equal(15, count, 'find_all must stop once the limit is reached')
116
+ assert_equal(
117
+ (1..15).map { |i| "m#{i}" },
118
+ collected.map(&:message_id),
119
+ 'find_all must yield exactly the first `limit` messages'
120
+ )