shoryuken 7.0.0.alpha2 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/push.yml +2 -2
  3. data/.github/workflows/specs.yml +38 -43
  4. data/.github/workflows/verify-action-pins.yml +1 -1
  5. data/.gitignore +3 -0
  6. data/.rspec +1 -0
  7. data/.ruby-version +1 -1
  8. data/.yard-lint.yml +279 -0
  9. data/CHANGELOG.md +69 -1
  10. data/Gemfile +1 -1
  11. data/README.md +2 -7
  12. data/Rakefile +4 -10
  13. data/bin/clean_localstack +52 -0
  14. data/bin/cli/base.rb +21 -0
  15. data/bin/cli/sqs.rb +61 -2
  16. data/bin/integrations +275 -0
  17. data/bin/scenario +154 -0
  18. data/bin/shoryuken +1 -1
  19. data/lib/{shoryuken/extensions/active_job_extensions.rb → active_job/extensions.rb} +15 -4
  20. data/lib/active_job/queue_adapters/shoryuken_adapter.rb +208 -0
  21. data/lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter.rb +78 -0
  22. data/lib/shoryuken/active_job/current_attributes.rb +139 -0
  23. data/lib/shoryuken/active_job/job_wrapper.rb +28 -0
  24. data/lib/shoryuken/body_parser.rb +8 -0
  25. data/lib/shoryuken/client.rb +14 -0
  26. data/lib/shoryuken/default_exception_handler.rb +9 -0
  27. data/lib/shoryuken/default_worker_registry.rb +29 -1
  28. data/lib/shoryuken/environment_loader.rb +78 -8
  29. data/lib/shoryuken/errors.rb +33 -0
  30. data/lib/shoryuken/fetcher.rb +37 -1
  31. data/lib/shoryuken/helpers/atomic_boolean.rb +19 -5
  32. data/lib/shoryuken/helpers/timer_task.rb +80 -0
  33. data/lib/shoryuken/launcher.rb +53 -0
  34. data/lib/shoryuken/logging/base.rb +26 -0
  35. data/lib/shoryuken/logging/pretty.rb +25 -0
  36. data/lib/shoryuken/logging/without_timestamp.rb +25 -0
  37. data/lib/shoryuken/logging.rb +39 -25
  38. data/lib/shoryuken/manager.rb +70 -1
  39. data/lib/shoryuken/message.rb +114 -1
  40. data/lib/shoryuken/middleware/chain.rb +139 -43
  41. data/lib/shoryuken/middleware/entry.rb +30 -0
  42. data/lib/shoryuken/middleware/server/active_record.rb +8 -0
  43. data/lib/shoryuken/middleware/server/auto_delete.rb +10 -0
  44. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +27 -1
  45. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +29 -0
  46. data/lib/shoryuken/middleware/server/timing.rb +11 -0
  47. data/lib/shoryuken/options.rb +129 -6
  48. data/lib/shoryuken/polling/base_strategy.rb +1 -0
  49. data/lib/shoryuken/polling/strict_priority.rb +39 -0
  50. data/lib/shoryuken/polling/weighted_round_robin.rb +42 -0
  51. data/lib/shoryuken/processor.rb +32 -1
  52. data/lib/shoryuken/queue.rb +93 -4
  53. data/lib/shoryuken/runner.rb +45 -4
  54. data/lib/shoryuken/util.rb +26 -1
  55. data/lib/shoryuken/version.rb +2 -1
  56. data/lib/shoryuken/worker/default_executor.rb +21 -1
  57. data/lib/shoryuken/worker/inline_executor.rb +24 -0
  58. data/lib/shoryuken/worker.rb +193 -0
  59. data/lib/shoryuken/worker_registry.rb +33 -0
  60. data/lib/shoryuken.rb +18 -6
  61. data/renovate.json +29 -2
  62. data/shoryuken.gemspec +2 -1
  63. data/spec/integration/.rspec +1 -0
  64. data/spec/integration/active_job/adapter_configuration/configuration_spec.rb +26 -0
  65. data/spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb +53 -0
  66. data/spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb +50 -0
  67. data/spec/integration/active_job/current_attributes/complex_types_spec.rb +55 -0
  68. data/spec/integration/active_job/current_attributes/empty_context_spec.rb +41 -0
  69. data/spec/integration/active_job/current_attributes/full_context_spec.rb +63 -0
  70. data/spec/integration/active_job/current_attributes/partial_context_spec.rb +57 -0
  71. data/spec/integration/active_job/custom_attributes/number_attributes_spec.rb +37 -0
  72. data/spec/integration/active_job/custom_attributes/string_attributes_spec.rb +39 -0
  73. data/spec/integration/active_job/error_handling/job_wrapper_spec.rb +53 -0
  74. data/spec/integration/active_job/fifo_and_attributes/deduplication_spec.rb +86 -0
  75. data/spec/integration/active_job/retry/discard_on_spec.rb +43 -0
  76. data/spec/integration/active_job/retry/retry_on_spec.rb +36 -0
  77. data/spec/integration/active_job/roundtrip/roundtrip_spec.rb +52 -0
  78. data/spec/integration/active_job/scheduled/scheduled_spec.rb +76 -0
  79. data/spec/integration/active_record_middleware/active_record_middleware_spec.rb +84 -0
  80. data/spec/integration/auto_delete/auto_delete_spec.rb +53 -0
  81. data/spec/integration/auto_extend_visibility/auto_extend_visibility_spec.rb +57 -0
  82. data/spec/integration/aws_config/aws_config_spec.rb +59 -0
  83. data/spec/integration/batch_processing/batch_processing_spec.rb +37 -0
  84. data/spec/integration/body_parser/json_parser_spec.rb +45 -0
  85. data/spec/integration/body_parser/proc_parser_spec.rb +54 -0
  86. data/spec/integration/body_parser/text_parser_spec.rb +43 -0
  87. data/spec/integration/concurrent_processing/concurrent_processing_spec.rb +45 -0
  88. data/spec/integration/dead_letter_queue/dead_letter_queue_spec.rb +91 -0
  89. data/spec/integration/exception_handlers/exception_handlers_spec.rb +69 -0
  90. data/spec/integration/exponential_backoff/exponential_backoff_spec.rb +67 -0
  91. data/spec/integration/fifo_ordering/fifo_ordering_spec.rb +44 -0
  92. data/spec/integration/large_payloads/large_payloads_spec.rb +30 -0
  93. data/spec/integration/launcher/launcher_spec.rb +40 -0
  94. data/spec/integration/message_attributes/message_attributes_spec.rb +54 -0
  95. data/spec/integration/message_operations/message_operations_spec.rb +59 -0
  96. data/spec/integration/middleware_chain/empty_chain_spec.rb +11 -0
  97. data/spec/integration/middleware_chain/execution_order_spec.rb +33 -0
  98. data/spec/integration/middleware_chain/removal_spec.rb +31 -0
  99. data/spec/integration/middleware_chain/short_circuit_spec.rb +40 -0
  100. data/spec/integration/polling_strategies/polling_strategies_spec.rb +46 -0
  101. data/spec/integration/queue_operations/queue_operations_spec.rb +84 -0
  102. data/spec/integration/rails/rails_72/Gemfile +6 -0
  103. data/spec/integration/rails/rails_72/activejob_adapter_spec.rb +98 -0
  104. data/spec/integration/rails/rails_80/Gemfile +6 -0
  105. data/spec/integration/rails/rails_80/activejob_adapter_spec.rb +98 -0
  106. data/spec/integration/rails/rails_80/continuation_spec.rb +79 -0
  107. data/spec/integration/rails/rails_81/Gemfile +6 -0
  108. data/spec/integration/rails/rails_81/activejob_adapter_spec.rb +98 -0
  109. data/spec/integration/rails/rails_81/continuation_spec.rb +79 -0
  110. data/spec/integration/retry_behavior/retry_behavior_spec.rb +45 -0
  111. data/spec/integration/spec_helper.rb +7 -0
  112. data/spec/integration/strict_priority_polling/strict_priority_polling_spec.rb +58 -0
  113. data/spec/integration/visibility_timeout/visibility_timeout_spec.rb +37 -0
  114. data/spec/integration/worker_enqueueing/worker_enqueueing_spec.rb +60 -0
  115. data/spec/integration/worker_groups/worker_groups_spec.rb +79 -0
  116. data/spec/integration/worker_lifecycle/worker_lifecycle_spec.rb +33 -0
  117. data/spec/integrations_helper.rb +243 -0
  118. data/spec/lib/active_job/extensions_spec.rb +149 -0
  119. data/spec/lib/active_job/queue_adapters/shoryuken_adapter_spec.rb +29 -0
  120. data/spec/{shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb → lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter_spec.rb} +3 -3
  121. data/spec/{shoryuken/extensions/active_job_wrapper_spec.rb → lib/shoryuken/active_job/job_wrapper_spec.rb} +4 -4
  122. data/spec/{shoryuken → lib/shoryuken}/environment_loader_spec.rb +1 -1
  123. data/spec/{shoryuken → lib/shoryuken}/helpers/hash_utils_spec.rb +14 -14
  124. data/spec/{shoryuken → lib/shoryuken}/helpers/string_utils_spec.rb +3 -3
  125. data/spec/lib/shoryuken/helpers/timer_task_spec.rb +298 -0
  126. data/spec/{shoryuken → lib/shoryuken}/helpers_integration_spec.rb +9 -9
  127. data/spec/{shoryuken → lib/shoryuken}/launcher_spec.rb +22 -0
  128. data/spec/lib/shoryuken/logging_spec.rb +242 -0
  129. data/spec/lib/shoryuken/message_spec.rb +109 -0
  130. data/spec/lib/shoryuken/middleware/entry_spec.rb +68 -0
  131. data/spec/lib/shoryuken/middleware/server/active_record_spec.rb +133 -0
  132. data/spec/{shoryuken → lib/shoryuken}/middleware/server/auto_extend_visibility_spec.rb +50 -0
  133. data/spec/{shoryuken → lib/shoryuken}/options_spec.rb +2 -2
  134. data/spec/{shoryuken → lib/shoryuken}/util_spec.rb +1 -1
  135. data/spec/lib/shoryuken/version_spec.rb +17 -0
  136. data/spec/lib/shoryuken/worker_registry_spec.rb +63 -0
  137. data/spec/shared_examples_for_active_job.rb +29 -9
  138. data/spec/spec_helper.rb +34 -3
  139. metadata +230 -91
  140. data/.devcontainer/Dockerfile +0 -17
  141. data/.devcontainer/base.Dockerfile +0 -43
  142. data/.devcontainer/devcontainer.json +0 -35
  143. data/Appraisals +0 -23
  144. data/gemfiles/.gitignore +0 -1
  145. data/gemfiles/rails_7_0.gemfile +0 -19
  146. data/gemfiles/rails_7_1.gemfile +0 -19
  147. data/gemfiles/rails_7_2.gemfile +0 -19
  148. data/gemfiles/rails_8_0.gemfile +0 -19
  149. data/lib/shoryuken/extensions/active_job_adapter.rb +0 -110
  150. data/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb +0 -50
  151. data/spec/integration/launcher_spec.rb +0 -127
  152. data/spec/shoryuken/extensions/active_job_adapter_spec.rb +0 -8
  153. data/spec/shoryuken/extensions/active_job_base_spec.rb +0 -85
  154. /data/spec/{shoryuken → lib/shoryuken}/body_parser_spec.rb +0 -0
  155. /data/spec/{shoryuken → lib/shoryuken}/client_spec.rb +0 -0
  156. /data/spec/{shoryuken → lib/shoryuken}/default_exception_handler_spec.rb +0 -0
  157. /data/spec/{shoryuken → lib/shoryuken}/default_worker_registry_spec.rb +0 -0
  158. /data/spec/{shoryuken → lib/shoryuken}/fetcher_spec.rb +0 -0
  159. /data/spec/{shoryuken → lib/shoryuken}/helpers/atomic_boolean_spec.rb +0 -0
  160. /data/spec/{shoryuken → lib/shoryuken}/helpers/atomic_counter_spec.rb +0 -0
  161. /data/spec/{shoryuken → lib/shoryuken}/helpers/atomic_hash_spec.rb +0 -0
  162. /data/spec/{shoryuken → lib/shoryuken}/inline_message_spec.rb +0 -0
  163. /data/spec/{shoryuken → lib/shoryuken}/manager_spec.rb +0 -0
  164. /data/spec/{shoryuken → lib/shoryuken}/middleware/chain_spec.rb +0 -0
  165. /data/spec/{shoryuken → lib/shoryuken}/middleware/server/auto_delete_spec.rb +0 -0
  166. /data/spec/{shoryuken → lib/shoryuken}/middleware/server/exponential_backoff_retry_spec.rb +0 -0
  167. /data/spec/{shoryuken → lib/shoryuken}/middleware/server/timing_spec.rb +0 -0
  168. /data/spec/{shoryuken → lib/shoryuken}/polling/base_strategy_spec.rb +0 -0
  169. /data/spec/{shoryuken → lib/shoryuken}/polling/queue_configuration_spec.rb +0 -0
  170. /data/spec/{shoryuken → lib/shoryuken}/polling/strict_priority_spec.rb +0 -0
  171. /data/spec/{shoryuken → lib/shoryuken}/polling/weighted_round_robin_spec.rb +0 -0
  172. /data/spec/{shoryuken → lib/shoryuken}/processor_spec.rb +0 -0
  173. /data/spec/{shoryuken → lib/shoryuken}/queue_spec.rb +0 -0
  174. /data/spec/{shoryuken → lib/shoryuken}/runner_spec.rb +0 -0
  175. /data/spec/{shoryuken → lib/shoryuken}/worker/default_executor_spec.rb +0 -0
  176. /data/spec/{shoryuken → lib/shoryuken}/worker/inline_executor_spec.rb +0 -0
  177. /data/spec/{shoryuken → lib/shoryuken}/worker_spec.rb +0 -0
  178. /data/spec/{shoryuken_spec.rb → lib/shoryuken_spec.rb} +0 -0
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests the auto_visibility_timeout middleware functionality.
4
+ # When auto_visibility_timeout: true, the message visibility timeout should be
5
+ # automatically extended during long-running job processing to prevent re-delivery.
6
+
7
+ setup_localstack
8
+
9
+ queue_name = DT.uuid
10
+
11
+ # Create queue with short visibility timeout (10 seconds)
12
+ create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '10' })
13
+ Shoryuken.add_group('default', 1)
14
+ Shoryuken.add_queue(queue_name, 1, 'default')
15
+
16
+ # Worker with auto_visibility_timeout enabled that takes longer than visibility timeout
17
+ auto_visibility_worker = Class.new do
18
+ include Shoryuken::Worker
19
+
20
+ shoryuken_options auto_visibility_timeout: true, auto_delete: true
21
+
22
+ def perform(sqs_msg, body)
23
+ DT[:processing_started] << Time.now
24
+ # Sleep longer than the queue's visibility timeout (10s)
25
+ # The middleware should extend visibility before it expires
26
+ sleep 12
27
+ DT[:processing_completed] << Time.now
28
+ DT[:processed_messages] << { message_id: sqs_msg.message_id, body: body }
29
+ end
30
+ end
31
+
32
+ auto_visibility_worker.get_shoryuken_options['queue'] = queue_name
33
+ Shoryuken.register_worker(queue_name, auto_visibility_worker)
34
+
35
+ queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
36
+
37
+ # Send a message
38
+ Shoryuken::Client.sqs.send_message(
39
+ queue_url: queue_url,
40
+ message_body: 'long running job'
41
+ )
42
+
43
+ sleep 1
44
+
45
+ # Process the message - this should take ~12 seconds but not fail
46
+ poll_queues_until(timeout: 30) { DT[:processed_messages].size >= 1 }
47
+
48
+ # Verify message was processed exactly once (visibility was extended, not re-delivered)
49
+ assert_equal(1, DT[:processed_messages].size, "Message should be processed exactly once")
50
+ assert_equal('long running job', DT[:processed_messages].first[:body])
51
+
52
+ # Verify processing took longer than the visibility timeout
53
+ processing_time = DT[:processing_completed].first - DT[:processing_started].first
54
+ assert(processing_time >= 12, "Processing should have taken at least 12 seconds")
55
+
56
+ # Cleanup
57
+ delete_test_queue(queue_name)
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests that AWS configuration from Shoryuken.options[:aws]
4
+ # is properly passed to the SQS client initialization.
5
+ # This verifies the fix for issue #815: PORO setup does not load AWS config
6
+
7
+ # Reset any cached SQS client to ensure fresh initialization
8
+ Shoryuken.sqs_client = nil
9
+
10
+ # Configure AWS options programmatically (simulating PORO setup with config file)
11
+ Shoryuken.options[:aws] = {
12
+ region: 'us-east-1',
13
+ endpoint: 'http://localhost:4566',
14
+ access_key_id: 'test-key-from-config',
15
+ secret_access_key: 'test-secret-from-config'
16
+ }
17
+
18
+ # Get the SQS client - this should use the AWS config from options
19
+ client = Shoryuken.sqs_client
20
+
21
+ # Verify the client was configured with our options
22
+ config = client.config
23
+
24
+ assert_equal('us-east-1', config.region, "Region should be set from options[:aws]")
25
+ assert_equal('http://localhost:4566', config.endpoint.to_s, "Endpoint should be set from options[:aws]")
26
+
27
+ # Verify the client actually works by creating a queue
28
+ queue_name = "aws-config-test-#{SecureRandom.hex(6)}"
29
+
30
+ begin
31
+ result = client.create_queue(queue_name: queue_name)
32
+ assert(result.queue_url.include?(queue_name), "Should be able to create queue with configured client")
33
+
34
+ # Clean up
35
+ client.delete_queue(queue_url: result.queue_url)
36
+ rescue Aws::SQS::Errors::ServiceError => e
37
+ raise "SQS client should work with configured AWS options: #{e.message}"
38
+ end
39
+
40
+ # Test 2: Verify that reconfiguring options and resetting client works
41
+ Shoryuken.sqs_client = nil
42
+ Shoryuken.options[:aws][:region] = 'us-west-2'
43
+
44
+ client2 = Shoryuken.sqs_client
45
+ assert_equal('us-west-2', client2.config.region, "New client should use updated region")
46
+
47
+ # Test 3: Verify that credentials from options are used
48
+ # Reset and reconfigure with explicit credentials
49
+ Shoryuken.sqs_client = nil
50
+ Shoryuken.options[:aws] = {
51
+ region: 'us-east-1',
52
+ endpoint: 'http://localhost:4566',
53
+ access_key_id: 'another-key',
54
+ secret_access_key: 'another-secret'
55
+ }
56
+
57
+ client3 = Shoryuken.sqs_client
58
+ assert(client3.is_a?(Aws::SQS::Client), "Client should be created with new credentials")
59
+ assert_equal('us-east-1', client3.config.region, "Region should match configured value")
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests batch processing including batch message reception (up to 10
4
+ # messages), batch vs single worker behavior differences, JSON body parsing in
5
+ # batch mode, and maximum batch size handling.
6
+
7
+ setup_localstack
8
+
9
+ queue_name = DT.queue
10
+ create_test_queue(queue_name)
11
+ Shoryuken.add_group('default', 1)
12
+ Shoryuken.add_queue(queue_name, 1, 'default')
13
+
14
+ worker_class = Class.new do
15
+ include Shoryuken::Worker
16
+
17
+ def perform(sqs_msgs, bodies)
18
+ msgs = Array(sqs_msgs)
19
+ DT[:batch_sizes] << msgs.size
20
+ DT[:messages].concat(Array(bodies))
21
+ end
22
+ end
23
+
24
+ worker_class.get_shoryuken_options['queue'] = queue_name
25
+ worker_class.get_shoryuken_options['auto_delete'] = true
26
+ worker_class.get_shoryuken_options['batch'] = true
27
+ Shoryuken.register_worker(queue_name, worker_class)
28
+
29
+ entries = 5.times.map { |i| { id: SecureRandom.uuid, message_body: "message-#{i}" } }
30
+ Shoryuken::Client.queues(queue_name).send_messages(entries: entries)
31
+
32
+ sleep 1
33
+
34
+ poll_queues_until { DT[:messages].size >= 5 }
35
+
36
+ assert_equal(5, DT[:messages].size)
37
+ assert(DT[:batch_sizes].any? { |size| size > 1 }, "Expected at least one batch with size > 1")
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests the body_parser option with :json setting
4
+ # Verifies that JSON messages are automatically parsed into Ruby hashes
5
+
6
+ setup_localstack
7
+
8
+ queue_name = DT.uuid
9
+ create_test_queue(queue_name)
10
+ Shoryuken.add_group('default', 1)
11
+ Shoryuken.add_queue(queue_name, 1, 'default')
12
+
13
+ worker_class = Class.new do
14
+ include Shoryuken::Worker
15
+
16
+ shoryuken_options body_parser: :json
17
+
18
+ def perform(sqs_msg, body)
19
+ DT[:parsed_bodies] << body
20
+ DT[:body_classes] << body.class.name
21
+ end
22
+ end
23
+
24
+ worker_class.get_shoryuken_options['queue'] = queue_name
25
+ worker_class.get_shoryuken_options['auto_delete'] = true
26
+ Shoryuken.register_worker(queue_name, worker_class)
27
+
28
+ queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
29
+
30
+ # Send a JSON message
31
+ json_body = { 'name' => 'test', 'value' => 42, 'nested' => { 'key' => 'val' } }
32
+ Shoryuken::Client.sqs.send_message(
33
+ queue_url: queue_url,
34
+ message_body: JSON.dump(json_body)
35
+ )
36
+
37
+ sleep 1
38
+
39
+ poll_queues_until { DT[:parsed_bodies].size >= 1 }
40
+
41
+ assert_equal(1, DT[:parsed_bodies].size)
42
+ assert_equal('Hash', DT[:body_classes].first)
43
+ assert_equal('test', DT[:parsed_bodies].first['name'])
44
+ assert_equal(42, DT[:parsed_bodies].first['value'])
45
+ assert_equal('val', DT[:parsed_bodies].first['nested']['key'])
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests the body_parser option with a custom Proc
4
+ # Verifies that custom parsing logic can be applied to messages
5
+
6
+ setup_localstack
7
+
8
+ queue_name = DT.uuid
9
+ create_test_queue(queue_name)
10
+ Shoryuken.add_group('default', 1)
11
+ Shoryuken.add_queue(queue_name, 1, 'default')
12
+
13
+ # Custom parser that uppercases the body and adds metadata
14
+ custom_parser = proc do |sqs_msg|
15
+ {
16
+ original: sqs_msg.body,
17
+ transformed: sqs_msg.body.upcase,
18
+ message_id: sqs_msg.message_id
19
+ }
20
+ end
21
+
22
+ worker_class = Class.new do
23
+ include Shoryuken::Worker
24
+
25
+ def perform(sqs_msg, body)
26
+ DT[:parsed_bodies] << body
27
+ DT[:body_classes] << body.class.name
28
+ end
29
+ end
30
+
31
+ worker_class.get_shoryuken_options['queue'] = queue_name
32
+ worker_class.get_shoryuken_options['auto_delete'] = true
33
+ worker_class.get_shoryuken_options['body_parser'] = custom_parser
34
+ Shoryuken.register_worker(queue_name, worker_class)
35
+
36
+ queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
37
+
38
+ # Send a message to be processed by custom parser
39
+ Shoryuken::Client.sqs.send_message(
40
+ queue_url: queue_url,
41
+ message_body: 'hello world'
42
+ )
43
+
44
+ sleep 1
45
+
46
+ poll_queues_until { DT[:parsed_bodies].size >= 1 }
47
+
48
+ assert_equal(1, DT[:parsed_bodies].size)
49
+ assert_equal('Hash', DT[:body_classes].first)
50
+
51
+ parsed = DT[:parsed_bodies].first
52
+ assert_equal('hello world', parsed[:original])
53
+ assert_equal('HELLO WORLD', parsed[:transformed])
54
+ assert(parsed[:message_id], "Should include message_id from sqs_msg")
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests the body_parser option with :text setting (default)
4
+ # Verifies that messages are returned as plain strings
5
+
6
+ setup_localstack
7
+
8
+ queue_name = DT.uuid
9
+ create_test_queue(queue_name)
10
+ Shoryuken.add_group('default', 1)
11
+ Shoryuken.add_queue(queue_name, 1, 'default')
12
+
13
+ worker_class = Class.new do
14
+ include Shoryuken::Worker
15
+
16
+ shoryuken_options body_parser: :text
17
+
18
+ def perform(sqs_msg, body)
19
+ DT[:parsed_bodies] << body
20
+ DT[:body_classes] << body.class.name
21
+ end
22
+ end
23
+
24
+ worker_class.get_shoryuken_options['queue'] = queue_name
25
+ worker_class.get_shoryuken_options['auto_delete'] = true
26
+ Shoryuken.register_worker(queue_name, worker_class)
27
+
28
+ queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
29
+
30
+ # Send a plain text message
31
+ text_body = 'Hello, this is a plain text message!'
32
+ Shoryuken::Client.sqs.send_message(
33
+ queue_url: queue_url,
34
+ message_body: text_body
35
+ )
36
+
37
+ sleep 1
38
+
39
+ poll_queues_until { DT[:parsed_bodies].size >= 1 }
40
+
41
+ assert_equal(1, DT[:parsed_bodies].size)
42
+ assert_equal('String', DT[:body_classes].first)
43
+ assert_equal(text_body, DT[:parsed_bodies].first)
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests concurrent message processing with multiple processors.
4
+
5
+ require 'concurrent'
6
+
7
+ setup_localstack
8
+
9
+ queue_name = DT.queue
10
+ create_test_queue(queue_name)
11
+ Shoryuken.add_group('concurrent', 5) # 5 concurrent processors
12
+ Shoryuken.add_queue(queue_name, 1, 'concurrent')
13
+
14
+ # Atomic counters for tracking concurrency
15
+ concurrent_count = Concurrent::AtomicFixnum.new(0)
16
+ max_concurrent = Concurrent::AtomicFixnum.new(0)
17
+
18
+ worker_class = Class.new do
19
+ include Shoryuken::Worker
20
+
21
+ shoryuken_options auto_delete: true, batch: false
22
+
23
+ define_method(:perform) do |sqs_msg, body|
24
+ concurrent_count.increment
25
+ current = concurrent_count.value
26
+ max_concurrent.update { |max| [max, current].max }
27
+
28
+ sleep 0.5 # Simulate work
29
+
30
+ DT[:processing_times] << Time.now
31
+
32
+ concurrent_count.decrement
33
+ end
34
+ end
35
+
36
+ worker_class.get_shoryuken_options['queue'] = queue_name
37
+ Shoryuken.register_worker(queue_name, worker_class)
38
+
39
+ 10.times { |i| Shoryuken::Client.queues(queue_name).send_message(message_body: "msg-#{i}") }
40
+
41
+ poll_queues_until(timeout: 20) { DT[:processing_times].size >= 10 }
42
+
43
+ assert_equal(10, DT[:processing_times].size)
44
+ # With multiple processors, we should see concurrency > 1
45
+ assert(max_concurrent.value > 1, "Expected concurrency > 1, got #{max_concurrent.value}")
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests dead letter queue (DLQ) functionality.
4
+ # When a message exceeds maxReceiveCount, it should be moved to the DLQ.
5
+ # Note: This test doesn't use poll_queues_until because messages should fail
6
+ # and be moved to DLQ rather than being successfully processed.
7
+
8
+ setup_localstack
9
+
10
+ main_queue_name = DT.queues[0]
11
+ dlq_name = DT.queues[1]
12
+
13
+ # Create the dead letter queue first
14
+ create_test_queue(dlq_name)
15
+
16
+ dlq_url = Shoryuken::Client.sqs.get_queue_url(queue_name: dlq_name).queue_url
17
+ dlq_arn = Shoryuken::Client.sqs.get_queue_attributes(
18
+ queue_url: dlq_url,
19
+ attribute_names: ['QueueArn']
20
+ ).attributes['QueueArn']
21
+
22
+ # Create main queue with redrive policy - move to DLQ after 2 receives
23
+ redrive_policy = { maxReceiveCount: 2, deadLetterTargetArn: dlq_arn }.to_json
24
+ create_test_queue(main_queue_name, attributes: {
25
+ 'VisibilityTimeout' => '1',
26
+ 'RedrivePolicy' => redrive_policy
27
+ })
28
+
29
+ main_queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: main_queue_name).queue_url
30
+
31
+ # Send a message
32
+ Shoryuken::Client.sqs.send_message(
33
+ queue_url: main_queue_url,
34
+ message_body: 'dlq test message'
35
+ )
36
+
37
+ # Manually receive the message multiple times to trigger DLQ
38
+ # maxReceiveCount = 2, so after 2 receives without deletion, it goes to DLQ
39
+ 3.times do |i|
40
+ msgs = Shoryuken::Client.sqs.receive_message(
41
+ queue_url: main_queue_url,
42
+ max_number_of_messages: 1,
43
+ wait_time_seconds: 3,
44
+ attribute_names: ['ApproximateReceiveCount']
45
+ ).messages
46
+
47
+ if msgs.any?
48
+ receive_count = msgs.first.attributes['ApproximateReceiveCount'].to_i
49
+ DT[:receives] << { attempt: i + 1, receive_count: receive_count }
50
+ # Don't delete - let visibility timeout expire
51
+ sleep 2
52
+ else
53
+ DT[:receives] << { attempt: i + 1, no_message: true }
54
+ break
55
+ end
56
+ end
57
+
58
+ # Verify message was received at least twice
59
+ actual_receives = DT[:receives].reject { |r| r[:no_message] }
60
+ assert(actual_receives.size >= 2, "Message should have been received at least twice (was #{actual_receives.size})")
61
+
62
+ # Wait for message to be moved to DLQ
63
+ sleep 3
64
+
65
+ # Check that message is now in the DLQ
66
+ dlq_messages = Shoryuken::Client.sqs.receive_message(
67
+ queue_url: dlq_url,
68
+ max_number_of_messages: 10,
69
+ wait_time_seconds: 5,
70
+ attribute_names: ['All']
71
+ ).messages
72
+
73
+ assert(dlq_messages.size >= 1, 'Message should have been moved to DLQ')
74
+ assert_equal('dlq test message', dlq_messages.first.body)
75
+
76
+ # Verify message is no longer in main queue
77
+ main_attrs = Shoryuken::Client.sqs.get_queue_attributes(
78
+ queue_url: main_queue_url,
79
+ attribute_names: %w[ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible]
80
+ ).attributes
81
+ main_count = main_attrs['ApproximateNumberOfMessages'].to_i +
82
+ main_attrs['ApproximateNumberOfMessagesNotVisible'].to_i
83
+ assert_equal(0, main_count, 'Main queue should be empty after DLQ move')
84
+
85
+ # Clean up DLQ message
86
+ dlq_messages.each do |msg|
87
+ Shoryuken::Client.sqs.delete_message(
88
+ queue_url: dlq_url,
89
+ receipt_handle: msg.receipt_handle
90
+ )
91
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests custom exception handlers.
4
+ # Exception handlers are called when a worker raises an error.
5
+
6
+ setup_localstack
7
+
8
+ queue_name = DT.queue
9
+ create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' })
10
+ Shoryuken.add_group('default', 1)
11
+ Shoryuken.add_queue(queue_name, 1, 'default')
12
+
13
+ # Save original handlers to restore later
14
+ original_handlers = Shoryuken.exception_handlers.dup
15
+
16
+ # Custom exception handler that records exceptions
17
+ custom_handler = Object.new
18
+ custom_handler.define_singleton_method(:call) do |exception, queue, sqs_msg|
19
+ DT[:exceptions] << {
20
+ message: exception.message,
21
+ queue: queue,
22
+ message_id: sqs_msg.message_id
23
+ }
24
+ end
25
+
26
+ # Add custom handler alongside default
27
+ Shoryuken.exception_handlers << custom_handler
28
+
29
+ worker_class = Class.new do
30
+ include Shoryuken::Worker
31
+
32
+ shoryuken_options auto_delete: false, batch: false
33
+
34
+ def perform(sqs_msg, body)
35
+ receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
36
+ DT[:attempts] << receive_count
37
+
38
+ if receive_count < 2
39
+ raise 'Intentional failure for testing'
40
+ end
41
+
42
+ # Succeed on second attempt and delete
43
+ DT[:success] << body
44
+ sqs_msg.delete
45
+ end
46
+ end
47
+
48
+ worker_class.get_shoryuken_options['queue'] = queue_name
49
+ Shoryuken.register_worker(queue_name, worker_class)
50
+
51
+ Shoryuken::Client.queues(queue_name).send_message(message_body: 'exception test')
52
+
53
+ sleep 1
54
+
55
+ begin
56
+ poll_queues_until(timeout: 15) { DT[:success].size >= 1 }
57
+
58
+ # Verify exception handler was called on first attempt
59
+ assert(DT[:exceptions].size >= 1, 'Exception handler should have been called')
60
+ assert_equal('Intentional failure for testing', DT[:exceptions].first[:message])
61
+ assert_equal(queue_name, DT[:exceptions].first[:queue])
62
+
63
+ # Verify message was eventually processed successfully
64
+ assert_equal(1, DT[:success].size)
65
+ assert_equal('exception test', DT[:success].first)
66
+ ensure
67
+ # Restore original handlers to prevent cross-test interference
68
+ Shoryuken.exception_handlers.replace(original_handlers)
69
+ end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests the exponential_backoff_retry middleware functionality.
4
+ # When retry_intervals is configured, failed jobs should have their visibility
5
+ # timeout adjusted based on the retry attempt number.
6
+
7
+ setup_localstack
8
+
9
+ queue_name = DT.queue
10
+
11
+ # Create queue with short visibility timeout
12
+ create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' })
13
+ Shoryuken.add_group('default', 1)
14
+ Shoryuken.add_queue(queue_name, 1, 'default')
15
+
16
+ # Worker that fails on first attempts, succeeds after retry_intervals exhausted
17
+ backoff_worker = Class.new do
18
+ include Shoryuken::Worker
19
+
20
+ # Retry after 1 second, then 2 seconds
21
+ shoryuken_options retry_intervals: [1, 2], auto_delete: true
22
+
23
+ def perform(sqs_msg, body)
24
+ receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
25
+ DT[:attempts] << { receive_count: receive_count, time: Time.now }
26
+
27
+ # Fail on first 2 attempts, succeed on 3rd
28
+ if receive_count < 3
29
+ raise "Simulated failure on attempt #{receive_count}"
30
+ end
31
+
32
+ DT[:successful_processing] << { body: body, final_receive_count: receive_count }
33
+ end
34
+ end
35
+
36
+ backoff_worker.get_shoryuken_options['queue'] = queue_name
37
+ Shoryuken.register_worker(queue_name, backoff_worker)
38
+
39
+ queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
40
+
41
+ # Send a message
42
+ Shoryuken::Client.sqs.send_message(
43
+ queue_url: queue_url,
44
+ message_body: 'backoff test'
45
+ )
46
+
47
+ sleep 1
48
+
49
+ # Process - should fail twice then succeed on 3rd attempt
50
+ # Total time: ~1s (first retry) + ~2s (second retry) = ~3s minimum
51
+ poll_queues_until(timeout: 20) { DT[:successful_processing].size >= 1 }
52
+
53
+ # Verify the message was eventually processed successfully
54
+ assert_equal(1, DT[:successful_processing].size)
55
+ assert_equal('backoff test', DT[:successful_processing].first[:body])
56
+ assert_equal(3, DT[:successful_processing].first[:final_receive_count])
57
+
58
+ # Verify we had 3 attempts total
59
+ assert_equal(3, DT[:attempts].size, 'Should have 3 attempts total')
60
+
61
+ # Verify backoff timing - second attempt should be ~1s after first
62
+ first_to_second = DT[:attempts][1][:time] - DT[:attempts][0][:time]
63
+ assert(first_to_second >= 0.5, "Second attempt should be at least 0.5s after first (was #{first_to_second}s)")
64
+
65
+ # Verify backoff timing - third attempt should be ~2s after second
66
+ second_to_third = DT[:attempts][2][:time] - DT[:attempts][1][:time]
67
+ assert(second_to_third >= 1.0, "Third attempt should be at least 1s after second (was #{second_to_third}s)")
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests FIFO queue ordering guarantees including message ordering
4
+ # within the same message group.
5
+
6
+ setup_localstack
7
+
8
+ queue_name = "#{DT.uuid}.fifo"
9
+ create_fifo_queue(queue_name)
10
+ Shoryuken.add_group('default', 1)
11
+ Shoryuken.add_queue(queue_name, 1, 'default')
12
+
13
+ worker_class = Class.new do
14
+ include Shoryuken::Worker
15
+
16
+ def perform(sqs_msg, body)
17
+ DT[:messages] << body
18
+ end
19
+ end
20
+
21
+ worker_class.get_shoryuken_options['queue'] = queue_name
22
+ worker_class.get_shoryuken_options['auto_delete'] = true
23
+ worker_class.get_shoryuken_options['batch'] = false
24
+ Shoryuken.register_worker(queue_name, worker_class)
25
+
26
+ queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
27
+
28
+ 5.times do |i|
29
+ Shoryuken::Client.sqs.send_message(
30
+ queue_url: queue_url,
31
+ message_body: "msg-#{i}",
32
+ message_group_id: 'group-a',
33
+ message_deduplication_id: SecureRandom.uuid
34
+ )
35
+ end
36
+
37
+ sleep 1
38
+
39
+ poll_queues_until { DT[:messages].size >= 5 }
40
+
41
+ assert_equal(5, DT[:messages].size)
42
+
43
+ expected = (0..4).map { |i| "msg-#{i}" }
44
+ assert_equal(expected, DT[:messages])
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests large payload handling including payloads near the 256KB SQS limit.
4
+
5
+ setup_localstack
6
+
7
+ queue_name = DT.queue
8
+ create_test_queue(queue_name)
9
+ Shoryuken.add_group('default', 1)
10
+ Shoryuken.add_queue(queue_name, 1, 'default')
11
+
12
+ worker_class = Class.new do
13
+ include Shoryuken::Worker
14
+
15
+ def perform(sqs_msg, body)
16
+ DT[:bodies] << body
17
+ end
18
+ end
19
+
20
+ worker_class.get_shoryuken_options['queue'] = queue_name
21
+ worker_class.get_shoryuken_options['auto_delete'] = true
22
+ worker_class.get_shoryuken_options['batch'] = false
23
+ Shoryuken.register_worker(queue_name, worker_class)
24
+
25
+ payload = 'x' * (250 * 1024)
26
+ Shoryuken::Client.queues(queue_name).send_message(message_body: payload)
27
+
28
+ poll_queues_until { DT[:bodies].size >= 1 }
29
+
30
+ assert_equal(250 * 1024, DT[:bodies].first.size)
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests the Launcher's ability to consume messages from SQS queues,
4
+ # including single message consumption, batch consumption, and command workers.
5
+
6
+ require 'concurrent'
7
+
8
+ setup_localstack
9
+
10
+ # Use atomic counter for thread-safe message counting
11
+ message_counter = Concurrent::AtomicFixnum.new(0)
12
+
13
+ worker_class = Class.new do
14
+ include Shoryuken::Worker
15
+
16
+ shoryuken_options auto_delete: true
17
+
18
+ define_method(:perform) do |sqs_msg, _body|
19
+ message_counter.increment(Array(sqs_msg).size)
20
+ end
21
+ end
22
+
23
+ queue_name = DT.queue
24
+
25
+ create_test_queue(queue_name)
26
+ Shoryuken.add_group('default', 1)
27
+ Shoryuken.add_queue(queue_name, 1, 'default')
28
+ worker_class.get_shoryuken_options['queue'] = queue_name
29
+ worker_class.get_shoryuken_options['batch'] = true
30
+ Shoryuken.register_worker(queue_name, worker_class)
31
+
32
+ entries = 10.times.map { |i| { id: SecureRandom.uuid, message_body: i.to_s } }
33
+ Shoryuken::Client.queues(queue_name).send_messages(entries: entries)
34
+
35
+ # Give the messages a chance to hit the queue
36
+ sleep 2
37
+
38
+ poll_queues_until { message_counter.value > 0 }
39
+
40
+ assert(message_counter.value > 1, "Expected more than 1 message in batch, got #{message_counter.value}")