shoryuken 7.0.0.alpha2 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/push.yml +2 -2
  3. data/.github/workflows/specs.yml +38 -43
  4. data/.github/workflows/verify-action-pins.yml +1 -1
  5. data/.gitignore +3 -0
  6. data/.rspec +1 -0
  7. data/.ruby-version +1 -1
  8. data/.yard-lint.yml +279 -0
  9. data/CHANGELOG.md +69 -1
  10. data/Gemfile +1 -1
  11. data/README.md +2 -7
  12. data/Rakefile +4 -10
  13. data/bin/clean_localstack +52 -0
  14. data/bin/cli/base.rb +21 -0
  15. data/bin/cli/sqs.rb +61 -2
  16. data/bin/integrations +275 -0
  17. data/bin/scenario +154 -0
  18. data/bin/shoryuken +1 -1
  19. data/lib/{shoryuken/extensions/active_job_extensions.rb → active_job/extensions.rb} +15 -4
  20. data/lib/active_job/queue_adapters/shoryuken_adapter.rb +208 -0
  21. data/lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter.rb +78 -0
  22. data/lib/shoryuken/active_job/current_attributes.rb +139 -0
  23. data/lib/shoryuken/active_job/job_wrapper.rb +28 -0
  24. data/lib/shoryuken/body_parser.rb +8 -0
  25. data/lib/shoryuken/client.rb +14 -0
  26. data/lib/shoryuken/default_exception_handler.rb +9 -0
  27. data/lib/shoryuken/default_worker_registry.rb +29 -1
  28. data/lib/shoryuken/environment_loader.rb +78 -8
  29. data/lib/shoryuken/errors.rb +33 -0
  30. data/lib/shoryuken/fetcher.rb +37 -1
  31. data/lib/shoryuken/helpers/atomic_boolean.rb +19 -5
  32. data/lib/shoryuken/helpers/timer_task.rb +80 -0
  33. data/lib/shoryuken/launcher.rb +53 -0
  34. data/lib/shoryuken/logging/base.rb +26 -0
  35. data/lib/shoryuken/logging/pretty.rb +25 -0
  36. data/lib/shoryuken/logging/without_timestamp.rb +25 -0
  37. data/lib/shoryuken/logging.rb +39 -25
  38. data/lib/shoryuken/manager.rb +70 -1
  39. data/lib/shoryuken/message.rb +114 -1
  40. data/lib/shoryuken/middleware/chain.rb +139 -43
  41. data/lib/shoryuken/middleware/entry.rb +30 -0
  42. data/lib/shoryuken/middleware/server/active_record.rb +8 -0
  43. data/lib/shoryuken/middleware/server/auto_delete.rb +10 -0
  44. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +27 -1
  45. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +29 -0
  46. data/lib/shoryuken/middleware/server/timing.rb +11 -0
  47. data/lib/shoryuken/options.rb +129 -6
  48. data/lib/shoryuken/polling/base_strategy.rb +1 -0
  49. data/lib/shoryuken/polling/strict_priority.rb +39 -0
  50. data/lib/shoryuken/polling/weighted_round_robin.rb +42 -0
  51. data/lib/shoryuken/processor.rb +32 -1
  52. data/lib/shoryuken/queue.rb +93 -4
  53. data/lib/shoryuken/runner.rb +45 -4
  54. data/lib/shoryuken/util.rb +26 -1
  55. data/lib/shoryuken/version.rb +2 -1
  56. data/lib/shoryuken/worker/default_executor.rb +21 -1
  57. data/lib/shoryuken/worker/inline_executor.rb +24 -0
  58. data/lib/shoryuken/worker.rb +193 -0
  59. data/lib/shoryuken/worker_registry.rb +33 -0
  60. data/lib/shoryuken.rb +18 -6
  61. data/renovate.json +29 -2
  62. data/shoryuken.gemspec +2 -1
  63. data/spec/integration/.rspec +1 -0
  64. data/spec/integration/active_job/adapter_configuration/configuration_spec.rb +26 -0
  65. data/spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb +53 -0
  66. data/spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb +50 -0
  67. data/spec/integration/active_job/current_attributes/complex_types_spec.rb +55 -0
  68. data/spec/integration/active_job/current_attributes/empty_context_spec.rb +41 -0
  69. data/spec/integration/active_job/current_attributes/full_context_spec.rb +63 -0
  70. data/spec/integration/active_job/current_attributes/partial_context_spec.rb +57 -0
  71. data/spec/integration/active_job/custom_attributes/number_attributes_spec.rb +37 -0
  72. data/spec/integration/active_job/custom_attributes/string_attributes_spec.rb +39 -0
  73. data/spec/integration/active_job/error_handling/job_wrapper_spec.rb +53 -0
  74. data/spec/integration/active_job/fifo_and_attributes/deduplication_spec.rb +86 -0
  75. data/spec/integration/active_job/retry/discard_on_spec.rb +43 -0
  76. data/spec/integration/active_job/retry/retry_on_spec.rb +36 -0
  77. data/spec/integration/active_job/roundtrip/roundtrip_spec.rb +52 -0
  78. data/spec/integration/active_job/scheduled/scheduled_spec.rb +76 -0
  79. data/spec/integration/active_record_middleware/active_record_middleware_spec.rb +84 -0
  80. data/spec/integration/auto_delete/auto_delete_spec.rb +53 -0
  81. data/spec/integration/auto_extend_visibility/auto_extend_visibility_spec.rb +57 -0
  82. data/spec/integration/aws_config/aws_config_spec.rb +59 -0
  83. data/spec/integration/batch_processing/batch_processing_spec.rb +37 -0
  84. data/spec/integration/body_parser/json_parser_spec.rb +45 -0
  85. data/spec/integration/body_parser/proc_parser_spec.rb +54 -0
  86. data/spec/integration/body_parser/text_parser_spec.rb +43 -0
  87. data/spec/integration/concurrent_processing/concurrent_processing_spec.rb +45 -0
  88. data/spec/integration/dead_letter_queue/dead_letter_queue_spec.rb +91 -0
  89. data/spec/integration/exception_handlers/exception_handlers_spec.rb +69 -0
  90. data/spec/integration/exponential_backoff/exponential_backoff_spec.rb +67 -0
  91. data/spec/integration/fifo_ordering/fifo_ordering_spec.rb +44 -0
  92. data/spec/integration/large_payloads/large_payloads_spec.rb +30 -0
  93. data/spec/integration/launcher/launcher_spec.rb +40 -0
  94. data/spec/integration/message_attributes/message_attributes_spec.rb +54 -0
  95. data/spec/integration/message_operations/message_operations_spec.rb +59 -0
  96. data/spec/integration/middleware_chain/empty_chain_spec.rb +11 -0
  97. data/spec/integration/middleware_chain/execution_order_spec.rb +33 -0
  98. data/spec/integration/middleware_chain/removal_spec.rb +31 -0
  99. data/spec/integration/middleware_chain/short_circuit_spec.rb +40 -0
  100. data/spec/integration/polling_strategies/polling_strategies_spec.rb +46 -0
  101. data/spec/integration/queue_operations/queue_operations_spec.rb +84 -0
  102. data/spec/integration/rails/rails_72/Gemfile +6 -0
  103. data/spec/integration/rails/rails_72/activejob_adapter_spec.rb +98 -0
  104. data/spec/integration/rails/rails_80/Gemfile +6 -0
  105. data/spec/integration/rails/rails_80/activejob_adapter_spec.rb +98 -0
  106. data/spec/integration/rails/rails_80/continuation_spec.rb +79 -0
  107. data/spec/integration/rails/rails_81/Gemfile +6 -0
  108. data/spec/integration/rails/rails_81/activejob_adapter_spec.rb +98 -0
  109. data/spec/integration/rails/rails_81/continuation_spec.rb +79 -0
  110. data/spec/integration/retry_behavior/retry_behavior_spec.rb +45 -0
  111. data/spec/integration/spec_helper.rb +7 -0
  112. data/spec/integration/strict_priority_polling/strict_priority_polling_spec.rb +58 -0
  113. data/spec/integration/visibility_timeout/visibility_timeout_spec.rb +37 -0
  114. data/spec/integration/worker_enqueueing/worker_enqueueing_spec.rb +60 -0
  115. data/spec/integration/worker_groups/worker_groups_spec.rb +79 -0
  116. data/spec/integration/worker_lifecycle/worker_lifecycle_spec.rb +33 -0
  117. data/spec/integrations_helper.rb +243 -0
  118. data/spec/lib/active_job/extensions_spec.rb +149 -0
  119. data/spec/lib/active_job/queue_adapters/shoryuken_adapter_spec.rb +29 -0
  120. data/spec/{shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb → lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter_spec.rb} +3 -3
  121. data/spec/{shoryuken/extensions/active_job_wrapper_spec.rb → lib/shoryuken/active_job/job_wrapper_spec.rb} +4 -4
  122. data/spec/{shoryuken → lib/shoryuken}/environment_loader_spec.rb +1 -1
  123. data/spec/{shoryuken → lib/shoryuken}/helpers/hash_utils_spec.rb +14 -14
  124. data/spec/{shoryuken → lib/shoryuken}/helpers/string_utils_spec.rb +3 -3
  125. data/spec/lib/shoryuken/helpers/timer_task_spec.rb +298 -0
  126. data/spec/{shoryuken → lib/shoryuken}/helpers_integration_spec.rb +9 -9
  127. data/spec/{shoryuken → lib/shoryuken}/launcher_spec.rb +22 -0
  128. data/spec/lib/shoryuken/logging_spec.rb +242 -0
  129. data/spec/lib/shoryuken/message_spec.rb +109 -0
  130. data/spec/lib/shoryuken/middleware/entry_spec.rb +68 -0
  131. data/spec/lib/shoryuken/middleware/server/active_record_spec.rb +133 -0
  132. data/spec/{shoryuken → lib/shoryuken}/middleware/server/auto_extend_visibility_spec.rb +50 -0
  133. data/spec/{shoryuken → lib/shoryuken}/options_spec.rb +2 -2
  134. data/spec/{shoryuken → lib/shoryuken}/util_spec.rb +1 -1
  135. data/spec/lib/shoryuken/version_spec.rb +17 -0
  136. data/spec/lib/shoryuken/worker_registry_spec.rb +63 -0
  137. data/spec/shared_examples_for_active_job.rb +29 -9
  138. data/spec/spec_helper.rb +34 -3
  139. metadata +230 -91
  140. data/.devcontainer/Dockerfile +0 -17
  141. data/.devcontainer/base.Dockerfile +0 -43
  142. data/.devcontainer/devcontainer.json +0 -35
  143. data/Appraisals +0 -23
  144. data/gemfiles/.gitignore +0 -1
  145. data/gemfiles/rails_7_0.gemfile +0 -19
  146. data/gemfiles/rails_7_1.gemfile +0 -19
  147. data/gemfiles/rails_7_2.gemfile +0 -19
  148. data/gemfiles/rails_8_0.gemfile +0 -19
  149. data/lib/shoryuken/extensions/active_job_adapter.rb +0 -110
  150. data/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb +0 -50
  151. data/spec/integration/launcher_spec.rb +0 -127
  152. data/spec/shoryuken/extensions/active_job_adapter_spec.rb +0 -8
  153. data/spec/shoryuken/extensions/active_job_base_spec.rb +0 -85
  154. /data/spec/{shoryuken → lib/shoryuken}/body_parser_spec.rb +0 -0
  155. /data/spec/{shoryuken → lib/shoryuken}/client_spec.rb +0 -0
  156. /data/spec/{shoryuken → lib/shoryuken}/default_exception_handler_spec.rb +0 -0
  157. /data/spec/{shoryuken → lib/shoryuken}/default_worker_registry_spec.rb +0 -0
  158. /data/spec/{shoryuken → lib/shoryuken}/fetcher_spec.rb +0 -0
  159. /data/spec/{shoryuken → lib/shoryuken}/helpers/atomic_boolean_spec.rb +0 -0
  160. /data/spec/{shoryuken → lib/shoryuken}/helpers/atomic_counter_spec.rb +0 -0
  161. /data/spec/{shoryuken → lib/shoryuken}/helpers/atomic_hash_spec.rb +0 -0
  162. /data/spec/{shoryuken → lib/shoryuken}/inline_message_spec.rb +0 -0
  163. /data/spec/{shoryuken → lib/shoryuken}/manager_spec.rb +0 -0
  164. /data/spec/{shoryuken → lib/shoryuken}/middleware/chain_spec.rb +0 -0
  165. /data/spec/{shoryuken → lib/shoryuken}/middleware/server/auto_delete_spec.rb +0 -0
  166. /data/spec/{shoryuken → lib/shoryuken}/middleware/server/exponential_backoff_retry_spec.rb +0 -0
  167. /data/spec/{shoryuken → lib/shoryuken}/middleware/server/timing_spec.rb +0 -0
  168. /data/spec/{shoryuken → lib/shoryuken}/polling/base_strategy_spec.rb +0 -0
  169. /data/spec/{shoryuken → lib/shoryuken}/polling/queue_configuration_spec.rb +0 -0
  170. /data/spec/{shoryuken → lib/shoryuken}/polling/strict_priority_spec.rb +0 -0
  171. /data/spec/{shoryuken → lib/shoryuken}/polling/weighted_round_robin_spec.rb +0 -0
  172. /data/spec/{shoryuken → lib/shoryuken}/processor_spec.rb +0 -0
  173. /data/spec/{shoryuken → lib/shoryuken}/queue_spec.rb +0 -0
  174. /data/spec/{shoryuken → lib/shoryuken}/runner_spec.rb +0 -0
  175. /data/spec/{shoryuken → lib/shoryuken}/worker/default_executor_spec.rb +0 -0
  176. /data/spec/{shoryuken → lib/shoryuken}/worker/inline_executor_spec.rb +0 -0
  177. /data/spec/{shoryuken → lib/shoryuken}/worker_spec.rb +0 -0
  178. /data/spec/{shoryuken_spec.rb → lib/shoryuken_spec.rb} +0 -0
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ # CurrentAttributes with full context are persisted and restored during job execution
4
+
5
+ setup_localstack
6
+ setup_active_job
7
+
8
+ require 'active_support/current_attributes'
9
+ require 'shoryuken/active_job/current_attributes'
10
+
11
+ queue_name = DT.queue
12
+ create_test_queue(queue_name)
13
+
14
+ class TestCurrent < ActiveSupport::CurrentAttributes
15
+ attribute :user_id, :tenant_id, :request_id
16
+ end
17
+
18
+ class RequestContext < ActiveSupport::CurrentAttributes
19
+ attribute :locale, :timezone, :trace_id
20
+ end
21
+
22
+ Shoryuken::ActiveJob::CurrentAttributes.persist(TestCurrent, RequestContext)
23
+
24
+ class FullContextTestJob < ActiveJob::Base
25
+ def perform
26
+ DT[:executions] << {
27
+ user_id: TestCurrent.user_id,
28
+ tenant_id: TestCurrent.tenant_id,
29
+ request_id: TestCurrent.request_id,
30
+ locale: RequestContext.locale,
31
+ timezone: RequestContext.timezone,
32
+ trace_id: RequestContext.trace_id
33
+ }
34
+ end
35
+ end
36
+
37
+ FullContextTestJob.queue_as(queue_name)
38
+
39
+ Shoryuken.add_group('default', 1)
40
+ Shoryuken.add_queue(queue_name, 1, 'default')
41
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
42
+
43
+ TestCurrent.user_id = 42
44
+ TestCurrent.tenant_id = 'acme-corp'
45
+ TestCurrent.request_id = 'req-123-abc'
46
+ RequestContext.locale = 'en-US'
47
+ RequestContext.timezone = 'America/New_York'
48
+ RequestContext.trace_id = 'trace-xyz-789'
49
+
50
+ FullContextTestJob.perform_later
51
+
52
+ TestCurrent.reset
53
+ RequestContext.reset
54
+
55
+ poll_queues_until(timeout: 30) { DT[:executions].size >= 1 }
56
+
57
+ result = DT[:executions].first
58
+ assert_equal(42, result[:user_id])
59
+ assert_equal('acme-corp', result[:tenant_id])
60
+ assert_equal('req-123-abc', result[:request_id])
61
+ assert_equal('en-US', result[:locale])
62
+ assert_equal('America/New_York', result[:timezone])
63
+ assert_equal('trace-xyz-789', result[:trace_id])
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ # CurrentAttributes with partial values set preserve only set attributes during job execution
4
+
5
+ setup_localstack
6
+ setup_active_job
7
+
8
+ require 'active_support/current_attributes'
9
+ require 'shoryuken/active_job/current_attributes'
10
+
11
+ queue_name = DT.queue
12
+ create_test_queue(queue_name)
13
+
14
+ class TestCurrent < ActiveSupport::CurrentAttributes
15
+ attribute :user_id, :tenant_id, :request_id
16
+ end
17
+
18
+ class RequestContext < ActiveSupport::CurrentAttributes
19
+ attribute :locale, :timezone
20
+ end
21
+
22
+ Shoryuken::ActiveJob::CurrentAttributes.persist(TestCurrent, RequestContext)
23
+
24
+ class PartialContextTestJob < ActiveJob::Base
25
+ def perform
26
+ DT[:executions] << {
27
+ user_id: TestCurrent.user_id,
28
+ tenant_id: TestCurrent.tenant_id,
29
+ request_id: TestCurrent.request_id,
30
+ locale: RequestContext.locale,
31
+ timezone: RequestContext.timezone
32
+ }
33
+ end
34
+ end
35
+
36
+ PartialContextTestJob.queue_as(queue_name)
37
+
38
+ Shoryuken.add_group('default', 1)
39
+ Shoryuken.add_queue(queue_name, 1, 'default')
40
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
41
+
42
+ TestCurrent.user_id = 99
43
+ RequestContext.locale = 'fr-FR'
44
+
45
+ PartialContextTestJob.perform_later
46
+
47
+ TestCurrent.reset
48
+ RequestContext.reset
49
+
50
+ poll_queues_until(timeout: 30) { DT[:executions].size >= 1 }
51
+
52
+ result = DT[:executions].first
53
+ assert_equal(99, result[:user_id])
54
+ assert(result[:tenant_id].nil?)
55
+ assert(result[:request_id].nil?)
56
+ assert_equal('fr-FR', result[:locale])
57
+ assert(result[:timezone].nil?)
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ # ActiveJob custom numeric message attributes are sent to SQS with correct data type
4
+
5
+ setup_localstack
6
+ setup_active_job
7
+
8
+ queue_name = DT.queue
9
+ create_test_queue(queue_name)
10
+
11
+ class NumberAttributesTestJob < ActiveJob::Base
12
+ def perform
13
+ DT[:executions] << { job_id: job_id }
14
+ end
15
+ end
16
+
17
+ NumberAttributesTestJob.queue_as(queue_name)
18
+
19
+ Shoryuken.add_group('default', 1)
20
+ Shoryuken.add_queue(queue_name, 1, 'default')
21
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
22
+
23
+ job = NumberAttributesTestJob.new
24
+ job.sqs_send_message_parameters = {
25
+ message_attributes: {
26
+ 'priority' => { string_value: '10', data_type: 'Number' },
27
+ 'retry_count' => { string_value: '0', data_type: 'Number' }
28
+ }
29
+ }
30
+ ActiveJob::QueueAdapters::ShoryukenAdapter.enqueue(job)
31
+
32
+ poll_queues_until(timeout: 30) { DT[:executions].size >= 1 }
33
+
34
+ params = job.sqs_send_message_parameters
35
+ assert(params[:message_attributes].key?('priority'))
36
+ assert_equal('10', params[:message_attributes]['priority'][:string_value])
37
+ assert_equal('Number', params[:message_attributes]['priority'][:data_type])
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ # ActiveJob custom string message attributes are sent to SQS and preserved
4
+
5
+ setup_localstack
6
+ setup_active_job
7
+
8
+ queue_name = DT.queue
9
+ create_test_queue(queue_name)
10
+
11
+ class StringAttributesTestJob < ActiveJob::Base
12
+ def perform
13
+ DT[:executions] << { job_id: job_id }
14
+ end
15
+ end
16
+
17
+ StringAttributesTestJob.queue_as(queue_name)
18
+
19
+ Shoryuken.add_group('default', 1)
20
+ Shoryuken.add_queue(queue_name, 1, 'default')
21
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
22
+
23
+ job = StringAttributesTestJob.new
24
+ job.sqs_send_message_parameters = {
25
+ message_attributes: {
26
+ 'trace_id' => { string_value: 'trace-abc-123', data_type: 'String' },
27
+ 'correlation_id' => { string_value: 'corr-xyz-789', data_type: 'String' }
28
+ }
29
+ }
30
+ ActiveJob::QueueAdapters::ShoryukenAdapter.enqueue(job)
31
+
32
+ poll_queues_until(timeout: 30) { DT[:executions].size >= 1 }
33
+
34
+ params = job.sqs_send_message_parameters
35
+ assert(params[:message_attributes].key?('trace_id'))
36
+ assert(params[:message_attributes].key?('correlation_id'))
37
+ assert(params[:message_attributes].key?('shoryuken_class'))
38
+ assert_equal('trace-abc-123', params[:message_attributes]['trace_id'][:string_value])
39
+ assert_equal('corr-xyz-789', params[:message_attributes]['correlation_id'][:string_value])
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests error handling including retry configuration,
4
+ # discard configuration, and job processing through JobWrapper.
5
+
6
+ setup_active_job
7
+
8
+ class RetryableJob < ActiveJob::Base
9
+ queue_as :default
10
+ retry_on StandardError, wait: 1.second, attempts: 3
11
+
12
+ def perform(should_fail = true)
13
+ raise StandardError, 'Job failed!' if should_fail
14
+ 'Job succeeded!'
15
+ end
16
+ end
17
+
18
+ class DiscardableJob < ActiveJob::Base
19
+ queue_as :default
20
+ discard_on ArgumentError
21
+
22
+ def perform(should_fail = false)
23
+ raise ArgumentError, 'Invalid argument' if should_fail
24
+ 'Job succeeded!'
25
+ end
26
+ end
27
+
28
+ job_capture = JobCapture.new
29
+ job_capture.start_capturing
30
+
31
+ RetryableJob.perform_later(false)
32
+
33
+ assert_equal(1, job_capture.job_count)
34
+ job = job_capture.last_job
35
+ message_body = job[:message_body]
36
+ assert_equal('RetryableJob', message_body['job_class'])
37
+ assert_equal([false], message_body['arguments'])
38
+
39
+ job_capture2 = JobCapture.new
40
+ job_capture2.start_capturing
41
+
42
+ DiscardableJob.perform_later(false)
43
+
44
+ assert_equal(1, job_capture2.job_count)
45
+ job2 = job_capture2.last_job
46
+ message_body2 = job2[:message_body]
47
+ assert_equal('DiscardableJob', message_body2['job_class'])
48
+
49
+ wrapper_class = Shoryuken::ActiveJob::JobWrapper
50
+ options = wrapper_class.get_shoryuken_options
51
+
52
+ assert_equal(:json, options['body_parser'])
53
+ assert_equal(true, options['auto_delete'])
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'digest'
4
+
5
+ # This spec tests FIFO queue support including message deduplication ID generation
6
+ # and message attributes handling.
7
+
8
+ setup_active_job
9
+
10
+ class FifoTestJob < ActiveJob::Base
11
+ queue_as :test_fifo
12
+
13
+ def perform(order_id, action)
14
+ "Processed order #{order_id}: #{action}"
15
+ end
16
+ end
17
+
18
+ class AttributesTestJob < ActiveJob::Base
19
+ queue_as :attributes_test
20
+
21
+ def perform(data)
22
+ "Processed: #{data}"
23
+ end
24
+ end
25
+
26
+ fifo_queue_mock = Object.new
27
+ fifo_queue_mock.define_singleton_method(:fifo?) { true }
28
+ fifo_queue_mock.define_singleton_method(:name) { 'test_fifo.fifo' }
29
+
30
+ captured_params = nil
31
+ fifo_queue_mock.define_singleton_method(:send_message) do |params|
32
+ captured_params = params
33
+ end
34
+
35
+ Shoryuken::Client.define_singleton_method(:queues) do |queue_name = nil|
36
+ if queue_name
37
+ fifo_queue_mock
38
+ else
39
+ { test_fifo: fifo_queue_mock }
40
+ end
41
+ end
42
+
43
+ Shoryuken.define_singleton_method(:register_worker) { |*args| nil }
44
+
45
+ FifoTestJob.perform_later('order-123', 'process')
46
+
47
+ assert(captured_params.key?(:message_deduplication_id))
48
+ assert_equal(64, captured_params[:message_deduplication_id].length)
49
+
50
+ body = captured_params[:message_body]
51
+ body_without_variable_fields = body.except('job_id', 'enqueued_at')
52
+ expected_dedupe_id = Digest::SHA256.hexdigest(JSON.dump(body_without_variable_fields))
53
+ assert_equal(expected_dedupe_id, captured_params[:message_deduplication_id])
54
+
55
+ regular_queue_mock = Object.new
56
+ regular_queue_mock.define_singleton_method(:fifo?) { false }
57
+ regular_queue_mock.define_singleton_method(:name) { 'attributes_test' }
58
+
59
+ captured_attrs = nil
60
+ regular_queue_mock.define_singleton_method(:send_message) do |params|
61
+ captured_attrs = params
62
+ end
63
+
64
+ Shoryuken::Client.define_singleton_method(:queues) do |queue_name = nil|
65
+ regular_queue_mock
66
+ end
67
+
68
+ custom_attributes = {
69
+ 'trace_id' => { string_value: 'trace-123', data_type: 'String' },
70
+ 'priority' => { string_value: 'high', data_type: 'String' }
71
+ }
72
+
73
+ job = AttributesTestJob.new('test data')
74
+ job.sqs_send_message_parameters = { message_attributes: custom_attributes }
75
+ ActiveJob::QueueAdapters::ShoryukenAdapter.enqueue(job)
76
+
77
+ attributes = captured_attrs[:message_attributes]
78
+ assert_equal(custom_attributes['trace_id'], attributes['trace_id'])
79
+ assert_equal(custom_attributes['priority'], attributes['priority'])
80
+
81
+ # Should still include required Shoryuken attribute
82
+ expected_shoryuken_class = {
83
+ string_value: "Shoryuken::ActiveJob::JobWrapper",
84
+ data_type: 'String'
85
+ }
86
+ assert_equal(expected_shoryuken_class, attributes['shoryuken_class'])
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ # ActiveJob discard_on discards jobs that raise specific errors without retry
4
+
5
+ setup_localstack
6
+ setup_active_job
7
+
8
+ queue_name = DT.queue
9
+ create_test_queue(queue_name)
10
+
11
+ class DiscardOnTestJob < ActiveJob::Base
12
+ discard_on ArgumentError
13
+
14
+ def perform(should_fail)
15
+ DT[:attempts] << { job_id: job_id, should_fail: should_fail }
16
+
17
+ if should_fail
18
+ raise ArgumentError, "This should be discarded"
19
+ end
20
+
21
+ DT[:successes] << { job_id: job_id }
22
+ end
23
+ end
24
+
25
+ DiscardOnTestJob.queue_as(queue_name)
26
+
27
+ Shoryuken.add_group('default', 1)
28
+ Shoryuken.add_queue(queue_name, 1, 'default')
29
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
30
+
31
+ failing_job = DiscardOnTestJob.perform_later(true)
32
+ success_job = DiscardOnTestJob.perform_later(false)
33
+
34
+ poll_queues_until(timeout: 30) { DT[:attempts].size >= 2 }
35
+
36
+ failing_attempts = DT[:attempts].select { |a| a[:job_id] == failing_job.job_id }
37
+ assert_equal(1, failing_attempts.size, "Discarded job should only attempt once")
38
+
39
+ failing_successes = DT[:successes].select { |s| s[:job_id] == failing_job.job_id }
40
+ assert_equal(0, failing_successes.size, "Discarded job should not succeed")
41
+
42
+ success_successes = DT[:successes].select { |s| s[:job_id] == success_job.job_id }
43
+ assert_equal(1, success_successes.size, "Non-failing job should succeed")
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ # ActiveJob retry_on re-enqueues failed jobs until they succeed or exhaust attempts
4
+
5
+ setup_localstack
6
+ setup_active_job
7
+
8
+ queue_name = DT.queue
9
+ create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' })
10
+
11
+ class RetryOnTestJob < ActiveJob::Base
12
+ retry_on StandardError, wait: 0, attempts: 3
13
+
14
+ def perform
15
+ DT[:attempts] << { job_id: job_id, attempt: executions + 1, time: Time.now }
16
+
17
+ if DT[:attempts].count { |a| a[:job_id] == job_id } < 3
18
+ raise StandardError, "Simulated failure"
19
+ end
20
+
21
+ DT[:successes] << { job_id: job_id, final_attempt: executions + 1 }
22
+ end
23
+ end
24
+
25
+ RetryOnTestJob.queue_as(queue_name)
26
+
27
+ Shoryuken.add_group('default', 1)
28
+ Shoryuken.add_queue(queue_name, 1, 'default')
29
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
30
+
31
+ RetryOnTestJob.perform_later
32
+
33
+ poll_queues_until(timeout: 30) { DT[:successes].size >= 1 }
34
+
35
+ assert(DT[:attempts].size >= 2, "Expected at least 2 retry attempts, got #{DT[:attempts].size}")
36
+ assert_equal(1, DT[:successes].size)
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Full round-trip ActiveJob integration test
4
+ # Enqueues a job via ActiveJob → sends to LocalStack SQS → processes via Shoryuken → verifies execution
5
+
6
+ setup_localstack
7
+ setup_active_job
8
+
9
+ queue_name = DT.queue
10
+ create_test_queue(queue_name)
11
+
12
+ class RoundtripTestJob < ActiveJob::Base
13
+ def perform(payload)
14
+ DT[:executions] << {
15
+ payload: payload,
16
+ executed_at: Time.now,
17
+ job_id: job_id
18
+ }
19
+ end
20
+ end
21
+
22
+ RoundtripTestJob.queue_as(queue_name)
23
+
24
+ Shoryuken.add_group('default', 1)
25
+ Shoryuken.add_queue(queue_name, 1, 'default')
26
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
27
+
28
+ RoundtripTestJob.perform_later('first_payload')
29
+ RoundtripTestJob.perform_later('second_payload')
30
+ RoundtripTestJob.perform_later({ key: 'complex', data: [1, 2, 3] })
31
+
32
+ poll_queues_until(timeout: 30) do
33
+ DT[:executions].size >= 3
34
+ end
35
+
36
+ assert_equal(3, DT[:executions].size, "Expected 3 job executions, got #{DT[:executions].size}")
37
+
38
+ payloads = DT[:executions].map { |e| e[:payload] }
39
+ assert_includes(payloads, 'first_payload')
40
+ assert_includes(payloads, 'second_payload')
41
+
42
+ complex_payload = payloads.find { |p| p.is_a?(Hash) }
43
+ assert(complex_payload, "Expected to find complex payload")
44
+ # Keys may be strings or symbols depending on serialization
45
+ key_value = complex_payload['key'] || complex_payload[:key]
46
+ data_value = complex_payload['data'] || complex_payload[:data]
47
+ assert_equal('complex', key_value)
48
+ assert_equal([1, 2, 3], data_value)
49
+
50
+ job_ids = DT[:executions].map { |e| e[:job_id] }
51
+ assert(job_ids.all? { |id| id && !id.empty? }, "All jobs should have job IDs")
52
+ assert_equal(3, job_ids.uniq.size, "All job IDs should be unique")
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Scheduled ActiveJob integration test
4
+ # Tests jobs scheduled with set(wait:) are delivered after the delay
5
+
6
+ setup_localstack
7
+ setup_active_job
8
+
9
+ queue_name = DT.queue
10
+ create_test_queue(queue_name)
11
+
12
+ class ScheduledTestJob < ActiveJob::Base
13
+ def perform(label)
14
+ DT[:executions] << {
15
+ label: label,
16
+ job_id: job_id,
17
+ executed_at: Time.now
18
+ }
19
+ end
20
+ end
21
+
22
+ ScheduledTestJob.queue_as(queue_name)
23
+
24
+ Shoryuken.add_group('default', 1)
25
+ Shoryuken.add_queue(queue_name, 1, 'default')
26
+ Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
27
+
28
+ immediate_enqueue_time = Time.now
29
+ ScheduledTestJob.perform_later('immediate')
30
+ DT[:timestamps] << { label: 'immediate', time: immediate_enqueue_time }
31
+
32
+ delayed_enqueue_time = Time.now
33
+ ScheduledTestJob.set(wait: 3.seconds).perform_later('delayed_3s')
34
+ DT[:timestamps] << { label: 'delayed_3s', time: delayed_enqueue_time }
35
+
36
+ delayed_5s_enqueue_time = Time.now
37
+ ScheduledTestJob.set(wait: 5.seconds).perform_later('delayed_5s')
38
+ DT[:timestamps] << { label: 'delayed_5s', time: delayed_5s_enqueue_time }
39
+
40
+ poll_queues_until(timeout: 30) do
41
+ DT[:executions].size >= 3
42
+ end
43
+
44
+ assert_equal(3, DT[:executions].size, "Expected 3 job executions")
45
+
46
+ # Find each job's execution
47
+ immediate_job = DT[:executions].find { |e| e[:label] == 'immediate' }
48
+ delayed_3s_job = DT[:executions].find { |e| e[:label] == 'delayed_3s' }
49
+ delayed_5s_job = DT[:executions].find { |e| e[:label] == 'delayed_5s' }
50
+
51
+ assert(immediate_job, "Immediate job should have executed")
52
+ assert(delayed_3s_job, "3s delayed job should have executed")
53
+ assert(delayed_5s_job, "5s delayed job should have executed")
54
+
55
+ def enqueue_time(label)
56
+ DT[:timestamps].find { |t| t[:label] == label }[:time]
57
+ end
58
+
59
+ immediate_delay = immediate_job[:executed_at] - enqueue_time('immediate')
60
+ assert(immediate_delay < 10, "Immediate job should execute within 10 seconds, took #{immediate_delay}s")
61
+
62
+ # Using 2 seconds tolerance for SQS delivery variation
63
+ delayed_3s_actual_delay = delayed_3s_job[:executed_at] - enqueue_time('delayed_3s')
64
+ assert(delayed_3s_actual_delay >= 2, "3s delayed job should execute after at least 2s, took #{delayed_3s_actual_delay}s")
65
+
66
+ delayed_5s_actual_delay = delayed_5s_job[:executed_at] - enqueue_time('delayed_5s')
67
+ assert(delayed_5s_actual_delay >= 4, "5s delayed job should execute after at least 4s, took #{delayed_5s_actual_delay}s")
68
+
69
+ assert(
70
+ immediate_job[:executed_at] <= delayed_3s_job[:executed_at],
71
+ "Immediate job should execute before 3s delayed job"
72
+ )
73
+ assert(
74
+ delayed_3s_job[:executed_at] <= delayed_5s_job[:executed_at],
75
+ "3s delayed job should execute before 5s delayed job"
76
+ )
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests the ActiveRecord middleware functionality.
4
+ # The middleware clears database connections after each message is processed.
5
+
6
+ setup_localstack
7
+
8
+ queue_name = DT.queue
9
+ create_test_queue(queue_name)
10
+ Shoryuken.add_group('default', 1)
11
+ Shoryuken.add_queue(queue_name, 1, 'default')
12
+
13
+ # Mock ActiveRecord module to track connection clearing
14
+ module ActiveRecord
15
+ VERSION = Gem::Version.new('7.2.0')
16
+
17
+ def self.version
18
+ VERSION
19
+ end
20
+
21
+ class Base
22
+ class << self
23
+ attr_accessor :connections_cleared
24
+
25
+ def connection_handler
26
+ @connection_handler ||= ConnectionHandler.new
27
+ end
28
+ end
29
+
30
+ self.connections_cleared = []
31
+ end
32
+
33
+ class ConnectionHandler
34
+ def clear_active_connections!(scope)
35
+ ActiveRecord::Base.connections_cleared << { scope: scope, time: Time.now }
36
+ end
37
+ end
38
+ end
39
+
40
+ # Add the ActiveRecord middleware to the chain
41
+ require 'shoryuken/middleware/server/active_record'
42
+ Shoryuken.configure_server do |config|
43
+ config.server_middleware do |chain|
44
+ chain.add Shoryuken::Middleware::Server::ActiveRecord
45
+ end
46
+ end
47
+
48
+ worker_class = Class.new do
49
+ include Shoryuken::Worker
50
+
51
+ shoryuken_options auto_delete: true, batch: false
52
+
53
+ def perform(sqs_msg, body)
54
+ DT[:processed] << { message_id: sqs_msg.message_id, body: body }
55
+ end
56
+ end
57
+
58
+ worker_class.get_shoryuken_options['queue'] = queue_name
59
+ Shoryuken.register_worker(queue_name, worker_class)
60
+
61
+ # Clear any prior connection clearing records
62
+ ActiveRecord::Base.connections_cleared.clear
63
+
64
+ # Send multiple messages
65
+ 3.times { |i| Shoryuken::Client.queues(queue_name).send_message(message_body: "ar-test-#{i}") }
66
+
67
+ sleep 1
68
+
69
+ poll_queues_until { DT[:processed].size >= 3 }
70
+
71
+ # Verify all messages were processed
72
+ assert_equal(3, DT[:processed].size)
73
+
74
+ # Verify ActiveRecord connections were cleared after each message
75
+ # The middleware should have called clear_active_connections! for each message
76
+ assert(
77
+ ActiveRecord::Base.connections_cleared.size >= 3,
78
+ "ActiveRecord connections should be cleared after each message (cleared #{ActiveRecord::Base.connections_cleared.size} times)"
79
+ )
80
+
81
+ # Verify the :all scope was used (Rails 7.1+ behavior)
82
+ ActiveRecord::Base.connections_cleared.each do |record|
83
+ assert_equal(:all, record[:scope], 'Should use :all scope for Rails 7.1+')
84
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ # This spec tests the auto_delete middleware functionality.
4
+ # When auto_delete: true, messages should be automatically deleted after successful processing.
5
+
6
+ setup_localstack
7
+
8
+ queue_name = DT.queue
9
+ create_test_queue(queue_name)
10
+ Shoryuken.add_group('default', 1)
11
+ Shoryuken.add_queue(queue_name, 1, 'default')
12
+
13
+ auto_delete_worker = Class.new do
14
+ include Shoryuken::Worker
15
+
16
+ shoryuken_options auto_delete: true
17
+
18
+ def perform(sqs_msg, body)
19
+ DT[:auto_delete_processed] << { message_id: sqs_msg.message_id, body: body }
20
+ end
21
+ end
22
+
23
+ auto_delete_worker.get_shoryuken_options['queue'] = queue_name
24
+ Shoryuken.register_worker(queue_name, auto_delete_worker)
25
+
26
+ queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
27
+
28
+ # Send a message
29
+ Shoryuken::Client.sqs.send_message(
30
+ queue_url: queue_url,
31
+ message_body: 'auto delete test'
32
+ )
33
+
34
+ sleep 1
35
+
36
+ # Process the message
37
+ poll_queues_until { DT[:auto_delete_processed].size >= 1 }
38
+
39
+ assert_equal(1, DT[:auto_delete_processed].size)
40
+ assert_equal('auto delete test', DT[:auto_delete_processed].first[:body])
41
+
42
+ # Wait a moment for deletion to complete
43
+ sleep 2
44
+
45
+ # Verify message was deleted - queue should be empty
46
+ attributes = Shoryuken::Client.sqs.get_queue_attributes(
47
+ queue_url: queue_url,
48
+ attribute_names: ['ApproximateNumberOfMessages', 'ApproximateNumberOfMessagesNotVisible']
49
+ ).attributes
50
+
51
+ total_messages = attributes['ApproximateNumberOfMessages'].to_i +
52
+ attributes['ApproximateNumberOfMessagesNotVisible'].to_i
53
+ assert_equal(0, total_messages, "Message should be deleted when auto_delete: true")