shoryuken 7.0.1 → 7.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/push.yml +3 -3
  3. data/.github/workflows/specs.yml +37 -10
  4. data/.github/workflows/verify-action-pins.yml +1 -1
  5. data/.gitignore +0 -1
  6. data/.ruby-version +1 -1
  7. data/CHANGELOG.md +120 -0
  8. data/README.md +1 -1
  9. data/bin/{clean_localstack → clean_sqs} +4 -4
  10. data/bin/cli/sqs.rb +7 -0
  11. data/bin/integrations +52 -34
  12. data/docker-compose.yml +4 -20
  13. data/lib/active_job/queue_adapters/shoryuken_adapter.rb +3 -1
  14. data/lib/shoryuken/active_job/current_attributes.rb +35 -7
  15. data/lib/shoryuken/environment_loader.rb +1 -1
  16. data/lib/shoryuken/fetcher.rb +7 -1
  17. data/lib/shoryuken/helpers/timer_task.rb +19 -2
  18. data/lib/shoryuken/launcher.rb +21 -5
  19. data/lib/shoryuken/manager.rb +38 -5
  20. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +32 -2
  21. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +8 -3
  22. data/lib/shoryuken/middleware/server/non_retryable_exception.rb +17 -8
  23. data/lib/shoryuken/options.rb +22 -3
  24. data/lib/shoryuken/polling/strict_priority.rb +26 -14
  25. data/lib/shoryuken/polling/weighted_round_robin.rb +41 -27
  26. data/lib/shoryuken/queue.rb +8 -1
  27. data/lib/shoryuken/util.rb +4 -1
  28. data/lib/shoryuken/version.rb +1 -1
  29. data/lib/shoryuken/worker.rb +5 -1
  30. data/lib/shoryuken.rb +2 -0
  31. data/renovate.json +26 -2
  32. data/spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb +1 -1
  33. data/spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb +1 -1
  34. data/spec/integration/active_job/current_attributes/complex_types_spec.rb +1 -1
  35. data/spec/integration/active_job/current_attributes/cross_job_reset_spec.rb +47 -0
  36. data/spec/integration/active_job/current_attributes/empty_context_spec.rb +1 -1
  37. data/spec/integration/active_job/current_attributes/full_context_spec.rb +1 -1
  38. data/spec/integration/active_job/current_attributes/incremental_persist_spec.rb +76 -0
  39. data/spec/integration/active_job/current_attributes/partial_context_spec.rb +1 -1
  40. data/spec/integration/active_job/custom_attributes/number_attributes_spec.rb +1 -1
  41. data/spec/integration/active_job/custom_attributes/string_attributes_spec.rb +1 -1
  42. data/spec/integration/active_job/fifo_dedup_opt_out/fifo_dedup_opt_out_spec.rb +67 -0
  43. data/spec/integration/active_job/keyword_arguments/keyword_arguments_spec.rb +1 -1
  44. data/spec/integration/active_job/retry/discard_on_spec.rb +1 -1
  45. data/spec/integration/active_job/retry/retry_on_spec.rb +1 -1
  46. data/spec/integration/active_job/roundtrip/roundtrip_spec.rb +2 -2
  47. data/spec/integration/active_job/scheduled/scheduled_spec.rb +1 -1
  48. data/spec/integration/active_record_middleware/active_record_middleware_spec.rb +1 -1
  49. data/spec/integration/auto_delete/auto_delete_spec.rb +1 -1
  50. data/spec/integration/auto_extend_visibility/auto_extend_visibility_spec.rb +1 -1
  51. data/spec/integration/auto_extend_visibility/short_visibility_timeout_spec.rb +52 -0
  52. data/spec/integration/aws_config/aws_config_spec.rb +3 -3
  53. data/spec/integration/batch_processing/batch_processing_spec.rb +1 -1
  54. data/spec/integration/body_parser/json_parser_spec.rb +1 -1
  55. data/spec/integration/body_parser/proc_parser_spec.rb +1 -1
  56. data/spec/integration/body_parser/text_parser_spec.rb +1 -1
  57. data/spec/integration/concurrent_processing/concurrent_processing_spec.rb +1 -1
  58. data/spec/integration/concurrent_processing/processor_accounting_spec.rb +94 -0
  59. data/spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb +87 -0
  60. data/spec/integration/dead_letter_queue/dead_letter_queue_spec.rb +1 -1
  61. data/spec/integration/exception_handlers/exception_handlers_spec.rb +1 -1
  62. data/spec/integration/exponential_backoff/exponential_backoff_spec.rb +1 -1
  63. data/spec/integration/fifo_ordering/fifo_max_messages_cap_spec.rb +96 -0
  64. data/spec/integration/fifo_ordering/fifo_ordering_spec.rb +1 -1
  65. data/spec/integration/large_payloads/large_payloads_spec.rb +1 -1
  66. data/spec/integration/launcher/double_graceful_stop_spec.rb +71 -0
  67. data/spec/integration/launcher/embedded_dispatch_error_spec.rb +85 -0
  68. data/spec/integration/launcher/global_executor_preserved_spec.rb +76 -0
  69. data/spec/integration/launcher/graceful_stop_timeout_spec.rb +74 -0
  70. data/spec/integration/launcher/launcher_spec.rb +1 -1
  71. data/spec/integration/message_attributes/message_attributes_spec.rb +1 -1
  72. data/spec/integration/message_operations/message_operations_spec.rb +1 -1
  73. data/spec/integration/message_operations/partial_batch_delete_spec.rb +67 -0
  74. data/spec/integration/non_retryable_exception/non_retryable_exception_spec.rb +2 -2
  75. data/spec/integration/non_retryable_exception/with_retry_intervals_spec.rb +115 -0
  76. data/spec/integration/polling_strategies/polling_strategies_spec.rb +1 -1
  77. data/spec/integration/queue_operations/queue_operations_spec.rb +1 -1
  78. data/spec/integration/retry_behavior/retry_behavior_spec.rb +1 -1
  79. data/spec/integration/strict_priority_polling/strict_priority_polling_spec.rb +1 -1
  80. data/spec/integration/visibility_timeout/visibility_timeout_spec.rb +1 -1
  81. data/spec/integration/worker_enqueueing/worker_enqueueing_spec.rb +1 -1
  82. data/spec/integration/worker_groups/worker_groups_spec.rb +1 -1
  83. data/spec/integration/worker_lifecycle/worker_lifecycle_spec.rb +1 -1
  84. data/spec/integrations_helper.rb +13 -12
  85. data/spec/lib/shoryuken/fetcher_spec.rb +13 -0
  86. data/spec/lib/shoryuken/helpers/timer_task_spec.rb +24 -0
  87. data/spec/lib/shoryuken/launcher_spec.rb +38 -0
  88. data/spec/lib/shoryuken/manager_spec.rb +99 -0
  89. data/spec/lib/shoryuken/middleware/server/auto_extend_visibility_spec.rb +35 -0
  90. data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +56 -0
  91. data/spec/lib/shoryuken/options_spec.rb +43 -0
  92. data/spec/lib/shoryuken/polling/strict_priority_spec.rb +25 -0
  93. data/spec/lib/shoryuken/polling/weighted_round_robin_spec.rb +50 -0
  94. data/spec/lib/shoryuken/queue_spec.rb +37 -0
  95. data/spec/lib/shoryuken/util_spec.rb +26 -0
  96. data/spec/shared_examples_for_active_job.rb +18 -0
  97. data/spec/spec_helper.rb +26 -7
  98. metadata +29 -3
data/spec/spec_helper.rb CHANGED
@@ -1,10 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- Warning[:performance] = true if RUBY_VERSION >= '3.3'
4
- Warning[:deprecated] = true
3
+ require 'warning'
4
+
5
5
  $VERBOSE = true
6
6
 
7
- require 'warning'
7
+ if Warning.respond_to?(:categories)
8
+ (Warning.categories - %i[experimental]).each do |cat|
9
+ Warning[cat] = true
10
+ end
11
+ end
8
12
 
9
13
  Warning.process do |warning|
10
14
  # Only check warnings from our code (not dependencies)
@@ -13,10 +17,6 @@ Warning.process do |warning|
13
17
  # Filter out warnings we don't care about in specs
14
18
  next if warning.include?('_spec')
15
19
 
16
- # We redefine methods to simulate various scenarios in tests
17
- next if warning.include?('previous definition of')
18
- next if warning.include?('method redefined')
19
-
20
20
  # Ignore vendor and bundle directories
21
21
  next if warning.include?('vendor/')
22
22
  next if warning.include?('bundle/')
@@ -77,6 +77,24 @@ class TestWorker
77
77
  def perform(sqs_msg, body); end
78
78
  end
79
79
 
80
+ # Emit a full Ruby thread dump when the process receives USR1. The CI
81
+ # watchdog (specs.yml) sends this signal when unit specs run longer than
82
+ # expected so we can see exactly which thread is stuck without waiting for the
83
+ # job-level timeout to kill the process silently.
84
+ if Signal.list.key?('USR1')
85
+ Signal.trap('USR1') do
86
+ # Signal handlers run in a restricted context; use a plain IO write
87
+ # rather than puts/logger to stay async-signal safe.
88
+ output = +"\n=== Thread dump (USR1 watchdog) ===\n"
89
+ Thread.list.each do |t|
90
+ output << "--- Thread #{t.object_id} [#{t.status}] ---\n"
91
+ output << ((t.backtrace || ['(no backtrace)']).join("\n")) << "\n"
92
+ end
93
+ output << "=== End thread dump ===\n"
94
+ $stderr.write(output)
95
+ end
96
+ end
97
+
80
98
  RSpec.configure do |config|
81
99
  config.before do
82
100
  Shoryuken::Client.class_variable_set :@@queues, {}
@@ -98,6 +116,7 @@ RSpec.configure do |config|
98
116
  TestWorker.get_shoryuken_options['queue'] = 'default'
99
117
 
100
118
  Shoryuken.active_job_queue_name_prefixing = false
119
+ Shoryuken.active_job_fifo_message_deduplication = true
101
120
 
102
121
  Shoryuken.worker_registry.clear
103
122
  Shoryuken.register_worker('default', TestWorker)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: shoryuken
3
3
  version: !ruby/object:Gem::Version
4
- version: 7.0.1
4
+ version: 7.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pablo Cantero
@@ -158,7 +158,7 @@ files:
158
158
  - LICENSE
159
159
  - README.md
160
160
  - Rakefile
161
- - bin/clean_localstack
161
+ - bin/clean_sqs
162
162
  - bin/cli/base.rb
163
163
  - bin/cli/sqs.rb
164
164
  - bin/integrations
@@ -223,13 +223,16 @@ files:
223
223
  - spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb
224
224
  - spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb
225
225
  - spec/integration/active_job/current_attributes/complex_types_spec.rb
226
+ - spec/integration/active_job/current_attributes/cross_job_reset_spec.rb
226
227
  - spec/integration/active_job/current_attributes/empty_context_spec.rb
227
228
  - spec/integration/active_job/current_attributes/full_context_spec.rb
229
+ - spec/integration/active_job/current_attributes/incremental_persist_spec.rb
228
230
  - spec/integration/active_job/current_attributes/partial_context_spec.rb
229
231
  - spec/integration/active_job/custom_attributes/number_attributes_spec.rb
230
232
  - spec/integration/active_job/custom_attributes/string_attributes_spec.rb
231
233
  - spec/integration/active_job/error_handling/job_wrapper_spec.rb
232
234
  - spec/integration/active_job/fifo_and_attributes/deduplication_spec.rb
235
+ - spec/integration/active_job/fifo_dedup_opt_out/fifo_dedup_opt_out_spec.rb
233
236
  - spec/integration/active_job/keyword_arguments/keyword_arguments_spec.rb
234
237
  - spec/integration/active_job/retry/discard_on_spec.rb
235
238
  - spec/integration/active_job/retry/retry_on_spec.rb
@@ -238,25 +241,35 @@ files:
238
241
  - spec/integration/active_record_middleware/active_record_middleware_spec.rb
239
242
  - spec/integration/auto_delete/auto_delete_spec.rb
240
243
  - spec/integration/auto_extend_visibility/auto_extend_visibility_spec.rb
244
+ - spec/integration/auto_extend_visibility/short_visibility_timeout_spec.rb
241
245
  - spec/integration/aws_config/aws_config_spec.rb
242
246
  - spec/integration/batch_processing/batch_processing_spec.rb
243
247
  - spec/integration/body_parser/json_parser_spec.rb
244
248
  - spec/integration/body_parser/proc_parser_spec.rb
245
249
  - spec/integration/body_parser/text_parser_spec.rb
246
250
  - spec/integration/concurrent_processing/concurrent_processing_spec.rb
251
+ - spec/integration/concurrent_processing/processor_accounting_spec.rb
252
+ - spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb
247
253
  - spec/integration/dead_letter_queue/dead_letter_queue_spec.rb
248
254
  - spec/integration/exception_handlers/exception_handlers_spec.rb
249
255
  - spec/integration/exponential_backoff/exponential_backoff_spec.rb
256
+ - spec/integration/fifo_ordering/fifo_max_messages_cap_spec.rb
250
257
  - spec/integration/fifo_ordering/fifo_ordering_spec.rb
251
258
  - spec/integration/large_payloads/large_payloads_spec.rb
259
+ - spec/integration/launcher/double_graceful_stop_spec.rb
260
+ - spec/integration/launcher/embedded_dispatch_error_spec.rb
261
+ - spec/integration/launcher/global_executor_preserved_spec.rb
262
+ - spec/integration/launcher/graceful_stop_timeout_spec.rb
252
263
  - spec/integration/launcher/launcher_spec.rb
253
264
  - spec/integration/message_attributes/message_attributes_spec.rb
254
265
  - spec/integration/message_operations/message_operations_spec.rb
266
+ - spec/integration/message_operations/partial_batch_delete_spec.rb
255
267
  - spec/integration/middleware_chain/empty_chain_spec.rb
256
268
  - spec/integration/middleware_chain/execution_order_spec.rb
257
269
  - spec/integration/middleware_chain/removal_spec.rb
258
270
  - spec/integration/middleware_chain/short_circuit_spec.rb
259
271
  - spec/integration/non_retryable_exception/non_retryable_exception_spec.rb
272
+ - spec/integration/non_retryable_exception/with_retry_intervals_spec.rb
260
273
  - spec/integration/polling_strategies/polling_strategies_spec.rb
261
274
  - spec/integration/queue_operations/queue_operations_spec.rb
262
275
  - spec/integration/rails/rails_72/Gemfile
@@ -343,7 +356,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
343
356
  - !ruby/object:Gem::Version
344
357
  version: '0'
345
358
  requirements: []
346
- rubygems_version: 4.0.3
359
+ rubygems_version: 4.0.10
347
360
  specification_version: 4
348
361
  summary: Shoryuken is a super efficient AWS SQS thread based message processor
349
362
  test_files:
@@ -352,13 +365,16 @@ test_files:
352
365
  - spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb
353
366
  - spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb
354
367
  - spec/integration/active_job/current_attributes/complex_types_spec.rb
368
+ - spec/integration/active_job/current_attributes/cross_job_reset_spec.rb
355
369
  - spec/integration/active_job/current_attributes/empty_context_spec.rb
356
370
  - spec/integration/active_job/current_attributes/full_context_spec.rb
371
+ - spec/integration/active_job/current_attributes/incremental_persist_spec.rb
357
372
  - spec/integration/active_job/current_attributes/partial_context_spec.rb
358
373
  - spec/integration/active_job/custom_attributes/number_attributes_spec.rb
359
374
  - spec/integration/active_job/custom_attributes/string_attributes_spec.rb
360
375
  - spec/integration/active_job/error_handling/job_wrapper_spec.rb
361
376
  - spec/integration/active_job/fifo_and_attributes/deduplication_spec.rb
377
+ - spec/integration/active_job/fifo_dedup_opt_out/fifo_dedup_opt_out_spec.rb
362
378
  - spec/integration/active_job/keyword_arguments/keyword_arguments_spec.rb
363
379
  - spec/integration/active_job/retry/discard_on_spec.rb
364
380
  - spec/integration/active_job/retry/retry_on_spec.rb
@@ -367,25 +383,35 @@ test_files:
367
383
  - spec/integration/active_record_middleware/active_record_middleware_spec.rb
368
384
  - spec/integration/auto_delete/auto_delete_spec.rb
369
385
  - spec/integration/auto_extend_visibility/auto_extend_visibility_spec.rb
386
+ - spec/integration/auto_extend_visibility/short_visibility_timeout_spec.rb
370
387
  - spec/integration/aws_config/aws_config_spec.rb
371
388
  - spec/integration/batch_processing/batch_processing_spec.rb
372
389
  - spec/integration/body_parser/json_parser_spec.rb
373
390
  - spec/integration/body_parser/proc_parser_spec.rb
374
391
  - spec/integration/body_parser/text_parser_spec.rb
375
392
  - spec/integration/concurrent_processing/concurrent_processing_spec.rb
393
+ - spec/integration/concurrent_processing/processor_accounting_spec.rb
394
+ - spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb
376
395
  - spec/integration/dead_letter_queue/dead_letter_queue_spec.rb
377
396
  - spec/integration/exception_handlers/exception_handlers_spec.rb
378
397
  - spec/integration/exponential_backoff/exponential_backoff_spec.rb
398
+ - spec/integration/fifo_ordering/fifo_max_messages_cap_spec.rb
379
399
  - spec/integration/fifo_ordering/fifo_ordering_spec.rb
380
400
  - spec/integration/large_payloads/large_payloads_spec.rb
401
+ - spec/integration/launcher/double_graceful_stop_spec.rb
402
+ - spec/integration/launcher/embedded_dispatch_error_spec.rb
403
+ - spec/integration/launcher/global_executor_preserved_spec.rb
404
+ - spec/integration/launcher/graceful_stop_timeout_spec.rb
381
405
  - spec/integration/launcher/launcher_spec.rb
382
406
  - spec/integration/message_attributes/message_attributes_spec.rb
383
407
  - spec/integration/message_operations/message_operations_spec.rb
408
+ - spec/integration/message_operations/partial_batch_delete_spec.rb
384
409
  - spec/integration/middleware_chain/empty_chain_spec.rb
385
410
  - spec/integration/middleware_chain/execution_order_spec.rb
386
411
  - spec/integration/middleware_chain/removal_spec.rb
387
412
  - spec/integration/middleware_chain/short_circuit_spec.rb
388
413
  - spec/integration/non_retryable_exception/non_retryable_exception_spec.rb
414
+ - spec/integration/non_retryable_exception/with_retry_intervals_spec.rb
389
415
  - spec/integration/polling_strategies/polling_strategies_spec.rb
390
416
  - spec/integration/queue_operations/queue_operations_spec.rb
391
417
  - spec/integration/rails/rails_72/Gemfile