shoryuken 7.0.0.alpha2 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/push.yml +2 -2
- data/.github/workflows/specs.yml +38 -43
- data/.github/workflows/verify-action-pins.yml +1 -1
- data/.gitignore +3 -0
- data/.rspec +1 -0
- data/.ruby-version +1 -1
- data/.yard-lint.yml +279 -0
- data/CHANGELOG.md +69 -1
- data/Gemfile +1 -1
- data/README.md +2 -7
- data/Rakefile +4 -10
- data/bin/clean_localstack +52 -0
- data/bin/cli/base.rb +21 -0
- data/bin/cli/sqs.rb +61 -2
- data/bin/integrations +275 -0
- data/bin/scenario +154 -0
- data/bin/shoryuken +1 -1
- data/lib/{shoryuken/extensions/active_job_extensions.rb → active_job/extensions.rb} +15 -4
- data/lib/active_job/queue_adapters/shoryuken_adapter.rb +208 -0
- data/lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter.rb +78 -0
- data/lib/shoryuken/active_job/current_attributes.rb +139 -0
- data/lib/shoryuken/active_job/job_wrapper.rb +28 -0
- data/lib/shoryuken/body_parser.rb +8 -0
- data/lib/shoryuken/client.rb +14 -0
- data/lib/shoryuken/default_exception_handler.rb +9 -0
- data/lib/shoryuken/default_worker_registry.rb +29 -1
- data/lib/shoryuken/environment_loader.rb +78 -8
- data/lib/shoryuken/errors.rb +33 -0
- data/lib/shoryuken/fetcher.rb +37 -1
- data/lib/shoryuken/helpers/atomic_boolean.rb +19 -5
- data/lib/shoryuken/helpers/timer_task.rb +80 -0
- data/lib/shoryuken/launcher.rb +53 -0
- data/lib/shoryuken/logging/base.rb +26 -0
- data/lib/shoryuken/logging/pretty.rb +25 -0
- data/lib/shoryuken/logging/without_timestamp.rb +25 -0
- data/lib/shoryuken/logging.rb +39 -25
- data/lib/shoryuken/manager.rb +70 -1
- data/lib/shoryuken/message.rb +114 -1
- data/lib/shoryuken/middleware/chain.rb +139 -43
- data/lib/shoryuken/middleware/entry.rb +30 -0
- data/lib/shoryuken/middleware/server/active_record.rb +8 -0
- data/lib/shoryuken/middleware/server/auto_delete.rb +10 -0
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +27 -1
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +29 -0
- data/lib/shoryuken/middleware/server/timing.rb +11 -0
- data/lib/shoryuken/options.rb +129 -6
- data/lib/shoryuken/polling/base_strategy.rb +1 -0
- data/lib/shoryuken/polling/strict_priority.rb +39 -0
- data/lib/shoryuken/polling/weighted_round_robin.rb +42 -0
- data/lib/shoryuken/processor.rb +32 -1
- data/lib/shoryuken/queue.rb +93 -4
- data/lib/shoryuken/runner.rb +45 -4
- data/lib/shoryuken/util.rb +26 -1
- data/lib/shoryuken/version.rb +2 -1
- data/lib/shoryuken/worker/default_executor.rb +21 -1
- data/lib/shoryuken/worker/inline_executor.rb +24 -0
- data/lib/shoryuken/worker.rb +193 -0
- data/lib/shoryuken/worker_registry.rb +33 -0
- data/lib/shoryuken.rb +18 -6
- data/renovate.json +29 -2
- data/shoryuken.gemspec +2 -1
- data/spec/integration/.rspec +1 -0
- data/spec/integration/active_job/adapter_configuration/configuration_spec.rb +26 -0
- data/spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb +53 -0
- data/spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb +50 -0
- data/spec/integration/active_job/current_attributes/complex_types_spec.rb +55 -0
- data/spec/integration/active_job/current_attributes/empty_context_spec.rb +41 -0
- data/spec/integration/active_job/current_attributes/full_context_spec.rb +63 -0
- data/spec/integration/active_job/current_attributes/partial_context_spec.rb +57 -0
- data/spec/integration/active_job/custom_attributes/number_attributes_spec.rb +37 -0
- data/spec/integration/active_job/custom_attributes/string_attributes_spec.rb +39 -0
- data/spec/integration/active_job/error_handling/job_wrapper_spec.rb +53 -0
- data/spec/integration/active_job/fifo_and_attributes/deduplication_spec.rb +86 -0
- data/spec/integration/active_job/retry/discard_on_spec.rb +43 -0
- data/spec/integration/active_job/retry/retry_on_spec.rb +36 -0
- data/spec/integration/active_job/roundtrip/roundtrip_spec.rb +52 -0
- data/spec/integration/active_job/scheduled/scheduled_spec.rb +76 -0
- data/spec/integration/active_record_middleware/active_record_middleware_spec.rb +84 -0
- data/spec/integration/auto_delete/auto_delete_spec.rb +53 -0
- data/spec/integration/auto_extend_visibility/auto_extend_visibility_spec.rb +57 -0
- data/spec/integration/aws_config/aws_config_spec.rb +59 -0
- data/spec/integration/batch_processing/batch_processing_spec.rb +37 -0
- data/spec/integration/body_parser/json_parser_spec.rb +45 -0
- data/spec/integration/body_parser/proc_parser_spec.rb +54 -0
- data/spec/integration/body_parser/text_parser_spec.rb +43 -0
- data/spec/integration/concurrent_processing/concurrent_processing_spec.rb +45 -0
- data/spec/integration/dead_letter_queue/dead_letter_queue_spec.rb +91 -0
- data/spec/integration/exception_handlers/exception_handlers_spec.rb +69 -0
- data/spec/integration/exponential_backoff/exponential_backoff_spec.rb +67 -0
- data/spec/integration/fifo_ordering/fifo_ordering_spec.rb +44 -0
- data/spec/integration/large_payloads/large_payloads_spec.rb +30 -0
- data/spec/integration/launcher/launcher_spec.rb +40 -0
- data/spec/integration/message_attributes/message_attributes_spec.rb +54 -0
- data/spec/integration/message_operations/message_operations_spec.rb +59 -0
- data/spec/integration/middleware_chain/empty_chain_spec.rb +11 -0
- data/spec/integration/middleware_chain/execution_order_spec.rb +33 -0
- data/spec/integration/middleware_chain/removal_spec.rb +31 -0
- data/spec/integration/middleware_chain/short_circuit_spec.rb +40 -0
- data/spec/integration/polling_strategies/polling_strategies_spec.rb +46 -0
- data/spec/integration/queue_operations/queue_operations_spec.rb +84 -0
- data/spec/integration/rails/rails_72/Gemfile +6 -0
- data/spec/integration/rails/rails_72/activejob_adapter_spec.rb +98 -0
- data/spec/integration/rails/rails_80/Gemfile +6 -0
- data/spec/integration/rails/rails_80/activejob_adapter_spec.rb +98 -0
- data/spec/integration/rails/rails_80/continuation_spec.rb +79 -0
- data/spec/integration/rails/rails_81/Gemfile +6 -0
- data/spec/integration/rails/rails_81/activejob_adapter_spec.rb +98 -0
- data/spec/integration/rails/rails_81/continuation_spec.rb +79 -0
- data/spec/integration/retry_behavior/retry_behavior_spec.rb +45 -0
- data/spec/integration/spec_helper.rb +7 -0
- data/spec/integration/strict_priority_polling/strict_priority_polling_spec.rb +58 -0
- data/spec/integration/visibility_timeout/visibility_timeout_spec.rb +37 -0
- data/spec/integration/worker_enqueueing/worker_enqueueing_spec.rb +60 -0
- data/spec/integration/worker_groups/worker_groups_spec.rb +79 -0
- data/spec/integration/worker_lifecycle/worker_lifecycle_spec.rb +33 -0
- data/spec/integrations_helper.rb +243 -0
- data/spec/lib/active_job/extensions_spec.rb +149 -0
- data/spec/lib/active_job/queue_adapters/shoryuken_adapter_spec.rb +29 -0
- data/spec/{shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb → lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter_spec.rb} +3 -3
- data/spec/{shoryuken/extensions/active_job_wrapper_spec.rb → lib/shoryuken/active_job/job_wrapper_spec.rb} +4 -4
- data/spec/{shoryuken → lib/shoryuken}/environment_loader_spec.rb +1 -1
- data/spec/{shoryuken → lib/shoryuken}/helpers/hash_utils_spec.rb +14 -14
- data/spec/{shoryuken → lib/shoryuken}/helpers/string_utils_spec.rb +3 -3
- data/spec/lib/shoryuken/helpers/timer_task_spec.rb +298 -0
- data/spec/{shoryuken → lib/shoryuken}/helpers_integration_spec.rb +9 -9
- data/spec/{shoryuken → lib/shoryuken}/launcher_spec.rb +22 -0
- data/spec/lib/shoryuken/logging_spec.rb +242 -0
- data/spec/lib/shoryuken/message_spec.rb +109 -0
- data/spec/lib/shoryuken/middleware/entry_spec.rb +68 -0
- data/spec/lib/shoryuken/middleware/server/active_record_spec.rb +133 -0
- data/spec/{shoryuken → lib/shoryuken}/middleware/server/auto_extend_visibility_spec.rb +50 -0
- data/spec/{shoryuken → lib/shoryuken}/options_spec.rb +2 -2
- data/spec/{shoryuken → lib/shoryuken}/util_spec.rb +1 -1
- data/spec/lib/shoryuken/version_spec.rb +17 -0
- data/spec/lib/shoryuken/worker_registry_spec.rb +63 -0
- data/spec/shared_examples_for_active_job.rb +29 -9
- data/spec/spec_helper.rb +34 -3
- metadata +230 -91
- data/.devcontainer/Dockerfile +0 -17
- data/.devcontainer/base.Dockerfile +0 -43
- data/.devcontainer/devcontainer.json +0 -35
- data/Appraisals +0 -23
- data/gemfiles/.gitignore +0 -1
- data/gemfiles/rails_7_0.gemfile +0 -19
- data/gemfiles/rails_7_1.gemfile +0 -19
- data/gemfiles/rails_7_2.gemfile +0 -19
- data/gemfiles/rails_8_0.gemfile +0 -19
- data/lib/shoryuken/extensions/active_job_adapter.rb +0 -110
- data/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb +0 -50
- data/spec/integration/launcher_spec.rb +0 -127
- data/spec/shoryuken/extensions/active_job_adapter_spec.rb +0 -8
- data/spec/shoryuken/extensions/active_job_base_spec.rb +0 -85
- /data/spec/{shoryuken → lib/shoryuken}/body_parser_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/client_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/default_exception_handler_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/default_worker_registry_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/fetcher_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/helpers/atomic_boolean_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/helpers/atomic_counter_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/helpers/atomic_hash_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/inline_message_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/manager_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/middleware/chain_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/middleware/server/auto_delete_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/middleware/server/exponential_backoff_retry_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/middleware/server/timing_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/polling/base_strategy_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/polling/queue_configuration_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/polling/strict_priority_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/polling/weighted_round_robin_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/processor_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/queue_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/runner_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/worker/default_executor_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/worker/inline_executor_spec.rb +0 -0
- /data/spec/{shoryuken → lib/shoryuken}/worker_spec.rb +0 -0
- /data/spec/{shoryuken_spec.rb → lib/shoryuken_spec.rb} +0 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests the auto_visibility_timeout middleware functionality.
|
|
4
|
+
# When auto_visibility_timeout: true, the message visibility timeout should be
|
|
5
|
+
# automatically extended during long-running job processing to prevent re-delivery.
|
|
6
|
+
|
|
7
|
+
setup_localstack
|
|
8
|
+
|
|
9
|
+
queue_name = DT.uuid
|
|
10
|
+
|
|
11
|
+
# Create queue with short visibility timeout (10 seconds)
|
|
12
|
+
create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '10' })
|
|
13
|
+
Shoryuken.add_group('default', 1)
|
|
14
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
15
|
+
|
|
16
|
+
# Worker with auto_visibility_timeout enabled that takes longer than visibility timeout
|
|
17
|
+
auto_visibility_worker = Class.new do
|
|
18
|
+
include Shoryuken::Worker
|
|
19
|
+
|
|
20
|
+
shoryuken_options auto_visibility_timeout: true, auto_delete: true
|
|
21
|
+
|
|
22
|
+
def perform(sqs_msg, body)
|
|
23
|
+
DT[:processing_started] << Time.now
|
|
24
|
+
# Sleep longer than the queue's visibility timeout (10s)
|
|
25
|
+
# The middleware should extend visibility before it expires
|
|
26
|
+
sleep 12
|
|
27
|
+
DT[:processing_completed] << Time.now
|
|
28
|
+
DT[:processed_messages] << { message_id: sqs_msg.message_id, body: body }
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
auto_visibility_worker.get_shoryuken_options['queue'] = queue_name
|
|
33
|
+
Shoryuken.register_worker(queue_name, auto_visibility_worker)
|
|
34
|
+
|
|
35
|
+
queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
|
|
36
|
+
|
|
37
|
+
# Send a message
|
|
38
|
+
Shoryuken::Client.sqs.send_message(
|
|
39
|
+
queue_url: queue_url,
|
|
40
|
+
message_body: 'long running job'
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
sleep 1
|
|
44
|
+
|
|
45
|
+
# Process the message - this should take ~12 seconds but not fail
|
|
46
|
+
poll_queues_until(timeout: 30) { DT[:processed_messages].size >= 1 }
|
|
47
|
+
|
|
48
|
+
# Verify message was processed exactly once (visibility was extended, not re-delivered)
|
|
49
|
+
assert_equal(1, DT[:processed_messages].size, "Message should be processed exactly once")
|
|
50
|
+
assert_equal('long running job', DT[:processed_messages].first[:body])
|
|
51
|
+
|
|
52
|
+
# Verify processing took longer than the visibility timeout
|
|
53
|
+
processing_time = DT[:processing_completed].first - DT[:processing_started].first
|
|
54
|
+
assert(processing_time >= 12, "Processing should have taken at least 12 seconds")
|
|
55
|
+
|
|
56
|
+
# Cleanup
|
|
57
|
+
delete_test_queue(queue_name)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests that AWS configuration from Shoryuken.options[:aws]
|
|
4
|
+
# is properly passed to the SQS client initialization.
|
|
5
|
+
# This verifies the fix for issue #815: PORO setup does not load AWS config
|
|
6
|
+
|
|
7
|
+
# Reset any cached SQS client to ensure fresh initialization
|
|
8
|
+
Shoryuken.sqs_client = nil
|
|
9
|
+
|
|
10
|
+
# Configure AWS options programmatically (simulating PORO setup with config file)
|
|
11
|
+
Shoryuken.options[:aws] = {
|
|
12
|
+
region: 'us-east-1',
|
|
13
|
+
endpoint: 'http://localhost:4566',
|
|
14
|
+
access_key_id: 'test-key-from-config',
|
|
15
|
+
secret_access_key: 'test-secret-from-config'
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
# Get the SQS client - this should use the AWS config from options
|
|
19
|
+
client = Shoryuken.sqs_client
|
|
20
|
+
|
|
21
|
+
# Verify the client was configured with our options
|
|
22
|
+
config = client.config
|
|
23
|
+
|
|
24
|
+
assert_equal('us-east-1', config.region, "Region should be set from options[:aws]")
|
|
25
|
+
assert_equal('http://localhost:4566', config.endpoint.to_s, "Endpoint should be set from options[:aws]")
|
|
26
|
+
|
|
27
|
+
# Verify the client actually works by creating a queue
|
|
28
|
+
queue_name = "aws-config-test-#{SecureRandom.hex(6)}"
|
|
29
|
+
|
|
30
|
+
begin
|
|
31
|
+
result = client.create_queue(queue_name: queue_name)
|
|
32
|
+
assert(result.queue_url.include?(queue_name), "Should be able to create queue with configured client")
|
|
33
|
+
|
|
34
|
+
# Clean up
|
|
35
|
+
client.delete_queue(queue_url: result.queue_url)
|
|
36
|
+
rescue Aws::SQS::Errors::ServiceError => e
|
|
37
|
+
raise "SQS client should work with configured AWS options: #{e.message}"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Test 2: Verify that reconfiguring options and resetting client works
|
|
41
|
+
Shoryuken.sqs_client = nil
|
|
42
|
+
Shoryuken.options[:aws][:region] = 'us-west-2'
|
|
43
|
+
|
|
44
|
+
client2 = Shoryuken.sqs_client
|
|
45
|
+
assert_equal('us-west-2', client2.config.region, "New client should use updated region")
|
|
46
|
+
|
|
47
|
+
# Test 3: Verify that credentials from options are used
|
|
48
|
+
# Reset and reconfigure with explicit credentials
|
|
49
|
+
Shoryuken.sqs_client = nil
|
|
50
|
+
Shoryuken.options[:aws] = {
|
|
51
|
+
region: 'us-east-1',
|
|
52
|
+
endpoint: 'http://localhost:4566',
|
|
53
|
+
access_key_id: 'another-key',
|
|
54
|
+
secret_access_key: 'another-secret'
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
client3 = Shoryuken.sqs_client
|
|
58
|
+
assert(client3.is_a?(Aws::SQS::Client), "Client should be created with new credentials")
|
|
59
|
+
assert_equal('us-east-1', client3.config.region, "Region should match configured value")
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests batch processing including batch message reception (up to 10
|
|
4
|
+
# messages), batch vs single worker behavior differences, JSON body parsing in
|
|
5
|
+
# batch mode, and maximum batch size handling.
|
|
6
|
+
|
|
7
|
+
setup_localstack
|
|
8
|
+
|
|
9
|
+
queue_name = DT.queue
|
|
10
|
+
create_test_queue(queue_name)
|
|
11
|
+
Shoryuken.add_group('default', 1)
|
|
12
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
13
|
+
|
|
14
|
+
worker_class = Class.new do
|
|
15
|
+
include Shoryuken::Worker
|
|
16
|
+
|
|
17
|
+
def perform(sqs_msgs, bodies)
|
|
18
|
+
msgs = Array(sqs_msgs)
|
|
19
|
+
DT[:batch_sizes] << msgs.size
|
|
20
|
+
DT[:messages].concat(Array(bodies))
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
worker_class.get_shoryuken_options['queue'] = queue_name
|
|
25
|
+
worker_class.get_shoryuken_options['auto_delete'] = true
|
|
26
|
+
worker_class.get_shoryuken_options['batch'] = true
|
|
27
|
+
Shoryuken.register_worker(queue_name, worker_class)
|
|
28
|
+
|
|
29
|
+
entries = 5.times.map { |i| { id: SecureRandom.uuid, message_body: "message-#{i}" } }
|
|
30
|
+
Shoryuken::Client.queues(queue_name).send_messages(entries: entries)
|
|
31
|
+
|
|
32
|
+
sleep 1
|
|
33
|
+
|
|
34
|
+
poll_queues_until { DT[:messages].size >= 5 }
|
|
35
|
+
|
|
36
|
+
assert_equal(5, DT[:messages].size)
|
|
37
|
+
assert(DT[:batch_sizes].any? { |size| size > 1 }, "Expected at least one batch with size > 1")
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests the body_parser option with :json setting
|
|
4
|
+
# Verifies that JSON messages are automatically parsed into Ruby hashes
|
|
5
|
+
|
|
6
|
+
setup_localstack
|
|
7
|
+
|
|
8
|
+
queue_name = DT.uuid
|
|
9
|
+
create_test_queue(queue_name)
|
|
10
|
+
Shoryuken.add_group('default', 1)
|
|
11
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
12
|
+
|
|
13
|
+
worker_class = Class.new do
|
|
14
|
+
include Shoryuken::Worker
|
|
15
|
+
|
|
16
|
+
shoryuken_options body_parser: :json
|
|
17
|
+
|
|
18
|
+
def perform(sqs_msg, body)
|
|
19
|
+
DT[:parsed_bodies] << body
|
|
20
|
+
DT[:body_classes] << body.class.name
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
worker_class.get_shoryuken_options['queue'] = queue_name
|
|
25
|
+
worker_class.get_shoryuken_options['auto_delete'] = true
|
|
26
|
+
Shoryuken.register_worker(queue_name, worker_class)
|
|
27
|
+
|
|
28
|
+
queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
|
|
29
|
+
|
|
30
|
+
# Send a JSON message
|
|
31
|
+
json_body = { 'name' => 'test', 'value' => 42, 'nested' => { 'key' => 'val' } }
|
|
32
|
+
Shoryuken::Client.sqs.send_message(
|
|
33
|
+
queue_url: queue_url,
|
|
34
|
+
message_body: JSON.dump(json_body)
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
sleep 1
|
|
38
|
+
|
|
39
|
+
poll_queues_until { DT[:parsed_bodies].size >= 1 }
|
|
40
|
+
|
|
41
|
+
assert_equal(1, DT[:parsed_bodies].size)
|
|
42
|
+
assert_equal('Hash', DT[:body_classes].first)
|
|
43
|
+
assert_equal('test', DT[:parsed_bodies].first['name'])
|
|
44
|
+
assert_equal(42, DT[:parsed_bodies].first['value'])
|
|
45
|
+
assert_equal('val', DT[:parsed_bodies].first['nested']['key'])
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests the body_parser option with a custom Proc
|
|
4
|
+
# Verifies that custom parsing logic can be applied to messages
|
|
5
|
+
|
|
6
|
+
setup_localstack
|
|
7
|
+
|
|
8
|
+
queue_name = DT.uuid
|
|
9
|
+
create_test_queue(queue_name)
|
|
10
|
+
Shoryuken.add_group('default', 1)
|
|
11
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
12
|
+
|
|
13
|
+
# Custom parser that uppercases the body and adds metadata
|
|
14
|
+
custom_parser = proc do |sqs_msg|
|
|
15
|
+
{
|
|
16
|
+
original: sqs_msg.body,
|
|
17
|
+
transformed: sqs_msg.body.upcase,
|
|
18
|
+
message_id: sqs_msg.message_id
|
|
19
|
+
}
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
worker_class = Class.new do
|
|
23
|
+
include Shoryuken::Worker
|
|
24
|
+
|
|
25
|
+
def perform(sqs_msg, body)
|
|
26
|
+
DT[:parsed_bodies] << body
|
|
27
|
+
DT[:body_classes] << body.class.name
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
worker_class.get_shoryuken_options['queue'] = queue_name
|
|
32
|
+
worker_class.get_shoryuken_options['auto_delete'] = true
|
|
33
|
+
worker_class.get_shoryuken_options['body_parser'] = custom_parser
|
|
34
|
+
Shoryuken.register_worker(queue_name, worker_class)
|
|
35
|
+
|
|
36
|
+
queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
|
|
37
|
+
|
|
38
|
+
# Send a message to be processed by custom parser
|
|
39
|
+
Shoryuken::Client.sqs.send_message(
|
|
40
|
+
queue_url: queue_url,
|
|
41
|
+
message_body: 'hello world'
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
sleep 1
|
|
45
|
+
|
|
46
|
+
poll_queues_until { DT[:parsed_bodies].size >= 1 }
|
|
47
|
+
|
|
48
|
+
assert_equal(1, DT[:parsed_bodies].size)
|
|
49
|
+
assert_equal('Hash', DT[:body_classes].first)
|
|
50
|
+
|
|
51
|
+
parsed = DT[:parsed_bodies].first
|
|
52
|
+
assert_equal('hello world', parsed[:original])
|
|
53
|
+
assert_equal('HELLO WORLD', parsed[:transformed])
|
|
54
|
+
assert(parsed[:message_id], "Should include message_id from sqs_msg")
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests the body_parser option with :text setting (default)
|
|
4
|
+
# Verifies that messages are returned as plain strings
|
|
5
|
+
|
|
6
|
+
setup_localstack
|
|
7
|
+
|
|
8
|
+
queue_name = DT.uuid
|
|
9
|
+
create_test_queue(queue_name)
|
|
10
|
+
Shoryuken.add_group('default', 1)
|
|
11
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
12
|
+
|
|
13
|
+
worker_class = Class.new do
|
|
14
|
+
include Shoryuken::Worker
|
|
15
|
+
|
|
16
|
+
shoryuken_options body_parser: :text
|
|
17
|
+
|
|
18
|
+
def perform(sqs_msg, body)
|
|
19
|
+
DT[:parsed_bodies] << body
|
|
20
|
+
DT[:body_classes] << body.class.name
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
worker_class.get_shoryuken_options['queue'] = queue_name
|
|
25
|
+
worker_class.get_shoryuken_options['auto_delete'] = true
|
|
26
|
+
Shoryuken.register_worker(queue_name, worker_class)
|
|
27
|
+
|
|
28
|
+
queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
|
|
29
|
+
|
|
30
|
+
# Send a plain text message
|
|
31
|
+
text_body = 'Hello, this is a plain text message!'
|
|
32
|
+
Shoryuken::Client.sqs.send_message(
|
|
33
|
+
queue_url: queue_url,
|
|
34
|
+
message_body: text_body
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
sleep 1
|
|
38
|
+
|
|
39
|
+
poll_queues_until { DT[:parsed_bodies].size >= 1 }
|
|
40
|
+
|
|
41
|
+
assert_equal(1, DT[:parsed_bodies].size)
|
|
42
|
+
assert_equal('String', DT[:body_classes].first)
|
|
43
|
+
assert_equal(text_body, DT[:parsed_bodies].first)
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests concurrent message processing with multiple processors.
|
|
4
|
+
|
|
5
|
+
require 'concurrent'
|
|
6
|
+
|
|
7
|
+
setup_localstack
|
|
8
|
+
|
|
9
|
+
queue_name = DT.queue
|
|
10
|
+
create_test_queue(queue_name)
|
|
11
|
+
Shoryuken.add_group('concurrent', 5) # 5 concurrent processors
|
|
12
|
+
Shoryuken.add_queue(queue_name, 1, 'concurrent')
|
|
13
|
+
|
|
14
|
+
# Atomic counters for tracking concurrency
|
|
15
|
+
concurrent_count = Concurrent::AtomicFixnum.new(0)
|
|
16
|
+
max_concurrent = Concurrent::AtomicFixnum.new(0)
|
|
17
|
+
|
|
18
|
+
worker_class = Class.new do
|
|
19
|
+
include Shoryuken::Worker
|
|
20
|
+
|
|
21
|
+
shoryuken_options auto_delete: true, batch: false
|
|
22
|
+
|
|
23
|
+
define_method(:perform) do |sqs_msg, body|
|
|
24
|
+
concurrent_count.increment
|
|
25
|
+
current = concurrent_count.value
|
|
26
|
+
max_concurrent.update { |max| [max, current].max }
|
|
27
|
+
|
|
28
|
+
sleep 0.5 # Simulate work
|
|
29
|
+
|
|
30
|
+
DT[:processing_times] << Time.now
|
|
31
|
+
|
|
32
|
+
concurrent_count.decrement
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
worker_class.get_shoryuken_options['queue'] = queue_name
|
|
37
|
+
Shoryuken.register_worker(queue_name, worker_class)
|
|
38
|
+
|
|
39
|
+
10.times { |i| Shoryuken::Client.queues(queue_name).send_message(message_body: "msg-#{i}") }
|
|
40
|
+
|
|
41
|
+
poll_queues_until(timeout: 20) { DT[:processing_times].size >= 10 }
|
|
42
|
+
|
|
43
|
+
assert_equal(10, DT[:processing_times].size)
|
|
44
|
+
# With multiple processors, we should see concurrency > 1
|
|
45
|
+
assert(max_concurrent.value > 1, "Expected concurrency > 1, got #{max_concurrent.value}")
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests dead letter queue (DLQ) functionality.
|
|
4
|
+
# When a message exceeds maxReceiveCount, it should be moved to the DLQ.
|
|
5
|
+
# Note: This test doesn't use poll_queues_until because messages should fail
|
|
6
|
+
# and be moved to DLQ rather than being successfully processed.
|
|
7
|
+
|
|
8
|
+
setup_localstack
|
|
9
|
+
|
|
10
|
+
main_queue_name = DT.queues[0]
|
|
11
|
+
dlq_name = DT.queues[1]
|
|
12
|
+
|
|
13
|
+
# Create the dead letter queue first
|
|
14
|
+
create_test_queue(dlq_name)
|
|
15
|
+
|
|
16
|
+
dlq_url = Shoryuken::Client.sqs.get_queue_url(queue_name: dlq_name).queue_url
|
|
17
|
+
dlq_arn = Shoryuken::Client.sqs.get_queue_attributes(
|
|
18
|
+
queue_url: dlq_url,
|
|
19
|
+
attribute_names: ['QueueArn']
|
|
20
|
+
).attributes['QueueArn']
|
|
21
|
+
|
|
22
|
+
# Create main queue with redrive policy - move to DLQ after 2 receives
|
|
23
|
+
redrive_policy = { maxReceiveCount: 2, deadLetterTargetArn: dlq_arn }.to_json
|
|
24
|
+
create_test_queue(main_queue_name, attributes: {
|
|
25
|
+
'VisibilityTimeout' => '1',
|
|
26
|
+
'RedrivePolicy' => redrive_policy
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
main_queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: main_queue_name).queue_url
|
|
30
|
+
|
|
31
|
+
# Send a message
|
|
32
|
+
Shoryuken::Client.sqs.send_message(
|
|
33
|
+
queue_url: main_queue_url,
|
|
34
|
+
message_body: 'dlq test message'
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
# Manually receive the message multiple times to trigger DLQ
|
|
38
|
+
# maxReceiveCount = 2, so after 2 receives without deletion, it goes to DLQ
|
|
39
|
+
3.times do |i|
|
|
40
|
+
msgs = Shoryuken::Client.sqs.receive_message(
|
|
41
|
+
queue_url: main_queue_url,
|
|
42
|
+
max_number_of_messages: 1,
|
|
43
|
+
wait_time_seconds: 3,
|
|
44
|
+
attribute_names: ['ApproximateReceiveCount']
|
|
45
|
+
).messages
|
|
46
|
+
|
|
47
|
+
if msgs.any?
|
|
48
|
+
receive_count = msgs.first.attributes['ApproximateReceiveCount'].to_i
|
|
49
|
+
DT[:receives] << { attempt: i + 1, receive_count: receive_count }
|
|
50
|
+
# Don't delete - let visibility timeout expire
|
|
51
|
+
sleep 2
|
|
52
|
+
else
|
|
53
|
+
DT[:receives] << { attempt: i + 1, no_message: true }
|
|
54
|
+
break
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Verify message was received at least twice
|
|
59
|
+
actual_receives = DT[:receives].reject { |r| r[:no_message] }
|
|
60
|
+
assert(actual_receives.size >= 2, "Message should have been received at least twice (was #{actual_receives.size})")
|
|
61
|
+
|
|
62
|
+
# Wait for message to be moved to DLQ
|
|
63
|
+
sleep 3
|
|
64
|
+
|
|
65
|
+
# Check that message is now in the DLQ
|
|
66
|
+
dlq_messages = Shoryuken::Client.sqs.receive_message(
|
|
67
|
+
queue_url: dlq_url,
|
|
68
|
+
max_number_of_messages: 10,
|
|
69
|
+
wait_time_seconds: 5,
|
|
70
|
+
attribute_names: ['All']
|
|
71
|
+
).messages
|
|
72
|
+
|
|
73
|
+
assert(dlq_messages.size >= 1, 'Message should have been moved to DLQ')
|
|
74
|
+
assert_equal('dlq test message', dlq_messages.first.body)
|
|
75
|
+
|
|
76
|
+
# Verify message is no longer in main queue
|
|
77
|
+
main_attrs = Shoryuken::Client.sqs.get_queue_attributes(
|
|
78
|
+
queue_url: main_queue_url,
|
|
79
|
+
attribute_names: %w[ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible]
|
|
80
|
+
).attributes
|
|
81
|
+
main_count = main_attrs['ApproximateNumberOfMessages'].to_i +
|
|
82
|
+
main_attrs['ApproximateNumberOfMessagesNotVisible'].to_i
|
|
83
|
+
assert_equal(0, main_count, 'Main queue should be empty after DLQ move')
|
|
84
|
+
|
|
85
|
+
# Clean up DLQ message
|
|
86
|
+
dlq_messages.each do |msg|
|
|
87
|
+
Shoryuken::Client.sqs.delete_message(
|
|
88
|
+
queue_url: dlq_url,
|
|
89
|
+
receipt_handle: msg.receipt_handle
|
|
90
|
+
)
|
|
91
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests custom exception handlers.
|
|
4
|
+
# Exception handlers are called when a worker raises an error.
|
|
5
|
+
|
|
6
|
+
setup_localstack
|
|
7
|
+
|
|
8
|
+
queue_name = DT.queue
|
|
9
|
+
create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' })
|
|
10
|
+
Shoryuken.add_group('default', 1)
|
|
11
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
12
|
+
|
|
13
|
+
# Save original handlers to restore later
|
|
14
|
+
original_handlers = Shoryuken.exception_handlers.dup
|
|
15
|
+
|
|
16
|
+
# Custom exception handler that records exceptions
|
|
17
|
+
custom_handler = Object.new
|
|
18
|
+
custom_handler.define_singleton_method(:call) do |exception, queue, sqs_msg|
|
|
19
|
+
DT[:exceptions] << {
|
|
20
|
+
message: exception.message,
|
|
21
|
+
queue: queue,
|
|
22
|
+
message_id: sqs_msg.message_id
|
|
23
|
+
}
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Add custom handler alongside default
|
|
27
|
+
Shoryuken.exception_handlers << custom_handler
|
|
28
|
+
|
|
29
|
+
worker_class = Class.new do
|
|
30
|
+
include Shoryuken::Worker
|
|
31
|
+
|
|
32
|
+
shoryuken_options auto_delete: false, batch: false
|
|
33
|
+
|
|
34
|
+
def perform(sqs_msg, body)
|
|
35
|
+
receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
|
|
36
|
+
DT[:attempts] << receive_count
|
|
37
|
+
|
|
38
|
+
if receive_count < 2
|
|
39
|
+
raise 'Intentional failure for testing'
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Succeed on second attempt and delete
|
|
43
|
+
DT[:success] << body
|
|
44
|
+
sqs_msg.delete
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
worker_class.get_shoryuken_options['queue'] = queue_name
|
|
49
|
+
Shoryuken.register_worker(queue_name, worker_class)
|
|
50
|
+
|
|
51
|
+
Shoryuken::Client.queues(queue_name).send_message(message_body: 'exception test')
|
|
52
|
+
|
|
53
|
+
sleep 1
|
|
54
|
+
|
|
55
|
+
begin
|
|
56
|
+
poll_queues_until(timeout: 15) { DT[:success].size >= 1 }
|
|
57
|
+
|
|
58
|
+
# Verify exception handler was called on first attempt
|
|
59
|
+
assert(DT[:exceptions].size >= 1, 'Exception handler should have been called')
|
|
60
|
+
assert_equal('Intentional failure for testing', DT[:exceptions].first[:message])
|
|
61
|
+
assert_equal(queue_name, DT[:exceptions].first[:queue])
|
|
62
|
+
|
|
63
|
+
# Verify message was eventually processed successfully
|
|
64
|
+
assert_equal(1, DT[:success].size)
|
|
65
|
+
assert_equal('exception test', DT[:success].first)
|
|
66
|
+
ensure
|
|
67
|
+
# Restore original handlers to prevent cross-test interference
|
|
68
|
+
Shoryuken.exception_handlers.replace(original_handlers)
|
|
69
|
+
end
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests the exponential_backoff_retry middleware functionality.
|
|
4
|
+
# When retry_intervals is configured, failed jobs should have their visibility
|
|
5
|
+
# timeout adjusted based on the retry attempt number.
|
|
6
|
+
|
|
7
|
+
setup_localstack
|
|
8
|
+
|
|
9
|
+
queue_name = DT.queue
|
|
10
|
+
|
|
11
|
+
# Create queue with short visibility timeout
|
|
12
|
+
create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' })
|
|
13
|
+
Shoryuken.add_group('default', 1)
|
|
14
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
15
|
+
|
|
16
|
+
# Worker that fails on first attempts, succeeds after retry_intervals exhausted
|
|
17
|
+
backoff_worker = Class.new do
|
|
18
|
+
include Shoryuken::Worker
|
|
19
|
+
|
|
20
|
+
# Retry after 1 second, then 2 seconds
|
|
21
|
+
shoryuken_options retry_intervals: [1, 2], auto_delete: true
|
|
22
|
+
|
|
23
|
+
def perform(sqs_msg, body)
|
|
24
|
+
receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
|
|
25
|
+
DT[:attempts] << { receive_count: receive_count, time: Time.now }
|
|
26
|
+
|
|
27
|
+
# Fail on first 2 attempts, succeed on 3rd
|
|
28
|
+
if receive_count < 3
|
|
29
|
+
raise "Simulated failure on attempt #{receive_count}"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
DT[:successful_processing] << { body: body, final_receive_count: receive_count }
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
backoff_worker.get_shoryuken_options['queue'] = queue_name
|
|
37
|
+
Shoryuken.register_worker(queue_name, backoff_worker)
|
|
38
|
+
|
|
39
|
+
queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
|
|
40
|
+
|
|
41
|
+
# Send a message
|
|
42
|
+
Shoryuken::Client.sqs.send_message(
|
|
43
|
+
queue_url: queue_url,
|
|
44
|
+
message_body: 'backoff test'
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
sleep 1
|
|
48
|
+
|
|
49
|
+
# Process - should fail twice then succeed on 3rd attempt
|
|
50
|
+
# Total time: ~1s (first retry) + ~2s (second retry) = ~3s minimum
|
|
51
|
+
poll_queues_until(timeout: 20) { DT[:successful_processing].size >= 1 }
|
|
52
|
+
|
|
53
|
+
# Verify the message was eventually processed successfully
|
|
54
|
+
assert_equal(1, DT[:successful_processing].size)
|
|
55
|
+
assert_equal('backoff test', DT[:successful_processing].first[:body])
|
|
56
|
+
assert_equal(3, DT[:successful_processing].first[:final_receive_count])
|
|
57
|
+
|
|
58
|
+
# Verify we had 3 attempts total
|
|
59
|
+
assert_equal(3, DT[:attempts].size, 'Should have 3 attempts total')
|
|
60
|
+
|
|
61
|
+
# Verify backoff timing - second attempt should be ~1s after first
|
|
62
|
+
first_to_second = DT[:attempts][1][:time] - DT[:attempts][0][:time]
|
|
63
|
+
assert(first_to_second >= 0.5, "Second attempt should be at least 0.5s after first (was #{first_to_second}s)")
|
|
64
|
+
|
|
65
|
+
# Verify backoff timing - third attempt should be ~2s after second
|
|
66
|
+
second_to_third = DT[:attempts][2][:time] - DT[:attempts][1][:time]
|
|
67
|
+
assert(second_to_third >= 1.0, "Third attempt should be at least 1s after second (was #{second_to_third}s)")
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests FIFO queue ordering guarantees including message ordering
|
|
4
|
+
# within the same message group.
|
|
5
|
+
|
|
6
|
+
setup_localstack
|
|
7
|
+
|
|
8
|
+
queue_name = "#{DT.uuid}.fifo"
|
|
9
|
+
create_fifo_queue(queue_name)
|
|
10
|
+
Shoryuken.add_group('default', 1)
|
|
11
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
12
|
+
|
|
13
|
+
worker_class = Class.new do
|
|
14
|
+
include Shoryuken::Worker
|
|
15
|
+
|
|
16
|
+
def perform(sqs_msg, body)
|
|
17
|
+
DT[:messages] << body
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
worker_class.get_shoryuken_options['queue'] = queue_name
|
|
22
|
+
worker_class.get_shoryuken_options['auto_delete'] = true
|
|
23
|
+
worker_class.get_shoryuken_options['batch'] = false
|
|
24
|
+
Shoryuken.register_worker(queue_name, worker_class)
|
|
25
|
+
|
|
26
|
+
queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
|
|
27
|
+
|
|
28
|
+
5.times do |i|
|
|
29
|
+
Shoryuken::Client.sqs.send_message(
|
|
30
|
+
queue_url: queue_url,
|
|
31
|
+
message_body: "msg-#{i}",
|
|
32
|
+
message_group_id: 'group-a',
|
|
33
|
+
message_deduplication_id: SecureRandom.uuid
|
|
34
|
+
)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
sleep 1
|
|
38
|
+
|
|
39
|
+
poll_queues_until { DT[:messages].size >= 5 }
|
|
40
|
+
|
|
41
|
+
assert_equal(5, DT[:messages].size)
|
|
42
|
+
|
|
43
|
+
expected = (0..4).map { |i| "msg-#{i}" }
|
|
44
|
+
assert_equal(expected, DT[:messages])
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests large payload handling including payloads near the 256KB SQS limit.
|
|
4
|
+
|
|
5
|
+
setup_localstack
|
|
6
|
+
|
|
7
|
+
queue_name = DT.queue
|
|
8
|
+
create_test_queue(queue_name)
|
|
9
|
+
Shoryuken.add_group('default', 1)
|
|
10
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
11
|
+
|
|
12
|
+
worker_class = Class.new do
|
|
13
|
+
include Shoryuken::Worker
|
|
14
|
+
|
|
15
|
+
def perform(sqs_msg, body)
|
|
16
|
+
DT[:bodies] << body
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
worker_class.get_shoryuken_options['queue'] = queue_name
|
|
21
|
+
worker_class.get_shoryuken_options['auto_delete'] = true
|
|
22
|
+
worker_class.get_shoryuken_options['batch'] = false
|
|
23
|
+
Shoryuken.register_worker(queue_name, worker_class)
|
|
24
|
+
|
|
25
|
+
payload = 'x' * (250 * 1024)
|
|
26
|
+
Shoryuken::Client.queues(queue_name).send_message(message_body: payload)
|
|
27
|
+
|
|
28
|
+
poll_queues_until { DT[:bodies].size >= 1 }
|
|
29
|
+
|
|
30
|
+
assert_equal(250 * 1024, DT[:bodies].first.size)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests the Launcher's ability to consume messages from SQS queues,
|
|
4
|
+
# including single message consumption, batch consumption, and command workers.
|
|
5
|
+
|
|
6
|
+
require 'concurrent'
|
|
7
|
+
|
|
8
|
+
setup_localstack
|
|
9
|
+
|
|
10
|
+
# Use atomic counter for thread-safe message counting
|
|
11
|
+
message_counter = Concurrent::AtomicFixnum.new(0)
|
|
12
|
+
|
|
13
|
+
worker_class = Class.new do
|
|
14
|
+
include Shoryuken::Worker
|
|
15
|
+
|
|
16
|
+
shoryuken_options auto_delete: true
|
|
17
|
+
|
|
18
|
+
define_method(:perform) do |sqs_msg, _body|
|
|
19
|
+
message_counter.increment(Array(sqs_msg).size)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
queue_name = DT.queue
|
|
24
|
+
|
|
25
|
+
create_test_queue(queue_name)
|
|
26
|
+
Shoryuken.add_group('default', 1)
|
|
27
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
28
|
+
worker_class.get_shoryuken_options['queue'] = queue_name
|
|
29
|
+
worker_class.get_shoryuken_options['batch'] = true
|
|
30
|
+
Shoryuken.register_worker(queue_name, worker_class)
|
|
31
|
+
|
|
32
|
+
entries = 10.times.map { |i| { id: SecureRandom.uuid, message_body: i.to_s } }
|
|
33
|
+
Shoryuken::Client.queues(queue_name).send_messages(entries: entries)
|
|
34
|
+
|
|
35
|
+
# Give the messages a chance to hit the queue
|
|
36
|
+
sleep 2
|
|
37
|
+
|
|
38
|
+
poll_queues_until { message_counter.value > 0 }
|
|
39
|
+
|
|
40
|
+
assert(message_counter.value > 1, "Expected more than 1 message in batch, got #{message_counter.value}")
|