action_subscriber 4.5.1-java → 5.0.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/action_subscriber.rb +13 -2
- data/lib/action_subscriber/babou.rb +2 -1
- data/lib/action_subscriber/bunny/subscriber.rb +5 -47
- data/lib/action_subscriber/march_hare/subscriber.rb +10 -56
- data/lib/action_subscriber/message_retry.rb +1 -1
- data/lib/action_subscriber/rabbit_connection.rb +8 -32
- data/lib/action_subscriber/route.rb +7 -5
- data/lib/action_subscriber/route_set.rb +42 -2
- data/lib/action_subscriber/router.rb +12 -7
- data/lib/action_subscriber/thread_pools.rb +34 -0
- data/lib/action_subscriber/version.rb +1 -1
- data/spec/integration/automatic_reconnect_spec.rb +1 -1
- data/spec/spec_helper.rb +1 -1
- metadata +3 -4
- data/spec/integration/multiple_connections_spec.rb +0 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 72df8ebbbbeb14673f2ad50955e8afc42a4c8241
|
4
|
+
data.tar.gz: 0bdb966e111cf767ae90a8c176f25377c1c8dd46
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 14c897e069f09039b6c791d7d8f80a848ca05294dec383f0f1838f5fbd1f17dd4ed54ee3b3f202bfee6e35a517e8ac7bdf79769a27e11bf2da7c1e22103b2188
|
7
|
+
data.tar.gz: ac7fb71fe7317d2dd30ce954e93d7e1badbb5db79764b2c2279084bb7a5747c662aaafda2689e498b44ae11eda502c764c99849e93dee123284967be25c96635
|
data/lib/action_subscriber.rb
CHANGED
@@ -21,6 +21,7 @@ require "action_subscriber/message_retry"
|
|
21
21
|
require "action_subscriber/middleware"
|
22
22
|
require "action_subscriber/rabbit_connection"
|
23
23
|
require "action_subscriber/subscribable"
|
24
|
+
require "action_subscriber/thread_pools"
|
24
25
|
require "action_subscriber/bunny/subscriber"
|
25
26
|
require "action_subscriber/march_hare/subscriber"
|
26
27
|
require "action_subscriber/babou"
|
@@ -47,6 +48,16 @@ module ActionSubscriber
|
|
47
48
|
@draw_routes_block = block
|
48
49
|
end
|
49
50
|
|
51
|
+
def self.print_deprecation_warning(specific_warning)
|
52
|
+
logger.info ("#"*50)
|
53
|
+
logger.info ("# DEPRECATION NOTICE ")
|
54
|
+
logger.info ("# #{specific_warning}")
|
55
|
+
logger.info ("# The usage of multiple connections and the :concurrency setting have been deprecated in favor of using threadpools")
|
56
|
+
logger.info ("# Please see https://github.com/mxenabled/action_subscriber#connections-deprecated for details")
|
57
|
+
logger.info ("# If this change is a problem for your usage of action_subscriber please let us know here: https://github.com/mxenabled/action_subscriber/issues/92")
|
58
|
+
logger.info ("#"*50)
|
59
|
+
end
|
60
|
+
|
50
61
|
def self.print_subscriptions
|
51
62
|
logger.info configuration.inspect
|
52
63
|
route_set.print_subscriptions
|
@@ -56,8 +67,8 @@ module ActionSubscriber
|
|
56
67
|
route_set.print_threadpool_stats
|
57
68
|
end
|
58
69
|
|
59
|
-
def self.
|
60
|
-
::ActionSubscriber::
|
70
|
+
def self.setup_default_threadpool!
|
71
|
+
::ActionSubscriber::ThreadPools.setup_threadpool(:default, {})
|
61
72
|
end
|
62
73
|
|
63
74
|
def self.setup_subscriptions!
|
@@ -5,7 +5,7 @@ module ActionSubscriber
|
|
5
5
|
#
|
6
6
|
def self.start_subscribers
|
7
7
|
reload_active_record
|
8
|
-
::ActionSubscriber.
|
8
|
+
::ActionSubscriber.setup_default_threadpool!
|
9
9
|
::ActionSubscriber.setup_subscriptions!
|
10
10
|
::ActionSubscriber.print_subscriptions
|
11
11
|
::ActionSubscriber.start_subscribers!
|
@@ -21,6 +21,7 @@ module ActionSubscriber
|
|
21
21
|
logger.info "Shutting down"
|
22
22
|
::ActionSubscriber::RabbitConnection.subscriber_disconnect!
|
23
23
|
logger.info "Shutdown complete"
|
24
|
+
exit(0)
|
24
25
|
end
|
25
26
|
|
26
27
|
def self.logger
|
@@ -9,42 +9,8 @@ module ActionSubscriber
|
|
9
9
|
|
10
10
|
def cancel_consumers!
|
11
11
|
bunny_consumers.each(&:cancel)
|
12
|
-
|
13
|
-
|
14
|
-
def print_subscriptions
|
15
|
-
routes.group_by(&:subscriber).each do |subscriber, routes|
|
16
|
-
logger.info subscriber.name
|
17
|
-
routes.each do |route|
|
18
|
-
logger.info " -- method: #{route.action}"
|
19
|
-
logger.info " -- connection: #{route.connection_name}"
|
20
|
-
logger.info " -- concurrency: #{route.concurrency}"
|
21
|
-
logger.info " -- exchange: #{route.exchange}"
|
22
|
-
logger.info " -- queue: #{route.queue}"
|
23
|
-
logger.info " -- routing_key: #{route.routing_key}"
|
24
|
-
logger.info " -- prefetch: #{route.prefetch}"
|
25
|
-
logger.error "WARNING having a prefetch lower than your concurrency will prevent your subscriber from fully utilizing its threadpool" if route.prefetch < route.concurrency
|
26
|
-
if route.acknowledgements != subscriber.acknowledge_messages?
|
27
|
-
logger.error "WARNING subscriber has acknowledgements as #{subscriber.acknowledge_messages?} and route has acknowledgements as #{route.acknowledgements}"
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def print_threadpool_stats
|
34
|
-
logger.info "*DISCLAIMER* the number of running jobs is just a best guess. We don't have a good way to introspect the bunny threadpools so jobs that are sleeping or waiting on IO won't show up as running"
|
35
|
-
subscriptions.group_by{|subscription| subscription[:route].subscriber}.each do |subscriber, subscriptions|
|
36
|
-
logger.info subscriber.name
|
37
|
-
subscriptions.each do |subscription|
|
38
|
-
route = subscription[:route]
|
39
|
-
work_pool = subscription[:queue].channel.work_pool
|
40
|
-
running_threads = work_pool.threads.select{|thread| thread.status == "run"}.count
|
41
|
-
routes.each do |route|
|
42
|
-
logger.info " -- method: #{route.action}"
|
43
|
-
logger.info " -- concurrency: #{route.concurrency}"
|
44
|
-
logger.info " -- running jobs: #{running_threads}"
|
45
|
-
logger.info " -- backlog: #{work_pool.backlog}"
|
46
|
-
end
|
47
|
-
end
|
12
|
+
::ActionSubscriber::ThreadPools.threadpools.each do |name, threadpool|
|
13
|
+
threadpool.shutdown
|
48
14
|
end
|
49
15
|
end
|
50
16
|
|
@@ -63,6 +29,7 @@ module ActionSubscriber
|
|
63
29
|
route = subscription[:route]
|
64
30
|
queue = subscription[:queue]
|
65
31
|
channel = queue.channel
|
32
|
+
threadpool = ::ActionSubscriber::ThreadPools.threadpools.fetch(route.threadpool_name)
|
66
33
|
channel.prefetch(route.prefetch) if route.acknowledgements?
|
67
34
|
consumer = ::Bunny::Consumer.new(channel, queue, channel.generate_consumer_tag, !route.acknowledgements?)
|
68
35
|
consumer.on_delivery do |delivery_info, properties, encoded_payload|
|
@@ -79,26 +46,17 @@ module ActionSubscriber
|
|
79
46
|
:queue => queue.name,
|
80
47
|
}
|
81
48
|
env = ::ActionSubscriber::Middleware::Env.new(route.subscriber, encoded_payload, properties)
|
82
|
-
run_env(env)
|
49
|
+
run_env(env, threadpool)
|
83
50
|
end
|
84
51
|
bunny_consumers << consumer
|
85
52
|
queue.subscribe_with(consumer)
|
86
53
|
end
|
87
54
|
end
|
88
55
|
|
89
|
-
def wait_to_finish_with_timeout(timeout)
|
90
|
-
puts <<-MSG
|
91
|
-
Currently bunny doesn't have any sort of a graceful shutdown or
|
92
|
-
the ability to check on the status of its ConsumerWorkPool objects.
|
93
|
-
For now we just wait for #{timeout}sec to let the worker pools drain.
|
94
|
-
MSG
|
95
|
-
sleep(timeout)
|
96
|
-
end
|
97
|
-
|
98
56
|
private
|
99
57
|
|
100
58
|
def setup_queue(route)
|
101
|
-
channel = ::ActionSubscriber::RabbitConnection.with_connection
|
59
|
+
channel = ::ActionSubscriber::RabbitConnection.with_connection{|connection| connection.create_channel(nil, 1) }
|
102
60
|
exchange = channel.topic(route.exchange)
|
103
61
|
queue = channel.queue(route.queue, :durable => route.durable)
|
104
62
|
queue.bind(exchange, :routing_key => route.routing_key)
|
@@ -5,49 +5,22 @@ module ActionSubscriber
|
|
5
5
|
|
6
6
|
def cancel_consumers!
|
7
7
|
march_hare_consumers.each(&:cancel)
|
8
|
+
::ActionSubscriber::ThreadPools.threadpools.each do |name, threadpool|
|
9
|
+
threadpool.shutdown
|
10
|
+
end
|
8
11
|
end
|
9
12
|
|
10
13
|
def march_hare_consumers
|
11
14
|
@march_hare_consumers ||= []
|
12
15
|
end
|
13
16
|
|
14
|
-
def print_subscriptions
|
15
|
-
routes.group_by(&:subscriber).each do |subscriber, routes|
|
16
|
-
logger.info subscriber.name
|
17
|
-
routes.each do |route|
|
18
|
-
executor = ::ActionSubscriber::RabbitConnection.connection_threadpools[route.connection_name]
|
19
|
-
logger.info " -- method: #{route.action}"
|
20
|
-
logger.info " -- connection: #{route.connection_name} (#{executor.get_maximum_pool_size} threads)"
|
21
|
-
logger.info " -- concurrency: #{route.concurrency}"
|
22
|
-
logger.info " -- exchange: #{route.exchange}"
|
23
|
-
logger.info " -- queue: #{route.queue}"
|
24
|
-
logger.info " -- routing_key: #{route.routing_key}"
|
25
|
-
logger.info " -- prefetch: #{route.prefetch} per consumer (#{route.prefetch * route.concurrency} total)"
|
26
|
-
if route.acknowledgements != subscriber.acknowledge_messages?
|
27
|
-
logger.error "WARNING subscriber has acknowledgements as #{subscriber.acknowledge_messages?} and route has acknowledgements as #{route.acknowledgements}"
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def print_threadpool_stats
|
34
|
-
::ActionSubscriber::RabbitConnection.connection_threadpools.each do |name, executor|
|
35
|
-
logger.info "Connection #{name}"
|
36
|
-
logger.info " -- available threads: #{executor.get_maximum_pool_size}"
|
37
|
-
logger.info " -- running thread: #{executor.get_active_count}"
|
38
|
-
logger.info " -- backlog: #{executor.get_queue.size}"
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
17
|
def setup_subscriptions!
|
43
18
|
fail ::RuntimeError, "you cannot setup queues multiple times, this should only happen once at startup" unless subscriptions.empty?
|
44
19
|
routes.each do |route|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
}
|
50
|
-
end
|
20
|
+
subscriptions << {
|
21
|
+
:route => route,
|
22
|
+
:queue => setup_queue(route),
|
23
|
+
}
|
51
24
|
end
|
52
25
|
end
|
53
26
|
|
@@ -56,6 +29,7 @@ module ActionSubscriber
|
|
56
29
|
route = subscription[:route]
|
57
30
|
queue = subscription[:queue]
|
58
31
|
queue.channel.prefetch = route.prefetch if route.acknowledgements?
|
32
|
+
threadpool = ::ActionSubscriber::ThreadPools.threadpools.fetch(route.threadpool_name)
|
59
33
|
consumer = queue.subscribe(route.queue_subscription_options) do |metadata, encoded_payload|
|
60
34
|
::ActiveSupport::Notifications.instrument "received_event.action_subscriber", :payload_size => encoded_payload.bytesize, :queue => queue.name
|
61
35
|
properties = {
|
@@ -70,37 +44,17 @@ module ActionSubscriber
|
|
70
44
|
:queue => queue.name,
|
71
45
|
}
|
72
46
|
env = ::ActionSubscriber::Middleware::Env.new(route.subscriber, encoded_payload, properties)
|
73
|
-
run_env(env)
|
47
|
+
run_env(env, threadpool)
|
74
48
|
end
|
75
49
|
|
76
50
|
march_hare_consumers << consumer
|
77
51
|
end
|
78
52
|
end
|
79
53
|
|
80
|
-
def wait_to_finish_with_timeout(timeout)
|
81
|
-
wait_loops = 0
|
82
|
-
loop do
|
83
|
-
wait_loops = wait_loops + 1
|
84
|
-
any_threadpools_busy = false
|
85
|
-
::ActionSubscriber::RabbitConnection.connection_threadpools.each do |name, executor|
|
86
|
-
next if executor.get_active_count <= 0
|
87
|
-
logger.info " -- Connection #{name} (active: #{executor.get_active_count}, queued: #{executor.get_queue.size})"
|
88
|
-
any_threadpools_busy = true
|
89
|
-
end
|
90
|
-
if !any_threadpools_busy
|
91
|
-
logger.info "Connection threadpools empty"
|
92
|
-
break
|
93
|
-
end
|
94
|
-
break if wait_loops >= timeout
|
95
|
-
Thread.pass
|
96
|
-
sleep 1
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
54
|
private
|
101
55
|
|
102
56
|
def setup_queue(route)
|
103
|
-
channel = ::ActionSubscriber::RabbitConnection.with_connection
|
57
|
+
channel = ::ActionSubscriber::RabbitConnection.with_connection{|connection| connection.create_channel }
|
104
58
|
exchange = channel.topic(route.exchange)
|
105
59
|
queue = channel.queue(route.queue, :durable => route.durable)
|
106
60
|
queue.bind(exchange, :routing_key => route.routing_key)
|
@@ -46,7 +46,7 @@ module ActionSubscriber
|
|
46
46
|
end
|
47
47
|
|
48
48
|
def self.with_exchange(env, ttl, retry_queue_name)
|
49
|
-
channel = RabbitConnection.with_connection
|
49
|
+
channel = RabbitConnection.with_connection{|connection| connection.create_channel}
|
50
50
|
begin
|
51
51
|
channel.confirm_select
|
52
52
|
# an empty string is the default exchange [see bunny docs](http://rubybunny.info/articles/exchanges.html#default_exchange)
|
@@ -5,46 +5,27 @@ module ActionSubscriber
|
|
5
5
|
SUBSCRIBER_CONNECTION_MUTEX = ::Mutex.new
|
6
6
|
NETWORK_RECOVERY_INTERVAL = 1.freeze
|
7
7
|
|
8
|
-
def self.connection_threadpools
|
9
|
-
if ::RUBY_PLATFORM == "java"
|
10
|
-
subscriber_connections.each_with_object({}) do |(name, connection), hash|
|
11
|
-
hash[name] = connection.instance_variable_get("@executor")
|
12
|
-
end
|
13
|
-
else
|
14
|
-
[] # TODO can I get a hold of the thredpool that bunny uses?
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def self.setup_connection(name, settings)
|
19
|
-
SUBSCRIBER_CONNECTION_MUTEX.synchronize do
|
20
|
-
fail ArgumentError, "a #{name} connection already exists" if subscriber_connections[name]
|
21
|
-
subscriber_connections[name] = create_connection(settings)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
8
|
def self.subscriber_connected?
|
26
|
-
|
27
|
-
subscriber_connections.all?{|_name, connection| connection.connected?}
|
28
|
-
end
|
9
|
+
with_connection{|connection| connection.connected? }
|
29
10
|
end
|
30
11
|
|
31
12
|
def self.subscriber_disconnect!
|
32
13
|
SUBSCRIBER_CONNECTION_MUTEX.synchronize do
|
33
|
-
|
34
|
-
@
|
14
|
+
@subscriber_connection.close if @subscriber_connection
|
15
|
+
@subscriber_connection = nil
|
35
16
|
end
|
36
17
|
end
|
37
18
|
|
38
|
-
def self.with_connection
|
19
|
+
def self.with_connection
|
39
20
|
SUBSCRIBER_CONNECTION_MUTEX.synchronize do
|
40
|
-
|
41
|
-
yield(
|
21
|
+
@subscriber_connection ||= create_connection
|
22
|
+
yield(@subscriber_connection)
|
42
23
|
end
|
43
24
|
end
|
44
25
|
|
45
26
|
# Private API
|
46
|
-
def self.create_connection
|
47
|
-
options = connection_options
|
27
|
+
def self.create_connection
|
28
|
+
options = connection_options
|
48
29
|
if ::RUBY_PLATFORM == "java"
|
49
30
|
options[:executor_factory] = ::Proc.new do
|
50
31
|
::MarchHare::ThreadPools.fixed_of_size(options[:threadpool_size])
|
@@ -79,10 +60,5 @@ module ActionSubscriber
|
|
79
60
|
}
|
80
61
|
end
|
81
62
|
private_class_method :connection_options
|
82
|
-
|
83
|
-
def self.subscriber_connections
|
84
|
-
@subscriber_connections ||= {}
|
85
|
-
end
|
86
|
-
private_class_method :subscriber_connections
|
87
63
|
end
|
88
64
|
end
|
@@ -2,27 +2,29 @@ module ActionSubscriber
|
|
2
2
|
class Route
|
3
3
|
attr_reader :acknowledgements,
|
4
4
|
:action,
|
5
|
-
:concurrency,
|
6
|
-
:connection_name,
|
7
5
|
:durable,
|
8
6
|
:exchange,
|
9
7
|
:prefetch,
|
10
8
|
:queue,
|
11
9
|
:routing_key,
|
12
10
|
:subscriber,
|
13
|
-
:
|
11
|
+
:threadpool_name
|
14
12
|
|
15
13
|
def initialize(attributes)
|
16
14
|
@acknowledgements = attributes.fetch(:acknowledgements)
|
17
15
|
@action = attributes.fetch(:action)
|
18
|
-
@concurrency = attributes.fetch(:concurrency, 1)
|
19
|
-
@connection_name = attributes.fetch(:connection_name)
|
20
16
|
@durable = attributes.fetch(:durable)
|
21
17
|
@exchange = attributes.fetch(:exchange).to_s
|
22
18
|
@prefetch = attributes.fetch(:prefetch) { ::ActionSubscriber.config.prefetch }
|
23
19
|
@queue = attributes.fetch(:queue)
|
24
20
|
@routing_key = attributes.fetch(:routing_key)
|
25
21
|
@subscriber = attributes.fetch(:subscriber)
|
22
|
+
@threadpool_name = attributes.fetch(:threadpool_name)
|
23
|
+
if attributes.has_key?(:concurrency)
|
24
|
+
concurrency = attributes[:concurrency]
|
25
|
+
::ActionSubscriber.print_deprecation_warning("setting prefetch for #{@queue} to #{concurrency}")
|
26
|
+
@prefetch = concurrency
|
27
|
+
end
|
26
28
|
end
|
27
29
|
|
28
30
|
def acknowledgements?
|
@@ -12,16 +12,56 @@ module ActionSubscriber
|
|
12
12
|
@routes = routes
|
13
13
|
end
|
14
14
|
|
15
|
+
def print_subscriptions
|
16
|
+
routes.group_by(&:subscriber).each do |subscriber, routes|
|
17
|
+
logger.info subscriber.name
|
18
|
+
routes.each do |route|
|
19
|
+
threadpool = ::ActionSubscriber::ThreadPools.threadpools[route.threadpool_name]
|
20
|
+
logger.info " -- method: #{route.action}"
|
21
|
+
logger.info " -- threadpool: #{route.threadpool_name} (#{threadpool.max_length} threads)"
|
22
|
+
logger.info " -- exchange: #{route.exchange}"
|
23
|
+
logger.info " -- queue: #{route.queue}"
|
24
|
+
logger.info " -- routing_key: #{route.routing_key}"
|
25
|
+
logger.info " -- prefetch: #{route.prefetch}"
|
26
|
+
if route.acknowledgements != subscriber.acknowledge_messages?
|
27
|
+
logger.error "WARNING subscriber has acknowledgements as #{subscriber.acknowledge_messages?} and route has acknowledgements as #{route.acknowledgements}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def print_threadpool_stats
|
34
|
+
::ActionSubscriber::ThreadPools.threadpools.each do |name, threadpool|
|
35
|
+
logger.info "Threadpool #{name}"
|
36
|
+
logger.info " -- available threads: #{threadpool.length}"
|
37
|
+
logger.info " -- backlog: #{threadpool.queue_length}"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def wait_to_finish_with_timeout(timeout)
|
42
|
+
::ActionSubscriber::ThreadPools.threadpools.map do |name, threadpool|
|
43
|
+
logger.info " -- Threadpool #{name} (queued: #{threadpool.queue_length})"
|
44
|
+
::Thread.new do
|
45
|
+
completed = threadpool.wait_for_termination(timeout)
|
46
|
+
unless completed
|
47
|
+
logger.error " -- FAILED #{name} did not finish shutting down within #{timeout}sec"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end.each(&:join)
|
51
|
+
end
|
52
|
+
|
15
53
|
private
|
16
54
|
|
17
55
|
def subscriptions
|
18
56
|
@subscriptions ||= []
|
19
57
|
end
|
20
58
|
|
21
|
-
def run_env(env)
|
59
|
+
def run_env(env, threadpool)
|
22
60
|
logger.info "RECEIVED #{env.message_id} from #{env.queue}"
|
23
61
|
::ActiveSupport::Notifications.instrument "process_event.action_subscriber", :subscriber => env.subscriber.to_s, :routing_key => env.routing_key, :queue => env.queue do
|
24
|
-
|
62
|
+
threadpool << lambda do
|
63
|
+
::ActionSubscriber.config.middleware.call(env)
|
64
|
+
end
|
25
65
|
end
|
26
66
|
end
|
27
67
|
end
|
@@ -13,14 +13,19 @@ module ActionSubscriber
|
|
13
13
|
}.freeze
|
14
14
|
|
15
15
|
def initialize
|
16
|
-
@
|
16
|
+
@current_threadpool_name = :default
|
17
17
|
end
|
18
18
|
|
19
|
-
def connection(name, settings)
|
20
|
-
::ActionSubscriber
|
21
|
-
|
19
|
+
def connection(name, settings, &block)
|
20
|
+
::ActionSubscriber.print_deprecation_warning("setting up a threadpool for #{name} instead of a new connection")
|
21
|
+
threadpool(name, settings, &block)
|
22
|
+
end
|
23
|
+
|
24
|
+
def threadpool(name, settings)
|
25
|
+
::ActionSubscriber::ThreadPools.setup_threadpool(name, settings)
|
26
|
+
@current_threadpool_name = name
|
22
27
|
yield
|
23
|
-
@
|
28
|
+
@current_threadpool_name = :default
|
24
29
|
end
|
25
30
|
|
26
31
|
def default_routing_key_for(route_settings)
|
@@ -41,7 +46,7 @@ module ActionSubscriber
|
|
41
46
|
end
|
42
47
|
|
43
48
|
def default_routes_for(subscriber, options = {})
|
44
|
-
options = options.merge({:
|
49
|
+
options = options.merge({:threadpool_name => @current_threadpool_name})
|
45
50
|
subscriber.routes(options).each do |route|
|
46
51
|
routes << route
|
47
52
|
end
|
@@ -66,7 +71,7 @@ module ActionSubscriber
|
|
66
71
|
end
|
67
72
|
|
68
73
|
def route(subscriber, action, options = {})
|
69
|
-
route_settings = DEFAULT_SETTINGS.merge(:
|
74
|
+
route_settings = DEFAULT_SETTINGS.merge(:threadpool_name => @current_threadpool_name).merge(options).merge(:subscriber => subscriber, :action => action)
|
70
75
|
route_settings[:routing_key] ||= default_routing_key_for(route_settings)
|
71
76
|
route_settings[:queue] ||= default_queue_for(route_settings)
|
72
77
|
routes << Route.new(route_settings)
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require "concurrent"
|
2
|
+
require "thread"
|
3
|
+
|
4
|
+
module ActionSubscriber
|
5
|
+
module ThreadPools
|
6
|
+
MUTEX = ::Mutex.new
|
7
|
+
THREADPOOL_DEFAULTS = {
|
8
|
+
:auto_terminate => true,
|
9
|
+
:fallback_policy => :caller_runs,
|
10
|
+
:max_queue => 10_000,
|
11
|
+
}.freeze
|
12
|
+
|
13
|
+
def self.threadpools
|
14
|
+
MUTEX.synchronize do
|
15
|
+
@threadpools ||= {}
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.setup_threadpool(name, settings)
|
20
|
+
MUTEX.synchronize do
|
21
|
+
@threadpools ||= {}
|
22
|
+
fail ArgumentError, "a #{name} threadpool already exists" if @threadpools.has_key?(name)
|
23
|
+
@threadpools[name] = create_threadpool(settings)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.create_threadpool(settings)
|
28
|
+
settings = THREADPOOL_DEFAULTS.merge(settings)
|
29
|
+
num_threads = settings.delete(:threadpool_size) || ::ActionSubscriber.configuration.threadpool_size
|
30
|
+
::Concurrent::FixedThreadPool.new(num_threads, settings)
|
31
|
+
end
|
32
|
+
private_class_method :create_threadpool
|
33
|
+
end
|
34
|
+
end
|
@@ -25,7 +25,7 @@ describe "Automatically reconnect on connection failure", :integration => true,
|
|
25
25
|
close_all_connections!
|
26
26
|
sleep 5.0
|
27
27
|
verify_expectation_within(5.0) do
|
28
|
-
expect(::ActionSubscriber::RabbitConnection.with_connection
|
28
|
+
expect(::ActionSubscriber::RabbitConnection.with_connection{|connection| connection.open?}).to eq(true)
|
29
29
|
end
|
30
30
|
|
31
31
|
::ActivePublisher.publish("gus.spoke", "Second", "events")
|
data/spec/spec_helper.rb
CHANGED
@@ -15,7 +15,7 @@ require 'action_subscriber/rspec'
|
|
15
15
|
# Silence the Logger
|
16
16
|
$TESTING = true
|
17
17
|
::ActionSubscriber::Logging.initialize_logger(nil)
|
18
|
-
::ActionSubscriber.
|
18
|
+
::ActionSubscriber.setup_default_threadpool!
|
19
19
|
|
20
20
|
RSpec.configure do |config|
|
21
21
|
config.mock_with :rspec do |mocks|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: action_subscriber
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 5.0.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Brian Stien
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2017-
|
15
|
+
date: 2017-10-24 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
requirement: !ruby/object:Gem::Requirement
|
@@ -234,6 +234,7 @@ files:
|
|
234
234
|
- lib/action_subscriber/router.rb
|
235
235
|
- lib/action_subscriber/rspec.rb
|
236
236
|
- lib/action_subscriber/subscribable.rb
|
237
|
+
- lib/action_subscriber/thread_pools.rb
|
237
238
|
- lib/action_subscriber/uri.rb
|
238
239
|
- lib/action_subscriber/version.rb
|
239
240
|
- routing.md
|
@@ -246,7 +247,6 @@ files:
|
|
246
247
|
- spec/integration/custom_headers_spec.rb
|
247
248
|
- spec/integration/decoding_payloads_spec.rb
|
248
249
|
- spec/integration/manual_acknowledgement_spec.rb
|
249
|
-
- spec/integration/multiple_connections_spec.rb
|
250
250
|
- spec/lib/action_subscriber/base_spec.rb
|
251
251
|
- spec/lib/action_subscriber/configuration_spec.rb
|
252
252
|
- spec/lib/action_subscriber/dsl_spec.rb
|
@@ -296,7 +296,6 @@ test_files:
|
|
296
296
|
- spec/integration/custom_headers_spec.rb
|
297
297
|
- spec/integration/decoding_payloads_spec.rb
|
298
298
|
- spec/integration/manual_acknowledgement_spec.rb
|
299
|
-
- spec/integration/multiple_connections_spec.rb
|
300
299
|
- spec/lib/action_subscriber/base_spec.rb
|
301
300
|
- spec/lib/action_subscriber/configuration_spec.rb
|
302
301
|
- spec/lib/action_subscriber/dsl_spec.rb
|
@@ -1,36 +0,0 @@
|
|
1
|
-
class MultipleConnectionsSubscriber < ::ActionSubscriber::Base
|
2
|
-
MUTEX = ::Mutex.new
|
3
|
-
at_least_once!
|
4
|
-
|
5
|
-
def burp
|
6
|
-
MUTEX.synchronize do
|
7
|
-
$messages << payload
|
8
|
-
end
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
describe "Separate connections to get multiple threadpools", :integration => true do
|
13
|
-
let(:draw_routes) do
|
14
|
-
::ActionSubscriber.draw_routes do
|
15
|
-
connection(:background_work, :thread_pool_size => 20) do
|
16
|
-
route MultipleConnectionsSubscriber, :burp,
|
17
|
-
:acknowledgements => true,
|
18
|
-
:concurrency => 20
|
19
|
-
end
|
20
|
-
route MultipleConnectionsSubscriber, :burp,
|
21
|
-
:acknowledgements => true,
|
22
|
-
:concurrency => 8 # match the default threadpool size
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
it "spreads the load across multiple threadpools and consumer" do
|
27
|
-
::ActionSubscriber.start_subscribers!
|
28
|
-
1.upto(10).each do |i|
|
29
|
-
::ActivePublisher.publish("multiple_connections.burp", "belch#{i}", "events")
|
30
|
-
end
|
31
|
-
|
32
|
-
verify_expectation_within(5.0) do
|
33
|
-
expect($messages).to eq(Set.new(["belch1", "belch2", "belch3", "belch4", "belch5", "belch6", "belch7", "belch8", "belch9", "belch10"]))
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|