action_subscriber 4.5.1 → 5.0.0.pre0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/action_subscriber/babou.rb +2 -1
- data/lib/action_subscriber/bunny/subscriber.rb +5 -47
- data/lib/action_subscriber/march_hare/subscriber.rb +10 -56
- data/lib/action_subscriber/message_retry.rb +1 -1
- data/lib/action_subscriber/rabbit_connection.rb +8 -32
- data/lib/action_subscriber/route.rb +7 -5
- data/lib/action_subscriber/route_set.rb +42 -2
- data/lib/action_subscriber/router.rb +12 -7
- data/lib/action_subscriber/thread_pools.rb +34 -0
- data/lib/action_subscriber/version.rb +1 -1
- data/lib/action_subscriber.rb +13 -2
- data/spec/integration/automatic_reconnect_spec.rb +1 -1
- data/spec/spec_helper.rb +1 -1
- metadata +5 -6
- data/spec/integration/multiple_connections_spec.rb +0 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c7d90f8bf0d2f27b2da875ddec284a9925f91331
|
4
|
+
data.tar.gz: 53b08c262c00deaf0efe1e28ef557201e7e9a469
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d5e2e940738dc058be7a8566ac12431cd0723143939ee845a6dd1cea9148feeb530155ecbda3a20278f2f6343a130cabe372758cfcb4390c61d4ac33563eb4eb
|
7
|
+
data.tar.gz: 2af092b3486426282771e5f99416836301516047e29809833de223ea00f1d91672878b5d22f6c670864ea0402f9a88afa795d31e9fa35d7c4cac935c84e02ba2
|
@@ -5,7 +5,7 @@ module ActionSubscriber
|
|
5
5
|
#
|
6
6
|
def self.start_subscribers
|
7
7
|
reload_active_record
|
8
|
-
::ActionSubscriber.
|
8
|
+
::ActionSubscriber.setup_default_threadpool!
|
9
9
|
::ActionSubscriber.setup_subscriptions!
|
10
10
|
::ActionSubscriber.print_subscriptions
|
11
11
|
::ActionSubscriber.start_subscribers!
|
@@ -21,6 +21,7 @@ module ActionSubscriber
|
|
21
21
|
logger.info "Shutting down"
|
22
22
|
::ActionSubscriber::RabbitConnection.subscriber_disconnect!
|
23
23
|
logger.info "Shutdown complete"
|
24
|
+
exit(0)
|
24
25
|
end
|
25
26
|
|
26
27
|
def self.logger
|
@@ -9,42 +9,8 @@ module ActionSubscriber
|
|
9
9
|
|
10
10
|
def cancel_consumers!
|
11
11
|
bunny_consumers.each(&:cancel)
|
12
|
-
|
13
|
-
|
14
|
-
def print_subscriptions
|
15
|
-
routes.group_by(&:subscriber).each do |subscriber, routes|
|
16
|
-
logger.info subscriber.name
|
17
|
-
routes.each do |route|
|
18
|
-
logger.info " -- method: #{route.action}"
|
19
|
-
logger.info " -- connection: #{route.connection_name}"
|
20
|
-
logger.info " -- concurrency: #{route.concurrency}"
|
21
|
-
logger.info " -- exchange: #{route.exchange}"
|
22
|
-
logger.info " -- queue: #{route.queue}"
|
23
|
-
logger.info " -- routing_key: #{route.routing_key}"
|
24
|
-
logger.info " -- prefetch: #{route.prefetch}"
|
25
|
-
logger.error "WARNING having a prefetch lower than your concurrency will prevent your subscriber from fully utilizing its threadpool" if route.prefetch < route.concurrency
|
26
|
-
if route.acknowledgements != subscriber.acknowledge_messages?
|
27
|
-
logger.error "WARNING subscriber has acknowledgements as #{subscriber.acknowledge_messages?} and route has acknowledgements as #{route.acknowledgements}"
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def print_threadpool_stats
|
34
|
-
logger.info "*DISCLAIMER* the number of running jobs is just a best guess. We don't have a good way to introspect the bunny threadpools so jobs that are sleeping or waiting on IO won't show up as running"
|
35
|
-
subscriptions.group_by{|subscription| subscription[:route].subscriber}.each do |subscriber, subscriptions|
|
36
|
-
logger.info subscriber.name
|
37
|
-
subscriptions.each do |subscription|
|
38
|
-
route = subscription[:route]
|
39
|
-
work_pool = subscription[:queue].channel.work_pool
|
40
|
-
running_threads = work_pool.threads.select{|thread| thread.status == "run"}.count
|
41
|
-
routes.each do |route|
|
42
|
-
logger.info " -- method: #{route.action}"
|
43
|
-
logger.info " -- concurrency: #{route.concurrency}"
|
44
|
-
logger.info " -- running jobs: #{running_threads}"
|
45
|
-
logger.info " -- backlog: #{work_pool.backlog}"
|
46
|
-
end
|
47
|
-
end
|
12
|
+
::ActionSubscriber::ThreadPools.threadpools.each do |name, threadpool|
|
13
|
+
threadpool.shutdown
|
48
14
|
end
|
49
15
|
end
|
50
16
|
|
@@ -63,6 +29,7 @@ module ActionSubscriber
|
|
63
29
|
route = subscription[:route]
|
64
30
|
queue = subscription[:queue]
|
65
31
|
channel = queue.channel
|
32
|
+
threadpool = ::ActionSubscriber::ThreadPools.threadpools.fetch(route.threadpool_name)
|
66
33
|
channel.prefetch(route.prefetch) if route.acknowledgements?
|
67
34
|
consumer = ::Bunny::Consumer.new(channel, queue, channel.generate_consumer_tag, !route.acknowledgements?)
|
68
35
|
consumer.on_delivery do |delivery_info, properties, encoded_payload|
|
@@ -79,26 +46,17 @@ module ActionSubscriber
|
|
79
46
|
:queue => queue.name,
|
80
47
|
}
|
81
48
|
env = ::ActionSubscriber::Middleware::Env.new(route.subscriber, encoded_payload, properties)
|
82
|
-
run_env(env)
|
49
|
+
run_env(env, threadpool)
|
83
50
|
end
|
84
51
|
bunny_consumers << consumer
|
85
52
|
queue.subscribe_with(consumer)
|
86
53
|
end
|
87
54
|
end
|
88
55
|
|
89
|
-
def wait_to_finish_with_timeout(timeout)
|
90
|
-
puts <<-MSG
|
91
|
-
Currently bunny doesn't have any sort of a graceful shutdown or
|
92
|
-
the ability to check on the status of its ConsumerWorkPool objects.
|
93
|
-
For now we just wait for #{timeout}sec to let the worker pools drain.
|
94
|
-
MSG
|
95
|
-
sleep(timeout)
|
96
|
-
end
|
97
|
-
|
98
56
|
private
|
99
57
|
|
100
58
|
def setup_queue(route)
|
101
|
-
channel = ::ActionSubscriber::RabbitConnection.with_connection
|
59
|
+
channel = ::ActionSubscriber::RabbitConnection.with_connection{|connection| connection.create_channel(nil, 1) }
|
102
60
|
exchange = channel.topic(route.exchange)
|
103
61
|
queue = channel.queue(route.queue, :durable => route.durable)
|
104
62
|
queue.bind(exchange, :routing_key => route.routing_key)
|
@@ -5,49 +5,22 @@ module ActionSubscriber
|
|
5
5
|
|
6
6
|
def cancel_consumers!
|
7
7
|
march_hare_consumers.each(&:cancel)
|
8
|
+
::ActionSubscriber::ThreadPools.threadpools.each do |name, threadpool|
|
9
|
+
threadpool.shutdown
|
10
|
+
end
|
8
11
|
end
|
9
12
|
|
10
13
|
def march_hare_consumers
|
11
14
|
@march_hare_consumers ||= []
|
12
15
|
end
|
13
16
|
|
14
|
-
def print_subscriptions
|
15
|
-
routes.group_by(&:subscriber).each do |subscriber, routes|
|
16
|
-
logger.info subscriber.name
|
17
|
-
routes.each do |route|
|
18
|
-
executor = ::ActionSubscriber::RabbitConnection.connection_threadpools[route.connection_name]
|
19
|
-
logger.info " -- method: #{route.action}"
|
20
|
-
logger.info " -- connection: #{route.connection_name} (#{executor.get_maximum_pool_size} threads)"
|
21
|
-
logger.info " -- concurrency: #{route.concurrency}"
|
22
|
-
logger.info " -- exchange: #{route.exchange}"
|
23
|
-
logger.info " -- queue: #{route.queue}"
|
24
|
-
logger.info " -- routing_key: #{route.routing_key}"
|
25
|
-
logger.info " -- prefetch: #{route.prefetch} per consumer (#{route.prefetch * route.concurrency} total)"
|
26
|
-
if route.acknowledgements != subscriber.acknowledge_messages?
|
27
|
-
logger.error "WARNING subscriber has acknowledgements as #{subscriber.acknowledge_messages?} and route has acknowledgements as #{route.acknowledgements}"
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def print_threadpool_stats
|
34
|
-
::ActionSubscriber::RabbitConnection.connection_threadpools.each do |name, executor|
|
35
|
-
logger.info "Connection #{name}"
|
36
|
-
logger.info " -- available threads: #{executor.get_maximum_pool_size}"
|
37
|
-
logger.info " -- running thread: #{executor.get_active_count}"
|
38
|
-
logger.info " -- backlog: #{executor.get_queue.size}"
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
17
|
def setup_subscriptions!
|
43
18
|
fail ::RuntimeError, "you cannot setup queues multiple times, this should only happen once at startup" unless subscriptions.empty?
|
44
19
|
routes.each do |route|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
}
|
50
|
-
end
|
20
|
+
subscriptions << {
|
21
|
+
:route => route,
|
22
|
+
:queue => setup_queue(route),
|
23
|
+
}
|
51
24
|
end
|
52
25
|
end
|
53
26
|
|
@@ -56,6 +29,7 @@ module ActionSubscriber
|
|
56
29
|
route = subscription[:route]
|
57
30
|
queue = subscription[:queue]
|
58
31
|
queue.channel.prefetch = route.prefetch if route.acknowledgements?
|
32
|
+
threadpool = ::ActionSubscriber::ThreadPools.threadpools.fetch(route.threadpool_name)
|
59
33
|
consumer = queue.subscribe(route.queue_subscription_options) do |metadata, encoded_payload|
|
60
34
|
::ActiveSupport::Notifications.instrument "received_event.action_subscriber", :payload_size => encoded_payload.bytesize, :queue => queue.name
|
61
35
|
properties = {
|
@@ -70,37 +44,17 @@ module ActionSubscriber
|
|
70
44
|
:queue => queue.name,
|
71
45
|
}
|
72
46
|
env = ::ActionSubscriber::Middleware::Env.new(route.subscriber, encoded_payload, properties)
|
73
|
-
run_env(env)
|
47
|
+
run_env(env, threadpool)
|
74
48
|
end
|
75
49
|
|
76
50
|
march_hare_consumers << consumer
|
77
51
|
end
|
78
52
|
end
|
79
53
|
|
80
|
-
def wait_to_finish_with_timeout(timeout)
|
81
|
-
wait_loops = 0
|
82
|
-
loop do
|
83
|
-
wait_loops = wait_loops + 1
|
84
|
-
any_threadpools_busy = false
|
85
|
-
::ActionSubscriber::RabbitConnection.connection_threadpools.each do |name, executor|
|
86
|
-
next if executor.get_active_count <= 0
|
87
|
-
logger.info " -- Connection #{name} (active: #{executor.get_active_count}, queued: #{executor.get_queue.size})"
|
88
|
-
any_threadpools_busy = true
|
89
|
-
end
|
90
|
-
if !any_threadpools_busy
|
91
|
-
logger.info "Connection threadpools empty"
|
92
|
-
break
|
93
|
-
end
|
94
|
-
break if wait_loops >= timeout
|
95
|
-
Thread.pass
|
96
|
-
sleep 1
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
54
|
private
|
101
55
|
|
102
56
|
def setup_queue(route)
|
103
|
-
channel = ::ActionSubscriber::RabbitConnection.with_connection
|
57
|
+
channel = ::ActionSubscriber::RabbitConnection.with_connection{|connection| connection.create_channel }
|
104
58
|
exchange = channel.topic(route.exchange)
|
105
59
|
queue = channel.queue(route.queue, :durable => route.durable)
|
106
60
|
queue.bind(exchange, :routing_key => route.routing_key)
|
@@ -46,7 +46,7 @@ module ActionSubscriber
|
|
46
46
|
end
|
47
47
|
|
48
48
|
def self.with_exchange(env, ttl, retry_queue_name)
|
49
|
-
channel = RabbitConnection.with_connection
|
49
|
+
channel = RabbitConnection.with_connection{|connection| connection.create_channel}
|
50
50
|
begin
|
51
51
|
channel.confirm_select
|
52
52
|
# an empty string is the default exchange [see bunny docs](http://rubybunny.info/articles/exchanges.html#default_exchange)
|
@@ -5,46 +5,27 @@ module ActionSubscriber
|
|
5
5
|
SUBSCRIBER_CONNECTION_MUTEX = ::Mutex.new
|
6
6
|
NETWORK_RECOVERY_INTERVAL = 1.freeze
|
7
7
|
|
8
|
-
def self.connection_threadpools
|
9
|
-
if ::RUBY_PLATFORM == "java"
|
10
|
-
subscriber_connections.each_with_object({}) do |(name, connection), hash|
|
11
|
-
hash[name] = connection.instance_variable_get("@executor")
|
12
|
-
end
|
13
|
-
else
|
14
|
-
[] # TODO can I get a hold of the thredpool that bunny uses?
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def self.setup_connection(name, settings)
|
19
|
-
SUBSCRIBER_CONNECTION_MUTEX.synchronize do
|
20
|
-
fail ArgumentError, "a #{name} connection already exists" if subscriber_connections[name]
|
21
|
-
subscriber_connections[name] = create_connection(settings)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
8
|
def self.subscriber_connected?
|
26
|
-
|
27
|
-
subscriber_connections.all?{|_name, connection| connection.connected?}
|
28
|
-
end
|
9
|
+
with_connection{|connection| connection.connected? }
|
29
10
|
end
|
30
11
|
|
31
12
|
def self.subscriber_disconnect!
|
32
13
|
SUBSCRIBER_CONNECTION_MUTEX.synchronize do
|
33
|
-
|
34
|
-
@
|
14
|
+
@subscriber_connection.close if @subscriber_connection
|
15
|
+
@subscriber_connection = nil
|
35
16
|
end
|
36
17
|
end
|
37
18
|
|
38
|
-
def self.with_connection
|
19
|
+
def self.with_connection
|
39
20
|
SUBSCRIBER_CONNECTION_MUTEX.synchronize do
|
40
|
-
|
41
|
-
yield(
|
21
|
+
@subscriber_connection ||= create_connection
|
22
|
+
yield(@subscriber_connection)
|
42
23
|
end
|
43
24
|
end
|
44
25
|
|
45
26
|
# Private API
|
46
|
-
def self.create_connection
|
47
|
-
options = connection_options
|
27
|
+
def self.create_connection
|
28
|
+
options = connection_options
|
48
29
|
if ::RUBY_PLATFORM == "java"
|
49
30
|
options[:executor_factory] = ::Proc.new do
|
50
31
|
::MarchHare::ThreadPools.fixed_of_size(options[:threadpool_size])
|
@@ -79,10 +60,5 @@ module ActionSubscriber
|
|
79
60
|
}
|
80
61
|
end
|
81
62
|
private_class_method :connection_options
|
82
|
-
|
83
|
-
def self.subscriber_connections
|
84
|
-
@subscriber_connections ||= {}
|
85
|
-
end
|
86
|
-
private_class_method :subscriber_connections
|
87
63
|
end
|
88
64
|
end
|
@@ -2,27 +2,29 @@ module ActionSubscriber
|
|
2
2
|
class Route
|
3
3
|
attr_reader :acknowledgements,
|
4
4
|
:action,
|
5
|
-
:concurrency,
|
6
|
-
:connection_name,
|
7
5
|
:durable,
|
8
6
|
:exchange,
|
9
7
|
:prefetch,
|
10
8
|
:queue,
|
11
9
|
:routing_key,
|
12
10
|
:subscriber,
|
13
|
-
:
|
11
|
+
:threadpool_name
|
14
12
|
|
15
13
|
def initialize(attributes)
|
16
14
|
@acknowledgements = attributes.fetch(:acknowledgements)
|
17
15
|
@action = attributes.fetch(:action)
|
18
|
-
@concurrency = attributes.fetch(:concurrency, 1)
|
19
|
-
@connection_name = attributes.fetch(:connection_name)
|
20
16
|
@durable = attributes.fetch(:durable)
|
21
17
|
@exchange = attributes.fetch(:exchange).to_s
|
22
18
|
@prefetch = attributes.fetch(:prefetch) { ::ActionSubscriber.config.prefetch }
|
23
19
|
@queue = attributes.fetch(:queue)
|
24
20
|
@routing_key = attributes.fetch(:routing_key)
|
25
21
|
@subscriber = attributes.fetch(:subscriber)
|
22
|
+
@threadpool_name = attributes.fetch(:threadpool_name)
|
23
|
+
if attributes.has_key?(:concurrency)
|
24
|
+
concurrency = attributes[:concurrency]
|
25
|
+
::ActionSubscriber.print_deprecation_warning("setting prefetch for #{@queue} to #{concurrency}")
|
26
|
+
@prefetch = concurrency
|
27
|
+
end
|
26
28
|
end
|
27
29
|
|
28
30
|
def acknowledgements?
|
@@ -12,16 +12,56 @@ module ActionSubscriber
|
|
12
12
|
@routes = routes
|
13
13
|
end
|
14
14
|
|
15
|
+
def print_subscriptions
|
16
|
+
routes.group_by(&:subscriber).each do |subscriber, routes|
|
17
|
+
logger.info subscriber.name
|
18
|
+
routes.each do |route|
|
19
|
+
threadpool = ::ActionSubscriber::ThreadPools.threadpools[route.threadpool_name]
|
20
|
+
logger.info " -- method: #{route.action}"
|
21
|
+
logger.info " -- threadpool: #{route.threadpool_name} (#{threadpool.max_length} threads)"
|
22
|
+
logger.info " -- exchange: #{route.exchange}"
|
23
|
+
logger.info " -- queue: #{route.queue}"
|
24
|
+
logger.info " -- routing_key: #{route.routing_key}"
|
25
|
+
logger.info " -- prefetch: #{route.prefetch}"
|
26
|
+
if route.acknowledgements != subscriber.acknowledge_messages?
|
27
|
+
logger.error "WARNING subscriber has acknowledgements as #{subscriber.acknowledge_messages?} and route has acknowledgements as #{route.acknowledgements}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def print_threadpool_stats
|
34
|
+
::ActionSubscriber::ThreadPools.threadpools.each do |name, threadpool|
|
35
|
+
logger.info "Threadpool #{name}"
|
36
|
+
logger.info " -- available threads: #{threadpool.length}"
|
37
|
+
logger.info " -- backlog: #{threadpool.queue_length}"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def wait_to_finish_with_timeout(timeout)
|
42
|
+
::ActionSubscriber::ThreadPools.threadpools.map do |name, threadpool|
|
43
|
+
logger.info " -- Threadpool #{name} (queued: #{threadpool.queue_length})"
|
44
|
+
::Thread.new do
|
45
|
+
completed = threadpool.wait_for_termination(timeout)
|
46
|
+
unless completed
|
47
|
+
logger.error " -- FAILED #{name} did not finish shutting down within #{timeout}sec"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end.each(&:join)
|
51
|
+
end
|
52
|
+
|
15
53
|
private
|
16
54
|
|
17
55
|
def subscriptions
|
18
56
|
@subscriptions ||= []
|
19
57
|
end
|
20
58
|
|
21
|
-
def run_env(env)
|
59
|
+
def run_env(env, threadpool)
|
22
60
|
logger.info "RECEIVED #{env.message_id} from #{env.queue}"
|
23
61
|
::ActiveSupport::Notifications.instrument "process_event.action_subscriber", :subscriber => env.subscriber.to_s, :routing_key => env.routing_key, :queue => env.queue do
|
24
|
-
|
62
|
+
threadpool << lambda do
|
63
|
+
::ActionSubscriber.config.middleware.call(env)
|
64
|
+
end
|
25
65
|
end
|
26
66
|
end
|
27
67
|
end
|
@@ -13,14 +13,19 @@ module ActionSubscriber
|
|
13
13
|
}.freeze
|
14
14
|
|
15
15
|
def initialize
|
16
|
-
@
|
16
|
+
@current_threadpool_name = :default
|
17
17
|
end
|
18
18
|
|
19
|
-
def connection(name, settings)
|
20
|
-
::ActionSubscriber
|
21
|
-
|
19
|
+
def connection(name, settings, &block)
|
20
|
+
::ActionSubscriber.print_deprecation_warning("setting up a threadpool for #{name} instead of a new connection")
|
21
|
+
threadpool(name, settings, &block)
|
22
|
+
end
|
23
|
+
|
24
|
+
def threadpool(name, settings)
|
25
|
+
::ActionSubscriber::ThreadPools.setup_threadpool(name, settings)
|
26
|
+
@current_threadpool_name = name
|
22
27
|
yield
|
23
|
-
@
|
28
|
+
@current_threadpool_name = :default
|
24
29
|
end
|
25
30
|
|
26
31
|
def default_routing_key_for(route_settings)
|
@@ -41,7 +46,7 @@ module ActionSubscriber
|
|
41
46
|
end
|
42
47
|
|
43
48
|
def default_routes_for(subscriber, options = {})
|
44
|
-
options = options.merge({:
|
49
|
+
options = options.merge({:threadpool_name => @current_threadpool_name})
|
45
50
|
subscriber.routes(options).each do |route|
|
46
51
|
routes << route
|
47
52
|
end
|
@@ -66,7 +71,7 @@ module ActionSubscriber
|
|
66
71
|
end
|
67
72
|
|
68
73
|
def route(subscriber, action, options = {})
|
69
|
-
route_settings = DEFAULT_SETTINGS.merge(:
|
74
|
+
route_settings = DEFAULT_SETTINGS.merge(:threadpool_name => @current_threadpool_name).merge(options).merge(:subscriber => subscriber, :action => action)
|
70
75
|
route_settings[:routing_key] ||= default_routing_key_for(route_settings)
|
71
76
|
route_settings[:queue] ||= default_queue_for(route_settings)
|
72
77
|
routes << Route.new(route_settings)
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require "concurrent"
|
2
|
+
require "thread"
|
3
|
+
|
4
|
+
module ActionSubscriber
|
5
|
+
module ThreadPools
|
6
|
+
MUTEX = ::Mutex.new
|
7
|
+
THREADPOOL_DEFAULTS = {
|
8
|
+
:auto_terminate => true,
|
9
|
+
:fallback_policy => :caller_runs,
|
10
|
+
:max_queue => 10_000,
|
11
|
+
}.freeze
|
12
|
+
|
13
|
+
def self.threadpools
|
14
|
+
MUTEX.synchronize do
|
15
|
+
@threadpools ||= {}
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.setup_threadpool(name, settings)
|
20
|
+
MUTEX.synchronize do
|
21
|
+
@threadpools ||= {}
|
22
|
+
fail ArgumentError, "a #{name} threadpool already exists" if @threadpools.has_key?(name)
|
23
|
+
@threadpools[name] = create_threadpool(settings)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.create_threadpool(settings)
|
28
|
+
settings = THREADPOOL_DEFAULTS.merge(settings)
|
29
|
+
num_threads = settings.delete(:threadpool_size) || ::ActionSubscriber.configuration.threadpool_size
|
30
|
+
::Concurrent::FixedThreadPool.new(num_threads, settings)
|
31
|
+
end
|
32
|
+
private_class_method :create_threadpool
|
33
|
+
end
|
34
|
+
end
|
data/lib/action_subscriber.rb
CHANGED
@@ -21,6 +21,7 @@ require "action_subscriber/message_retry"
|
|
21
21
|
require "action_subscriber/middleware"
|
22
22
|
require "action_subscriber/rabbit_connection"
|
23
23
|
require "action_subscriber/subscribable"
|
24
|
+
require "action_subscriber/thread_pools"
|
24
25
|
require "action_subscriber/bunny/subscriber"
|
25
26
|
require "action_subscriber/march_hare/subscriber"
|
26
27
|
require "action_subscriber/babou"
|
@@ -47,6 +48,16 @@ module ActionSubscriber
|
|
47
48
|
@draw_routes_block = block
|
48
49
|
end
|
49
50
|
|
51
|
+
def self.print_deprecation_warning(specific_warning)
|
52
|
+
logger.info ("#"*50)
|
53
|
+
logger.info ("# DEPRECATION NOTICE ")
|
54
|
+
logger.info ("# #{specific_warning}")
|
55
|
+
logger.info ("# The usage of multiple connections and the :concurrency setting have been deprecated in favor of using threadpools")
|
56
|
+
logger.info ("# Please see https://github.com/mxenabled/action_subscriber#connections-deprecated for details")
|
57
|
+
logger.info ("# If this change is a problem for your usage of action_subscriber please let us know here: https://github.com/mxenabled/action_subscriber/issues/92")
|
58
|
+
logger.info ("#"*50)
|
59
|
+
end
|
60
|
+
|
50
61
|
def self.print_subscriptions
|
51
62
|
logger.info configuration.inspect
|
52
63
|
route_set.print_subscriptions
|
@@ -56,8 +67,8 @@ module ActionSubscriber
|
|
56
67
|
route_set.print_threadpool_stats
|
57
68
|
end
|
58
69
|
|
59
|
-
def self.
|
60
|
-
::ActionSubscriber::
|
70
|
+
def self.setup_default_threadpool!
|
71
|
+
::ActionSubscriber::ThreadPools.setup_threadpool(:default, {})
|
61
72
|
end
|
62
73
|
|
63
74
|
def self.setup_subscriptions!
|
@@ -25,7 +25,7 @@ describe "Automatically reconnect on connection failure", :integration => true,
|
|
25
25
|
close_all_connections!
|
26
26
|
sleep 5.0
|
27
27
|
verify_expectation_within(5.0) do
|
28
|
-
expect(::ActionSubscriber::RabbitConnection.with_connection
|
28
|
+
expect(::ActionSubscriber::RabbitConnection.with_connection{|connection| connection.open?}).to eq(true)
|
29
29
|
end
|
30
30
|
|
31
31
|
::ActivePublisher.publish("gus.spoke", "Second", "events")
|
data/spec/spec_helper.rb
CHANGED
@@ -15,7 +15,7 @@ require 'action_subscriber/rspec'
|
|
15
15
|
# Silence the Logger
|
16
16
|
$TESTING = true
|
17
17
|
::ActionSubscriber::Logging.initialize_logger(nil)
|
18
|
-
::ActionSubscriber.
|
18
|
+
::ActionSubscriber.setup_default_threadpool!
|
19
19
|
|
20
20
|
RSpec.configure do |config|
|
21
21
|
config.mock_with :rspec do |mocks|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: action_subscriber
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 5.0.0.pre0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Stien
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2017-
|
15
|
+
date: 2017-10-20 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: activesupport
|
@@ -235,6 +235,7 @@ files:
|
|
235
235
|
- lib/action_subscriber/router.rb
|
236
236
|
- lib/action_subscriber/rspec.rb
|
237
237
|
- lib/action_subscriber/subscribable.rb
|
238
|
+
- lib/action_subscriber/thread_pools.rb
|
238
239
|
- lib/action_subscriber/uri.rb
|
239
240
|
- lib/action_subscriber/version.rb
|
240
241
|
- routing.md
|
@@ -247,7 +248,6 @@ files:
|
|
247
248
|
- spec/integration/custom_headers_spec.rb
|
248
249
|
- spec/integration/decoding_payloads_spec.rb
|
249
250
|
- spec/integration/manual_acknowledgement_spec.rb
|
250
|
-
- spec/integration/multiple_connections_spec.rb
|
251
251
|
- spec/lib/action_subscriber/base_spec.rb
|
252
252
|
- spec/lib/action_subscriber/configuration_spec.rb
|
253
253
|
- spec/lib/action_subscriber/dsl_spec.rb
|
@@ -278,9 +278,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
278
278
|
version: '0'
|
279
279
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
280
280
|
requirements:
|
281
|
-
- - "
|
281
|
+
- - ">"
|
282
282
|
- !ruby/object:Gem::Version
|
283
|
-
version:
|
283
|
+
version: 1.3.1
|
284
284
|
requirements: []
|
285
285
|
rubyforge_project:
|
286
286
|
rubygems_version: 2.6.13
|
@@ -298,7 +298,6 @@ test_files:
|
|
298
298
|
- spec/integration/custom_headers_spec.rb
|
299
299
|
- spec/integration/decoding_payloads_spec.rb
|
300
300
|
- spec/integration/manual_acknowledgement_spec.rb
|
301
|
-
- spec/integration/multiple_connections_spec.rb
|
302
301
|
- spec/lib/action_subscriber/base_spec.rb
|
303
302
|
- spec/lib/action_subscriber/configuration_spec.rb
|
304
303
|
- spec/lib/action_subscriber/dsl_spec.rb
|
@@ -1,36 +0,0 @@
|
|
1
|
-
class MultipleConnectionsSubscriber < ::ActionSubscriber::Base
|
2
|
-
MUTEX = ::Mutex.new
|
3
|
-
at_least_once!
|
4
|
-
|
5
|
-
def burp
|
6
|
-
MUTEX.synchronize do
|
7
|
-
$messages << payload
|
8
|
-
end
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
describe "Separate connections to get multiple threadpools", :integration => true do
|
13
|
-
let(:draw_routes) do
|
14
|
-
::ActionSubscriber.draw_routes do
|
15
|
-
connection(:background_work, :thread_pool_size => 20) do
|
16
|
-
route MultipleConnectionsSubscriber, :burp,
|
17
|
-
:acknowledgements => true,
|
18
|
-
:concurrency => 20
|
19
|
-
end
|
20
|
-
route MultipleConnectionsSubscriber, :burp,
|
21
|
-
:acknowledgements => true,
|
22
|
-
:concurrency => 8 # match the default threadpool size
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
it "spreads the load across multiple threadpools and consumer" do
|
27
|
-
::ActionSubscriber.start_subscribers!
|
28
|
-
1.upto(10).each do |i|
|
29
|
-
::ActivePublisher.publish("multiple_connections.burp", "belch#{i}", "events")
|
30
|
-
end
|
31
|
-
|
32
|
-
verify_expectation_within(5.0) do
|
33
|
-
expect($messages).to eq(Set.new(["belch1", "belch2", "belch3", "belch4", "belch5", "belch6", "belch7", "belch8", "belch9", "belch10"]))
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|