active_publisher 1.2.5 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/lib/active_publisher/async/in_memory_adapter/async_queue.rb +30 -21
- data/lib/active_publisher/async/in_memory_adapter/consumer_thread.rb +3 -2
- data/lib/active_publisher/async/redis_adapter/consumer.rb +19 -14
- data/lib/active_publisher/async/redis_adapter.rb +5 -3
- data/lib/active_publisher/configuration.rb +4 -0
- data/lib/active_publisher/version.rb +1 -1
- data/lib/active_publisher.rb +1 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ccd0042255e498e41af848250980e92a2d51dec064906e25f88c0c20421935fe
|
4
|
+
data.tar.gz: 67180d5ef6503fac7ad5c421fc3c5db56c9adeaf656e440dc8cc52a1cac09fd0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3698729fa111230b7ae7093b42e4a62755aea8a9dbfde132f87c60d4710cf697377183c4f8b22d86707b0ee1c94e19a1cddae1a25c2ee664c5f4d961b62a53c5
|
7
|
+
data.tar.gz: c04783629ca01dfb09fe090f0081d92a111f82225c215fe00652e99fa9c49e8038932711bbca4be7c601c105cce9d1a69d6c99b91b5cafb7f958ac7caa574722
|
data/README.md
CHANGED
@@ -55,7 +55,9 @@ Defaults for the configuration are:
|
|
55
55
|
:network_recovery_interval => 1,
|
56
56
|
:password => "guest",
|
57
57
|
:port => 5672,
|
58
|
+
:publisher_threads => 1,
|
58
59
|
:publisher_confirms => false,
|
60
|
+
:publisher_confirms_timeout => 5_000,
|
59
61
|
:seconds_to_wait_for_graceful_shutdown => 30,
|
60
62
|
:timeout => 1,
|
61
63
|
:tls => false,
|
@@ -14,14 +14,15 @@ module ActivePublisher
|
|
14
14
|
:max_queue_size,
|
15
15
|
:supervisor_interval
|
16
16
|
|
17
|
-
attr_reader :
|
17
|
+
attr_reader :consumers, :queue, :supervisor
|
18
18
|
|
19
19
|
def initialize(back_pressure_strategy, max_queue_size, supervisor_interval)
|
20
20
|
self.back_pressure_strategy = back_pressure_strategy
|
21
21
|
@max_queue_size = max_queue_size
|
22
22
|
@supervisor_interval = supervisor_interval
|
23
23
|
@queue = ::MultiOpQueue::Queue.new
|
24
|
-
|
24
|
+
@consumers = {}
|
25
|
+
create_and_supervise_consumers!
|
25
26
|
end
|
26
27
|
|
27
28
|
def back_pressure_strategy=(strategy)
|
@@ -52,36 +53,44 @@ module ActivePublisher
|
|
52
53
|
def size
|
53
54
|
# Requests might be in flight (out of the queue, but not yet published), so taking the max should be
|
54
55
|
# good enough to make sure we're honest about the actual queue size.
|
55
|
-
return queue.size if
|
56
|
-
[queue.size,
|
56
|
+
return queue.size if consumers.empty?
|
57
|
+
[queue.size, consumer_sampled_queue_size].max
|
57
58
|
end
|
58
59
|
|
59
60
|
private
|
60
61
|
|
61
|
-
def
|
62
|
-
|
62
|
+
def create_and_supervise_consumers!
|
63
|
+
::ActivePublisher.configuration.publisher_threads.times do
|
64
|
+
consumer_id = ::SecureRandom.uuid
|
65
|
+
consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
66
|
+
supervisor_task = ::Concurrent::TimerTask.new(:execution_interval => supervisor_interval) do
|
67
|
+
current_time = ::Time.now
|
68
|
+
consumer = consumers[consumer_id]
|
63
69
|
|
64
|
-
|
65
|
-
|
70
|
+
# Consumer is lagging if it does not "tick" at least once every 10 seconds.
|
71
|
+
seconds_since_last_tick = current_time - consumer.last_tick_at
|
72
|
+
consumer_is_lagging = seconds_since_last_tick > ::ActivePublisher.configuration.max_async_publisher_lag_time
|
73
|
+
logger.error "ActivePublisher consumer is lagging. Last consumer tick was #{seconds_since_last_tick} seconds ago." if consumer_is_lagging
|
66
74
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
75
|
+
# Check to see if we should restart the consumer.
|
76
|
+
if !consumer.alive? || consumer_is_lagging
|
77
|
+
consumer.kill rescue nil
|
78
|
+
consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
79
|
+
::ActiveSupport::Notifications.instrument "async_queue.thread_restart"
|
80
|
+
end
|
71
81
|
|
72
|
-
|
73
|
-
|
74
|
-
consumer.kill rescue nil
|
75
|
-
@consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
82
|
+
# Notify the current queue size.
|
83
|
+
::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
|
76
84
|
end
|
77
|
-
|
78
|
-
# Notify the current queue size.
|
79
|
-
::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
|
85
|
+
supervisor_task.execute
|
80
86
|
end
|
81
|
-
supervisor_task.execute
|
82
87
|
end
|
83
|
-
end
|
84
88
|
|
89
|
+
def consumer_sampled_queue_size
|
90
|
+
consumers.values.map(&:sampled_queue_size).max
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
85
94
|
end
|
86
95
|
end
|
87
96
|
end
|
@@ -2,7 +2,7 @@ module ActivePublisher
|
|
2
2
|
module Async
|
3
3
|
module InMemoryAdapter
|
4
4
|
class ConsumerThread
|
5
|
-
attr_reader :channel, :thread, :queue, :sampled_queue_size, :last_tick_at
|
5
|
+
attr_reader :channel, :flush_max, :thread, :queue, :sampled_queue_size, :last_tick_at
|
6
6
|
|
7
7
|
if ::RUBY_PLATFORM == "java"
|
8
8
|
CHANNEL_CLOSED_ERRORS = [::MarchHare::ChannelAlreadyClosed]
|
@@ -27,6 +27,7 @@ module ActivePublisher
|
|
27
27
|
def initialize(listen_queue)
|
28
28
|
@queue = listen_queue
|
29
29
|
@sampled_queue_size = queue.size
|
30
|
+
@flush_max = ::ActivePublisher.configuration.messages_per_batch
|
30
31
|
|
31
32
|
update_last_tick_at
|
32
33
|
start_thread
|
@@ -96,7 +97,7 @@ module ActivePublisher
|
|
96
97
|
loop do
|
97
98
|
# Sample the queue size so we don't shutdown when messages are in flight.
|
98
99
|
@sampled_queue_size = queue.size
|
99
|
-
current_messages = queue.pop_up_to(
|
100
|
+
current_messages = queue.pop_up_to(flush_max, :timeout => 0.1)
|
100
101
|
update_last_tick_at
|
101
102
|
# If the queue is empty, we should continue to update to "last_tick_at" time.
|
102
103
|
next if current_messages.nil?
|
@@ -9,28 +9,33 @@ module ActivePublisher
|
|
9
9
|
:timeout_interval => 5, # seconds
|
10
10
|
}
|
11
11
|
|
12
|
-
attr_reader :
|
12
|
+
attr_reader :consumers, :queue, :supervisor
|
13
13
|
|
14
14
|
def initialize(redis_pool)
|
15
15
|
@queue = ::ActivePublisher::Async::RedisAdapter::RedisMultiPopQueue.new(redis_pool, ::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY)
|
16
|
-
|
16
|
+
@consumers = {}
|
17
|
+
create_and_supervise_consumers!
|
17
18
|
end
|
18
19
|
|
19
|
-
def
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
consumer
|
26
|
-
|
20
|
+
def create_and_supervise_consumers!
|
21
|
+
::ActivePublisher.configuration.publisher_threads.times do
|
22
|
+
consumer_id = ::SecureRandom.uuid
|
23
|
+
consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
24
|
+
|
25
|
+
supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
|
26
|
+
consumer = consumers[consumer_id]
|
27
|
+
unless consumer.alive?
|
28
|
+
consumer.kill rescue nil
|
29
|
+
consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
30
|
+
::ActiveSupport::Notifications.instrument "async_queue.thread_restart"
|
31
|
+
end
|
32
|
+
|
33
|
+
# Notify the current queue size.
|
34
|
+
::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
|
27
35
|
end
|
28
36
|
|
29
|
-
|
30
|
-
::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
|
37
|
+
supervisor_task.execute
|
31
38
|
end
|
32
|
-
|
33
|
-
supervisor_task.execute
|
34
39
|
end
|
35
40
|
|
36
41
|
def size
|
@@ -19,7 +19,7 @@ module ActivePublisher
|
|
19
19
|
}
|
20
20
|
include ::ActivePublisher::Logging
|
21
21
|
|
22
|
-
attr_reader :async_queue, :redis_pool, :queue
|
22
|
+
attr_reader :async_queue, :flush_max, :flush_min, :redis_pool, :queue
|
23
23
|
|
24
24
|
def initialize(new_redis_pool)
|
25
25
|
logger.info "Starting redis publisher adapter"
|
@@ -27,6 +27,8 @@ module ActivePublisher
|
|
27
27
|
@redis_pool = new_redis_pool
|
28
28
|
@async_queue = ::ActivePublisher::Async::RedisAdapter::Consumer.new(redis_pool)
|
29
29
|
@queue = ::MultiOpQueue::Queue.new
|
30
|
+
@flush_max = ::ActivePublisher.configuration.messages_per_batch
|
31
|
+
@flush_min = @flush_max / 2
|
30
32
|
|
31
33
|
supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
|
32
34
|
queue_size = queue.size
|
@@ -41,7 +43,7 @@ module ActivePublisher
|
|
41
43
|
def publish(route, payload, exchange_name, options = {})
|
42
44
|
message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
|
43
45
|
queue << ::Marshal.dump(message)
|
44
|
-
flush_queue! if queue.size >=
|
46
|
+
flush_queue! if queue.size >= flush_min || options[:flush_queue]
|
45
47
|
|
46
48
|
nil
|
47
49
|
end
|
@@ -58,7 +60,7 @@ module ActivePublisher
|
|
58
60
|
|
59
61
|
def flush_queue!
|
60
62
|
return if queue.empty?
|
61
|
-
encoded_messages = queue.pop_up_to(
|
63
|
+
encoded_messages = queue.pop_up_to(flush_max, :timeout => 0.001)
|
62
64
|
|
63
65
|
return if encoded_messages.nil?
|
64
66
|
return unless encoded_messages.respond_to?(:each)
|
@@ -8,9 +8,11 @@ module ActivePublisher
|
|
8
8
|
:host,
|
9
9
|
:hosts,
|
10
10
|
:max_async_publisher_lag_time,
|
11
|
+
:messages_per_batch,
|
11
12
|
:network_recovery_interval,
|
12
13
|
:password,
|
13
14
|
:port,
|
15
|
+
:publisher_threads,
|
14
16
|
:publisher_confirms,
|
15
17
|
:publisher_confirms_timeout,
|
16
18
|
:seconds_to_wait_for_graceful_shutdown,
|
@@ -36,9 +38,11 @@ module ActivePublisher
|
|
36
38
|
:host => "localhost",
|
37
39
|
:hosts => [],
|
38
40
|
:password => "guest",
|
41
|
+
:messages_per_batch => 25,
|
39
42
|
:max_async_publisher_lag_time => 10,
|
40
43
|
:network_recovery_interval => NETWORK_RECOVERY_INTERVAL,
|
41
44
|
:port => 5672,
|
45
|
+
:publisher_threads => 1,
|
42
46
|
:publisher_confirms => false,
|
43
47
|
:publisher_confirms_timeout => 5_000, #specified as a number of milliseconds
|
44
48
|
:seconds_to_wait_for_graceful_shutdown => 30,
|
data/lib/active_publisher.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_publisher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Stien
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: exe
|
14
14
|
cert_chain: []
|
15
|
-
date:
|
15
|
+
date: 2021-10-19 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: bunny
|
@@ -223,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
223
223
|
- !ruby/object:Gem::Version
|
224
224
|
version: '0'
|
225
225
|
requirements: []
|
226
|
-
rubygems_version: 3.
|
226
|
+
rubygems_version: 3.2.28
|
227
227
|
signing_key:
|
228
228
|
specification_version: 4
|
229
229
|
summary: Aims to make publishing work across MRI and jRuby painless and add some nice
|